Skip to content

Commit

Permalink
Some YAML fixes (#2178)
Browse files Browse the repository at this point in the history
  • Loading branch information
@soywiz
soywiz committed Mar 7, 2024
1 parent a1dcef3 commit 3967026
Show file tree
Hide file tree
Showing 8 changed files with 150 additions and 25 deletions.
30 changes: 20 additions & 10 deletions korge-core/src/korlibs/serialization/Yaml.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,12 @@ package korlibs.io.serialization.yaml

import korlibs.datastructure.ListReader
import korlibs.io.lang.invalidOp
import korlibs.io.util.StrReader
import korlibs.io.util.unquote
import korlibs.io.util.*
import kotlin.collections.set

object Yaml {
fun decode(str: String) = read(ListReader(tokenize(str)), level = 0)
fun read(str: String) = read(ListReader(tokenize(str)), level = 0)
fun decode(str: String): Any? = read(ListReader(tokenize(str)), level = 0)
fun read(str: String): Any? = read(ListReader(tokenize(str)), level = 0)

private fun parseStr(toks: List<Token>): Any? {
if (toks.size == 1 && toks[0] is Token.STR) return toks[0].ustr
Expand Down Expand Up @@ -123,7 +122,7 @@ object Yaml {
return map ?: list
}

fun ListReader<Token>.readId(): List<Token> {
private fun ListReader<Token>.readId(): List<Token> {
val tokens = arrayListOf<Token>()
while (hasMore) {
val token = peek()
Expand All @@ -137,7 +136,7 @@ object Yaml {
return tokens
}

fun readOrString(s: ListReader<Token>, level: Int, delimiters: Set<String>, supportNonSpaceSymbols: Boolean): Any? {
private fun readOrString(s: ListReader<Token>, level: Int, delimiters: Set<String>, supportNonSpaceSymbols: Boolean): Any? {
val sp = s.peek()
return if (sp is Token.ID || (supportNonSpaceSymbols && sp is Token.SYMBOL && !sp.isNextWhite)) {
var str = ""
Expand Down Expand Up @@ -170,7 +169,7 @@ object Yaml {
linestart@ while (hasMore) {
// Line start
flush()
val indentStr = readWhile(Char::isWhitespace).replace("\t", " ")
val indentStr = readWhile(kotlin.Char::isWhitespace).replace("\t", " ")
if (indentStr.contains('\n')) continue@linestart // ignore empty lines with possible additional indent
val indent = indentStr.length
if (indents.isEmpty() || indent > indents.last()) {
Expand All @@ -189,15 +188,26 @@ object Yaml {
flush(); out += Token.SYMBOL("$c", peekChar())
}
'#' -> {
flush(); readUntilLineEnd(); skip(); continue@linestart
if (str.lastOrNull()?.isWhitespaceFast() == true || (str == "" && out.lastOrNull() is Token.LINE)) {
flush(); readUntilLineEnd(); skip(); continue@linestart
} else {
str += c
}
}
'\n' -> {
flush(); continue@linestart
}
'"', '\'' -> {
flush()
s.unread()
out += Token.STR(s.readStringLit())
val last = out.lastOrNull()
//println("out=$last, c='$c', reader=$this")
if (last is Token.SYMBOL && (last.str == ":" || last.str == "[" || last.str == "{" || last.str == "," || last.str == "-")) {
s.unread()
//println(" -> c='$c', reader=$this")
out += Token.STR(s.readStringLit())
} else {
str += c
}
}
else -> str += c
}
Expand Down
29 changes: 29 additions & 0 deletions korge-core/test/korlibs/io/serialization/yaml/YamlTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -318,4 +318,33 @@ class YamlTest {
- git::adder: :korlibs/kproject::/modules/adder::54f73b01cea9cb2e8368176ac45f2fca948e57db
""".trimIndent()))
}

@Test
fun testSingleQuoteInString() {
assertEquals(
mapOf(
"hello" to "world",
"title" to "What's Happening",
"demo" to listOf("hello", "world", "test", "what's happening", "yeah"),
"dependencies" to listOf(
"https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f",
"https://github.com/korlibs/kproject.git/samples/demo2 #95696dd942ebc8db4ee9d9f4835ce12d853ff16f",
"https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f",
"https://github.com/korlibs/kproject.git/samples/demo2",
),
),
Yaml.decode("""
hello: 'world'
title: What's Happening
demo: ["hello", "world", "test", what's happening, yeah]
# hi
dependencies:
- "https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f"
- "https://github.com/korlibs/kproject.git/samples/demo2 #95696dd942ebc8db4ee9d9f4835ce12d853ff16f"
- https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f
- https://github.com/korlibs/kproject.git/samples/demo2 #95696dd942ebc8db4ee9d9f4835ce12d853ff16f
# hello
""".trimIndent())
)
}
}
1 change: 1 addition & 0 deletions korge-gradle-plugin-common/src/main/kotlin/com/soywiz/kproject/internal/StrReader.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ internal class StrReader(val str: String, var pos: Int = 0) {

fun skip() = skip(1)
fun peek(): Char = if (hasMore) this.str[this.pos] else '\u0000'
fun peekChar(): Char = peek()
fun read(): Char = if (hasMore) this.str[posSkip(1)] else '\u0000'
fun unread() = skip(-1)

Expand Down
25 changes: 18 additions & 7 deletions korge-gradle-plugin-common/src/main/kotlin/com/soywiz/kproject/internal/Yaml.kt
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package com.soywiz.kproject.internal

object Yaml {
fun decode(str: String) = read(ListReader(tokenize(str)), level = 0)
fun read(str: String) = read(ListReader(tokenize(str)), level = 0)
fun decode(str: String): Any? = read(ListReader(tokenize(str)), level = 0)
fun read(str: String): Any? = read(ListReader(tokenize(str)), level = 0)

private fun parseStr(toks: List<Token>): Any? {
if (toks.size == 1 && toks[0] is Token.STR) return toks[0].ustr
Expand Down Expand Up @@ -162,7 +162,7 @@ object Yaml {
linestart@ while (hasMore) {
// Line start
flush()
val indentStr = readWhile(Char::isWhitespace).replace("\t", " ")
val indentStr = readWhile(kotlin.Char::isWhitespace).replace("\t", " ")
if (indentStr.contains('\n')) continue@linestart // ignore empty lines with possible additional indent
val indent = indentStr.length
if (indents.isEmpty() || indent > indents.last()) {
Expand All @@ -178,18 +178,29 @@ object Yaml {
val c = read()
when (c) {
':', '-', '[', ']', ',' -> {
flush(); out += Token.SYMBOL("$c", peek())
flush(); out += Token.SYMBOL("$c", peekChar())
}
'#' -> {
flush(); readUntilLineEnd(); skip(); continue@linestart
if (str.lastOrNull()?.isWhitespaceFast() == true || (str == "" && out.lastOrNull() is Token.LINE)) {
flush(); readUntilLineEnd(); skip(); continue@linestart
} else {
str += c
}
}
'\n' -> {
flush(); continue@linestart
}
'"', '\'' -> {
flush()
s.unread()
out += Token.STR(s.readStringLit())
val last = out.lastOrNull()
//println("out=$last, c='$c', reader=$this")
if (last is Token.SYMBOL && (last.str == ":" || last.str == "[" || last.str == "{" || last.str == "," || last.str == "-")) {
s.unread()
//println(" -> c='$c', reader=$this")
out += Token.STR(s.readStringLit())
} else {
str += c
}
}
else -> str += c
}
Expand Down
29 changes: 29 additions & 0 deletions korge-gradle-plugin-common/src/test/kotlin/com/soywiz/kproject/YamlTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,4 +293,33 @@ class YamlTest {
""".trimIndent())
)
}

@Test
fun testSingleQuoteInString() {
assertEquals(
mapOf(
"hello" to "world",
"title" to "What's Happening",
"demo" to listOf("hello", "world", "test", "what's happening", "yeah"),
"dependencies" to listOf(
"https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f",
"https://github.com/korlibs/kproject.git/samples/demo2 #95696dd942ebc8db4ee9d9f4835ce12d853ff16f",
"https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f",
"https://github.com/korlibs/kproject.git/samples/demo2",
),
),
Yaml.decode("""
hello: 'world'
title: What's Happening
demo: ["hello", "world", "test", what's happening, yeah]
# hi
dependencies:
- "https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f"
- "https://github.com/korlibs/kproject.git/samples/demo2 #95696dd942ebc8db4ee9d9f4835ce12d853ff16f"
- https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f
- https://github.com/korlibs/kproject.git/samples/demo2 #95696dd942ebc8db4ee9d9f4835ce12d853ff16f
# hello
""".trimIndent())
)
}
}
6 changes: 4 additions & 2 deletions korlibs-template/src/korlibs/template/Korte.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,9 @@ open class KorteTemplateConfig(
var unknownFilter: KorteFilter = KorteFilter("unknown") { tok.exception("Unknown filter '$name'") },
val autoEscapeMode: KorteAutoEscapeMode = KorteAutoEscapeMode.HTML,
// Here we can convert markdown into html if required. This is available at the template level + content + named blocks
val contentTypeProcessor: (content: String, contentType: String?) -> String = { content, _ -> content }
val contentTypeProcessor: (content: String, contentType: String?) -> String = { content, _ -> content },
val frontMatterParser: (String) -> Any? = { Yaml.decode(it) },
@Suppress("UNUSED_PARAMETER") dummy: Unit = Unit, // To avoid tailing lambda
) {
val extra = LinkedHashMap<String, Any>()

Expand Down Expand Up @@ -468,7 +470,7 @@ interface KorteBlock : KorteDynamicContext {
val yamlLines = slines.slice(0 until index)
val outside = slines.slice(index + 1 until slines.size)
val yamlText = yamlLines.joinToString("\n")
val yaml = Yaml.read(yamlText)
val yaml = parseContext.config.frontMatterParser(yamlText)
if (yaml is Map<*, *>) {
parseContext.template.frontMatter = yaml as Map<String, Any?>
}
Expand Down
26 changes: 20 additions & 6 deletions korlibs-template/src/korlibs/template/_Template.internal.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,7 @@ internal class StrReader(val str: String, var pos: Int = 0) {

fun skip() = skip(1)
fun peek(): Char = if (hasMore) this.str[this.pos] else '\u0000'
fun peekChar(): Char = peek()
fun read(): Char = if (hasMore) this.str[posSkip(1)] else '\u0000'
fun unread() = skip(-1)

Expand Down Expand Up @@ -363,6 +364,8 @@ internal class StrReader(val str: String, var pos: Int = 0) {
fun skipSpaces() = skipWhile { it.isWhitespaceFast() }
fun readWhile(f: (Char) -> Boolean): String = readBlock { skipWhile(f) }
fun readUntil(f: (Char) -> Boolean): String = readBlock { skipUntil(f) }

override fun toString(): String = "StrReader(str=${str.length}, pos=$pos, range='${str.substring(pos.coerceIn(str.indices), (pos + 10).coerceIn(str.indices)).replace("\n", "\\n")}')"
}

internal fun StrReader.readStringLit(reportErrors: Boolean = true): String {
Expand Down Expand Up @@ -399,8 +402,8 @@ internal fun StrReader.readStringLit(reportErrors: Boolean = true): String {
}

object Yaml {
fun decode(str: String) = read(ListReader(tokenize(str)), level = 0)
fun read(str: String) = read(ListReader(tokenize(str)), level = 0)
fun decode(str: String): Any? = read(ListReader(tokenize(str)), level = 0)
fun read(str: String): Any? = read(ListReader(tokenize(str)), level = 0)

private fun parseStr(toks: List<Token>): Any? {
if (toks.size == 1 && toks[0] is Token.STR) return toks[0].ustr
Expand Down Expand Up @@ -576,18 +579,29 @@ object Yaml {
val c = read()
when (c) {
':', '-', '[', ']', ',' -> {
flush(); out += Token.SYMBOL("$c", peek())
flush(); out += Token.SYMBOL("$c", peekChar())
}
'#' -> {
flush(); readUntilLineEnd(); skip(); continue@linestart
if (str.lastOrNull()?.isWhitespaceFast() == true || (str == "" && out.lastOrNull() is Token.LINE)) {
flush(); readUntilLineEnd(); skip(); continue@linestart
} else {
str += c
}
}
'\n' -> {
flush(); continue@linestart
}
'"', '\'' -> {
flush()
s.unread()
out += Token.STR(s.readStringLit())
val last = out.lastOrNull()
//println("out=$last, c='$c', reader=$this")
if (last is Token.SYMBOL && (last.str == ":" || last.str == "[" || last.str == "{" || last.str == "," || last.str == "-")) {
s.unread()
//println(" -> c='$c', reader=$this")
out += Token.STR(s.readStringLit())
} else {
str += c
}
}
else -> str += c
}
Expand Down
29 changes: 29 additions & 0 deletions korlibs-template/test/korlibs/template/YamlTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -319,4 +319,33 @@ class YamlTest {
- git::adder: :korlibs/kproject::/modules/adder::54f73b01cea9cb2e8368176ac45f2fca948e57db
""".trimIndent()))
}

@Test
fun testSingleQuoteInString() {
assertEquals(
mapOf(
"hello" to "world",
"title" to "What's Happening",
"demo" to listOf("hello", "world", "test", "what's happening", "yeah"),
"dependencies" to listOf(
"https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f",
"https://github.com/korlibs/kproject.git/samples/demo2 #95696dd942ebc8db4ee9d9f4835ce12d853ff16f",
"https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f",
"https://github.com/korlibs/kproject.git/samples/demo2",
),
),
Yaml.decode("""
hello: 'world'
title: What's Happening
demo: ["hello", "world", "test", what's happening, yeah]
# hi
dependencies:
- "https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f"
- "https://github.com/korlibs/kproject.git/samples/demo2 #95696dd942ebc8db4ee9d9f4835ce12d853ff16f"
- https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f
- https://github.com/korlibs/kproject.git/samples/demo2 #95696dd942ebc8db4ee9d9f4835ce12d853ff16f
# hello
""".trimIndent())
)
}
}

0 comments on commit 3967026

Please sign in to comment.