removeDuplicates function (#140)

* removeDuplicates function

* XML Core vulnerability fix

* License update

* cleanup

* improved test

* try/else bug fix

docs/modules/ROOT/pages/libraries-arrays.adoc

@@ -181,6 +181,66 @@ ds.arrays.duplicates(payload)
 ]
 ------------------------
 
+### `removeDuplicates(array arr, function compF)`
+Removes any duplicates from the array and returns the resulting array. An optional comparator function takes two parameters.
+
+*Example*
+
+.Payload
+----------
+[
+  1,
+  2,
+  2,
+  3,
+  4
+]
+----------
+.DataSonnet map:
+------------------------
+ds.arrays.removeDuplicates(payload)
+------------------------
+.Result
+------------------------
+[ 1, 2, 3, 4]
+------------------------
+
+.Payload
+----------
+[
+    {
+        x: 1, y: "a"
+    },
+    {
+        x: 1, y: "b"
+    },
+    {
+        x: 2, y: "a"
+    },
+    {
+        x: 3, y: "a"
+    }
+]
+----------
+.DataSonnet map:
+------------------------
+ds.arrays.removeDuplicates(payload, function(i1, i2) i1.x == i2.x)
+------------------------
+.Result
+------------------------
+[
+    {
+        x: 1, y: "a"
+    },
+    {
+        x: 2, y: "a"
+    },
+    {
+        x: 3, y: "a"
+    }
+]
+------------------------
+
 ### `every(array arr, function func)`
 Returns true if every value `arr` returns true in `func`.
 

pom.xml

@@ -103,7 +103,7 @@
         <scalatags.version>0.9.1</scalatags.version>
         <slf4j.version>2.0.9</slf4j.version>
         <ujson.version>1.2.0</ujson.version>
-        <xmlunit.version>2.9.1</xmlunit.version>
+        <xmlunit.version>2.10.0</xmlunit.version>
     </properties>
 
     <dependencies>

src/main/scala/com/datasonnet/DS.scala

@@ -1,7 +1,7 @@
 package com.datasonnet
 
 /*-
- * Copyright 2019-2023 the original author or authors.
+ * Copyright 2019-2024 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ import com.datasonnet.jsonnet.Std._
 import com.datasonnet.jsonnet.{Applyer, Error, EvalScope, Expr, FileScope, Materializer, Val}
 import com.datasonnet.modules.{Crypto, JsonPath, Regex}
 import com.datasonnet.spi.{DataFormatService, Library, ujsonUtils}
+import sourcecode.Macros.Chunk
 import ujson.Value
 
 import java.math.{BigDecimal, RoundingMode}
@@ -1543,6 +1544,23 @@ object DSLowercase extends Library {
           Val.Arr(out.toSeq)
       },
 
+      builtinWithDefaults("removeDuplicates", "array" -> None, "compF" -> Some(Expr.False(0))) { (args, ev) =>
+          val array = args("array").asInstanceOf[Val.Arr]
+          val compF = args("compF")
+
+          val out = mutable.Buffer.empty[Val.Lazy]
+          array.value.collect({
+            case item if !out.exists(
+              x => if (compF == Val.False) x.force == item.force else {
+                val compFunc = compF.asInstanceOf[Val.Func]
+                val compApplyer = Applyer(compFunc, ev, null)
+                compApplyer.apply(x, item).asInstanceOf[Val.Bool] == Val.True
+              }
+            ) => out.append(item)
+          })
+          Val.Arr(out.toSeq)
+      },
+
       builtin("every", "value", "funct") {
         (_, _, value: Val, funct: Applyer) =>
           value match {

src/main/scala/com/datasonnet/jsonnet/Evaluator.scala

@@ -1,7 +1,7 @@
 package com.datasonnet.jsonnet
 
 /*-
- * Copyright 2019-2023 the original author or authors.
+ * Copyright 2019-2024 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -262,7 +262,7 @@ class Evaluator(parseCacheP: collection.mutable.Map[String, fastparse.Parsed[(Ex
         .getOrElse(Error.fail("Cannot use `super` outside an object", offset))
         .value(name, offset, scope.self0.get, defaultValue)
     } else visitExpr(value) match {
-      case obj: Val.Obj => if (!tryCatch) obj.value(name, offset, obj, defaultValue) else Error.fail(s"Object does not have a field ${name}", offset)
+      case obj: Val.Obj => obj.value(name, offset, obj, if (tryCatch) null else defaultValue)
       case r => if (defaultValue != null && !tryCatch) Materializer.reverse(defaultValue) else Error.fail(s"attempted to index a ${r.prettyName} with string ${name}", offset)
     }
   }

src/test/java/com/datasonnet/ArraysTest.java

@@ -1,7 +1,7 @@
 package com.datasonnet;
 
 /*-
- * Copyright 2019-2023 the original author or authors.
+ * Copyright 2019-2024 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -372,4 +372,18 @@ void testArrays_takeWhile() {
         assertEquals("[0,1]", value);
     }
 
+    @Test
+    void testArrays_removeDuplicates() throws Exception {
+        Mapper mapper = new Mapper(lib + pack + ".removeDuplicates([0,1,2,1])\n", new ArrayList<>(), new HashMap<>(), true);
+        String value = mapper.transform("{}").replaceAll("\"", "");
+        JSONAssert.assertEquals("[0,1,2]", value, true);
+
+        mapper = new Mapper(lib + pack + ".removeDuplicates([{ x: 1, y: \"a\"},{ x: 1, y: \"b\"},{ x: 2, y: \"a\"},{ x: 3, y: \"a\"}], function(i1, i2) i1.x == i2.x)\n", new ArrayList<>(), new HashMap<>(), true);
+        value = mapper.transform("{}").replaceAll("\"", "");
+        JSONAssert.assertEquals("[{ x: 1, y: \"a\"},{ x: 2, y: \"a\"},{ x: 3, y: \"a\"}]", value, true);
+
+        mapper = new Mapper(lib + pack + ".removeDuplicates([{ x: 1, y: \"a\"},{ x: 1, y: \"a\"},{ x: 2, y: \"a\"},{ x: 3, y: \"a\"}], function(i1, i2) i1.x == i2.x)\n", new ArrayList<>(), new HashMap<>(), true);
+        value = mapper.transform("{}").replaceAll("\"", "");
+        JSONAssert.assertEquals("[{ x: 1, y: \"a\"},{ x: 2, y: \"a\"},{ x: 3, y: \"a\"}]", value, true);
+    }
 }

src/test/java/com/datasonnet/TryElseTest.java

@@ -1,7 +1,7 @@
 package com.datasonnet;
 
 /*-
- * Copyright 2019-2023 the original author or authors.
+ * Copyright 2019-2024 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -56,8 +56,8 @@ void testTryElseErrors() throws IOException, URISyntaxException, JSONException {
     @Test
     void testDefault() throws IOException, URISyntaxException, JSONException {
         Mapper mapper = new Mapper(TestResourceReader.readFileAsString("default.ds"));
-        Document<String>  response = mapper.transform(new DefaultDocument<>("{}", MediaTypes.APPLICATION_JSON));
-        JSONAssert.assertEquals("{\"tryNonexistent\":\"OK\",\"tryChain\":\"OK\",\"tryNaN\":-1}", response.getContent(), true);
+        Document<String>  response = mapper.transform(new DefaultDocument<>("{\"key\":\"value\"}", MediaTypes.APPLICATION_JSON));
+        JSONAssert.assertEquals("{\"tryNonexistent\":\"OK\",\"tryChain\":\"OK\",\"tryNaN\":-1,\"tryArr\":\"emptyArr\",\"tryArr2\":\"nullArr\",\"tryExists\":\"value\"}", response.getContent(), true);
     }
 
     @Test
@@ -73,4 +73,5 @@ void testDefaultHeaderPayloadAsDocument() throws IOException, URISyntaxException
         Document<String>  response = mapper.transform(new DefaultDocument<>("{}", MediaTypes.APPLICATION_JSON));
         JSONAssert.assertEquals("{\"tryNonexistent\":\"OK\",\"tryObj\":{\"x\":\"OK\"},\"tryOverride\":\"OverrideOK\",\"tryElseOverride\":\"OverrideOK\"}", response.getContent(), true);
     }
+
 }

src/test/resources/default.ds

@@ -1,5 +1,8 @@
 {
   tryNonexistent: payload.doesntExist default "OK",
   tryChain: payload.doesntExist default payload.missing default "OK",
-  tryNaN: std.parseInt("NotANumber") default -1
+  tryNaN: std.parseInt("NotANumber") default -1,
+  tryArr: [][0] default "emptyArr",
+  tryArr2: null[0] default "nullArr",
+  tryExists: payload.key default "none"
 }

src/test/resources/tryElseError.ds

@@ -1,3 +1,3 @@
 {
-  tryNonexistent: payload.doesntExist,
+  tryNonexistent: payload.doesntExist
 }