[AGE agtype_util.c] Fix issue #870 regarding orderability and added regression tests

Fixed issue #870
 Odd behavior in context of orderability of different agtypes.

Implemented the solution suggested here:
#870 (comment)

Clearance given here:
#870 (comment)

Author: CapnSpek

regress/expected/agtype.out

@@ -1523,7 +1523,7 @@ SELECT agtype_in('{"bool":true}') < agtype_in('{"bool":true, "null": null}');
 (1 row)
 
 -- Comparisons between types
--- Object < List < String < Boolean < Integer = Float = Numeric < Null
+-- Path < Edge < Vertex < Object < List < String < Boolean < Integer = Float = Numeric < Null
 SELECT agtype_in('1') < agtype_in('null');
  ?column? 
 ----------
@@ -1590,6 +1590,24 @@ SELECT agtype_in('{"bool":true, "integer":1}') < agtype_in('{"bool":true, "integ
  t
 (1 row)
 
+SELECT agtype_in('{"id":0, "label": "v", "properties":{"i":0}}::vertex') < agtype_in('{"bool":true, "i":0}');
+ ?column? 
+----------
+ t
+(1 row)
+
+SELECT agtype_in('{"id":2, "start_id":0, "end_id":1, "label": "e", "properties":{"i":0}}::edge') < agtype_in('{"id":0, "label": "v", "properties":{"i":0}}::vertex');
+ ?column? 
+----------
+ t
+(1 row)
+
+SELECT agtype_in('[{"id": 0, "label": "v", "properties": {"i": 0}}::vertex, {"id": 2, "start_id": 0, "end_id": 1, "label": "e", "properties": {"i": 0}}::edge, {"id": 1, "label": "v", "properties": {"i": 0}}::vertex]::path') < agtype_in('{"id":2, "start_id":0, "end_id":1, "label": "e", "properties":{"i":0}}::edge');
+ ?column? 
+----------
+ t
+(1 row)
+
 SELECT agtype_in('1::numeric') < agtype_in('null');
  ?column? 
 ----------

regress/sql/agtype.sql

@@ -403,7 +403,7 @@ SELECT agtype_in('{"bool":true, "null": null}') = agtype_in('{"null":null, "bool
 SELECT agtype_in('{"bool":true}') < agtype_in('{"bool":true, "null": null}');
 
 -- Comparisons between types
--- Object < List < String < Boolean < Integer = Float = Numeric < Null
+-- Path < Edge < Vertex < Object < List < String < Boolean < Integer = Float = Numeric < Null
 SELECT agtype_in('1') < agtype_in('null');
 SELECT agtype_in('NaN') < agtype_in('null');
 SELECT agtype_in('Infinity') < agtype_in('null');
@@ -415,6 +415,9 @@ SELECT agtype_in('[1,3,5,7,9,11]') < agtype_in('"string"');
 SELECT agtype_in('{"bool":true, "integer":1}') < agtype_in('[1,3,5,7,9,11]');
 SELECT agtype_in('[1, "string"]') < agtype_in('[1, 1]');
 SELECT agtype_in('{"bool":true, "integer":1}') < agtype_in('{"bool":true, "integer":null}');
+SELECT agtype_in('{"id":0, "label": "v", "properties":{"i":0}}::vertex') < agtype_in('{"bool":true, "i":0}');
+SELECT agtype_in('{"id":2, "start_id":0, "end_id":1, "label": "e", "properties":{"i":0}}::edge') < agtype_in('{"id":0, "label": "v", "properties":{"i":0}}::vertex');
+SELECT agtype_in('[{"id": 0, "label": "v", "properties": {"i": 0}}::vertex, {"id": 2, "start_id": 0, "end_id": 1, "label": "e", "properties": {"i": 0}}::edge, {"id": 1, "label": "v", "properties": {"i": 0}}::vertex]::path') < agtype_in('{"id":2, "start_id":0, "end_id":1, "label": "e", "properties":{"i":0}}::edge');
 SELECT agtype_in('1::numeric') < agtype_in('null');
 SELECT agtype_in('true') < agtype_in('1::numeric');
 

src/backend/utils/adt/agtype_util.c

@@ -197,20 +197,42 @@ uint32 get_agtype_length(const agtype_container *agtc, int index)
  */
 static int get_type_sort_priority(enum agtype_value_type type)
 {
-    if (type == AGTV_OBJECT)
+    if (type == AGTV_PATH)
+    {
         return 0;
-    if (type == AGTV_VERTEX)
+    }
+    if (type == AGTV_EDGE)
+    {
         return 1;
-    if (type == AGTV_ARRAY)
+    }
+    if (type == AGTV_VERTEX)
+    {
         return 2;
-    if (type == AGTV_STRING)
+    }
+    if (type == AGTV_OBJECT)
+    {
         return 3;
-    if (type == AGTV_BOOL)
+    }
+    if (type == AGTV_ARRAY)
+    {
         return 4;
-    if (type == AGTV_NUMERIC || type == AGTV_INTEGER || type == AGTV_FLOAT)
+    }
+    if (type == AGTV_STRING)
+    {
         return 5;
-    if (type == AGTV_NULL)
+    }
+    if (type == AGTV_BOOL)
+    {
         return 6;
+    }
+    if (type == AGTV_NUMERIC || type == AGTV_INTEGER || type == AGTV_FLOAT)
+    {
+        return 7;
+    }
+    if (type == AGTV_NULL)
+    {
+        return 8;
+    }
     return -1;
 }
 
@@ -356,6 +378,18 @@ int compare_agtype_containers_orderability(agtype_container *a,
                 break;
             }
 
+            /* Correction step because AGTV_ARRAY might be there just because of the container type */
+            /* Case 1: left side is assigned to an array, right is an object */
+            if(va.type == AGTV_ARRAY && vb.type == AGTV_OBJECT)
+            {
+                ra = agtype_iterator_next(&ita, &va, false);
+            }
+            /* Case 2: left side is an object, right side is assigned to an array */
+            else if(va.type == AGTV_OBJECT && vb.type == AGTV_ARRAY)
+            {
+                rb = agtype_iterator_next(&itb, &vb, false);
+            }
+            
             Assert(va.type != vb.type);
             Assert(va.type != AGTV_BINARY);
             Assert(vb.type != AGTV_BINARY);