Merge pull request #22 from btnguyen2k/feature_subpartitions

Prepare to release v0.3.0

.github/workflows/gocosmos.yaml

@@ -195,7 +195,7 @@ jobs:
       - name: Test
         run: |
           export COSMOSDB_URL="AccountEndpoint=https://127.0.0.1:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="
-          go test -cover -coverprofile="coverage_other.txt" -v -count 1 -p 1 -run "TestNew|TestStmt_NumInput|TestDriver_" .
+          go test -cover -coverprofile="coverage_other.txt" -v -count 1 -p 1 -run "TestNew|TestDriver_" .
       - name: Codecov
         uses: codecov/codecov-action@v3
         with:

README.md

@@ -20,7 +20,7 @@ Summary of supported SQL statements:
 |Delete an existing database                |`DROP DATABASE [IF EXISTS] <db-name>`|
 |List all existing databases                |`LIST DATABASES`|
 |Create a new collection                    |`CREATE COLLECTION [IF NOT EXISTS] [<db-name>.]<collection-name> <WITH [LARGE]PK=partitionKey>`|
-|Change collection's throughput             |`ALTER COLLECTION [<db-name>.]<collection-name> WITH  RU/MAXRU=<ru>`|
+|Change collection's throughput             |`ALTER COLLECTION [<db-name>.]<collection-name> WITH RU|MAXRU=<ru>`|
 |Delete an existing collection              |`DROP COLLECTION [IF EXISTS] [<db-name>.]<collection-name>`|
 |List all existing collections in a database|`LIST COLLECTIONS [FROM <db-name>]`|
 |Insert a new document into collection      |`INSERT INTO [<db-name>.]<collection-name> ...`|
@@ -77,7 +77,7 @@ AccountEndpoint=<cosmosdb-endpoint>
 - `AccountEndpoint`: (required) endpoint to access Cosmos DB. For example, the endpoint for Azure Cosmos DB Emulator running on local is `https://localhost:8081/`.
 - `AccountKey`: (required) account key to authenticate.
 - `TimeoutMs`: (optional) operation timeout in milliseconds. Default value is `10 seconds` if not specified.
-- `Version`: (optional) version of Cosmos DB to use. Default value is `2018-12-31` if not specified. See: https://learn.microsoft.com/rest/api/cosmos-db/#supported-rest-api-versions.
+- `Version`: (optional) version of Cosmos DB to use. Default value is `2020-07-15` if not specified. See: https://learn.microsoft.com/rest/api/cosmos-db/#supported-rest-api-versions.
 - `DefaultDb`: (optional, available since [v0.1.1](RELEASE-NOTES.md)) specify the default database used in Cosmos DB operations. Alias `Db` can also be used instead of `DefaultDb`.
 - `AutoId`: (optional, available since [v0.1.2](RELEASE-NOTES.md)) see [auto id](#auto-id) session.
 - `InsecureSkipVerify`: (optional, available since [v0.1.4](RELEASE-NOTES.md)) if `true`, disable CA verification for https endpoint (useful to run against test/dev env with local/docker Cosmos DB emulator).
@@ -156,7 +156,7 @@ AccountEndpoint=<cosmosdb-endpoint>
 - `AccountEndpoint`: (required) endpoint to access Cosmos DB. For example, the endpoint for Azure Cosmos DB Emulator running on local is `https://localhost:8081/`.
 - `AccountKey`: (required) account key to authenticate.
 - `TimeoutMs`: (optional) operation timeout in milliseconds. Default value is `10 seconds` if not specified.
-- `Version`: (optional) version of Cosmos DB to use. Default value is `2018-12-31` if not specified. See: https://learn.microsoft.com/rest/api/cosmos-db/#supported-rest-api-versions.
+- `Version`: (optional) version of Cosmos DB to use. Default value is `2020-07-15` if not specified. See: https://learn.microsoft.com/rest/api/cosmos-db/#supported-rest-api-versions.
 - `AutoId`: (optional, available since [v0.1.2](RELEASE-NOTES.md)) see [auto id](#auto-id) session.
 - `InsecureSkipVerify`: (optional, available since [v0.1.4](RELEASE-NOTES.md)) if `true`, disable CA verification for https endpoint (useful to run against test/dev env with local/docker Cosmos DB emulator).
 

RELEASE-NOTES.md

@@ -1,5 +1,11 @@
 # gocosmos - Release notes
 
+## 2023-06-16 - v0.3.0
+
+- Change default API version to `2020-07-15`.
+- Add [Hierarchical Partition Keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) (sub-partitions) support.
+- Use PartitionKey version 2 (replacing version 1), hence large PK is always enabled.
+
 ## 2023-06-09 - v0.2.1
 
 - Bug fixes, Refactoring & Enhancements.

SQL.md

@@ -168,7 +168,9 @@ fmt.Println(dbresult.RowsAffected())
 
 - Upon successful execution, `RowsAffected()` returns `(1, nil)`.
 - This statement returns error `ErrConflict` if the specified collection already existed. If `IF NOT EXISTS` is specified, `RowsAffected()` returns `(0, nil)`.
-- Partition key must be specified using `WITH pk=<partition-key>`. If partition key is larger than 100 bytes, use `WITH pk=<partition-key>` instead.
+- Partition key must be specified using `WITH pk=<partition-key>`.
+  - Since [v0.3.0](RELEASE-NOTES.md), large pk is always enabled, `WITH largepk` is for backward compatibility only.
+  - Since [v0.3.0](RELEASE-NOTES.md), Hierarchical Partition Key is supported, using `WITH pk=/path1,/path2...` (up to 3 path levels).
 - Provisioned capacity can be optionally specified via `WITH RU=<ru>` or `WITH MAXRU=<ru>`.
   - Only one of `RU` and `MAXRU` options should be specified, _not both_; error is returned if both optiosn are specified.
 - Unique keys are optionally specified via `WITH uk=/uk1_path:/uk2_path1,/uk2_path2:/uk3_path`. Each unique key is a comma-separated list of paths (e.g. `/uk_path1,/uk_path2`); unique keys are separated by colons (e.g. `/uk1:/uk2:/uk3`).
@@ -295,11 +297,16 @@ Description: insert a new document into an existing collection.
 Syntax:
 
 ```sql
-INSERT INTO [<db-name>.]<collection-name> (<field1>, <field2>,...<fieldN>) VALUES (<value1>, <value2>,...<valueN>)
+INSERT INTO [<db-name>.]<collection-name>
+(<field1>, <field2>,...<fieldN>)
+VALUES (<value1>, <value2>,...<valueN>)
+[WITH singlePK|SINGLE_PK[=true]]
 ```
 
 > `<db-name>` can be ommitted if `DefaultDb` is supplied in the Data Source Name (DSN).
 
+Since [v0.3.0](RELEASE-NOTES.md), `gocosmos` supports [Hierarchical Partition Keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) (or sub-partitions). If the collection is known not to have sub-partitions, supplying `WITH singlePK` (or `WITH SINGLE_PK`) can save one roundtrip to Cosmos DB server.
+
 Example:
 ```go
 sql := `INSERT INTO mydb.mytable (a, b, c, d, e) VALUES (1, "\"a string\"", :1, @2, $3)`
@@ -312,8 +319,7 @@ fmt.Println(dbresult.RowsAffected())
 
 > Use `sql.DB.Exec` to execute the statement, `Query` will return error.
 
-> Value of partition key _must_ be supplied at the last argument of `db.Exec()` call.
-
+> Values of partition keys _must_ be supplied at the end of the argument list when invoking `db.Exec()`.
 
 <a id="value"></a>A value is either:
 - a placeholder - which is a number prefixed by `$` or `@` or `:`, for example `$1`, `@2` or `:3`. Placeholders are 1-based index, that means starting from 1.
@@ -337,7 +343,10 @@ Description: insert a new document or replace an existing one.
 Syntax & Usage: similar to [INSERT](#insert).
 
 ```sql
-UPSERT INTO [<db-name>.]<collection-name> (<field1>, <field2>,...<fieldN>) VALUES (<value1>, <value2>,...<valueN>)
+UPSERT INTO [<db-name>.]<collection-name>
+(<field1>, <field2>,...<fieldN>)
+VALUES (<value1>, <value2>,...<valueN>)
+[WITH singlePK|SINGLE_PK[=true]]
 ```
 
 [Back to top](#top)
@@ -349,11 +358,15 @@ Description: delete an existing document.
 Syntax:
 
 ```sql
-DELETE FROM [<db-name>.]<collection-name> WHERE id=<id-value>
+DELETE FROM [<db-name>.]<collection-name>
+WHERE id=<id-value>
+[WITH singlePK|SINGLE_PK[=true]]
 ```
 
 > `<db-name>` can be ommitted if `DefaultDb` is supplied in the Data Source Name (DSN).
 
+Since [v0.3.0](RELEASE-NOTES.md), `gocosmos` supports [Hierarchical Partition Keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) (or sub-partitions). If the collection is known not to have sub-partitions, supplying `WITH singlePK` (or `WITH SINGLE_PK`) can save one roundtrip to Cosmos DB server.
+
 Example:
 ```go
 sql := `DELETE FROM mydb.mytable WHERE id=@1`
@@ -366,7 +379,7 @@ fmt.Println(dbresult.RowsAffected())
 
 > Use `sql.DB.Exec` to execute the statement, `Query` will return error.
 
-> Value of partition key _must_ be supplied at the last argument of `db.Exec()` call.
+> Values of partition keys _must_ be supplied at the end of the argument list when invoking `db.Exec()`.
 
 - `DELETE` removes only one document specified by id.
 - Upon successful execution, `RowsAffected()` returns `(1, nil)`. If no document matched, `RowsAffected()` returns `(0, nil)`.
@@ -381,11 +394,16 @@ Description: update an existing document.
 Syntax:
 
 ```sql
-UPDATE [<db-name>.]<collection-name> SET <fiel1>=<value1>[,<field2>=<value2>,...<fieldN>=<valueN>] WHERE id=<id-value>
+UPDATE [<db-name>.]<collection-name>
+SET <fiel1>=<value1>[,<field2>=<value2>,...<fieldN>=<valueN>]
+WHERE id=<id-value>
+[WITH singlePK|SINGLE_PK[=true]]
 ```
 
 > `<db-name>` can be ommitted if `DefaultDb` is supplied in the Data Source Name (DSN).
 
+Since [v0.3.0](RELEASE-NOTES.md), `gocosmos` supports [Hierarchical Partition Keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) (or sub-partitions). If the collection is known not to have sub-partitions, supplying `WITH singlePK` (or `WITH SINGLE_PK`) can save one roundtrip to Cosmos DB server.
+
 Example:
 ```go
 sql := `UPDATE mydb.mytable SET a=1, b="\"a string\"", c=true, d="[1,true,null,\"string\"]", e=:2 WHERE id=@1`
@@ -398,7 +416,7 @@ fmt.Println(dbresult.RowsAffected())
 
 > Use `sql.DB.Exec` to execute the statement, `Query` will return error.
 
-> Value of partition key _must_ be supplied at the last argument of `db.Exec()` call.
+> Values of partition keys _must_ be supplied at the end of the argument list when invoking `db.Exec()`.
 
 - `UPDATE` modifies only one document specified by id.
 - Upon successful execution, `RowsAffected()` returns `(1, nil)`. If no document matched, `RowsAffected()` returns `(0, nil)`.
@@ -417,7 +435,7 @@ Syntax:
 SELECT [CROSS PARTITION] ... FROM <collection-name> ...
 [WITH database=<db-name>]
 [[,] WITH collection=<collection-name>]
-[[,] WITH cross_partition=true]
+[[,] WITH cross_partition|CrossPartition[=true]]
 ```
 
 > `<db-name>` can be ommitted if `DefaultDb` is supplied in the Data Source Name (DSN).

data_test.go

@@ -14,11 +14,72 @@ import (
 
 /*======================================================================*/
 
+const numApps = 4
 const numLogicalPartitions = 16
 const numCategories = 19
 
 var dataList []DocInfo
 
+func _initDataSubPartitions(t *testing.T, testName string, client *RestClient, db, container string, numItem int) {
+	totalRu := 0.0
+	randList := make([]int, numItem)
+	for i := 0; i < numItem; i++ {
+		randList[i] = i*2 + 1
+	}
+	rand.Shuffle(numItem, func(i, j int) {
+		randList[i], randList[j] = randList[j], randList[i]
+	})
+	dataList = make([]DocInfo, numItem)
+	for i := 0; i < numItem; i++ {
+		category := randList[i] % numCategories
+		app := "app" + strconv.Itoa(i%numApps)
+		username := "user" + strconv.Itoa(i%numLogicalPartitions)
+		docInfo := DocInfo{
+			"id":       fmt.Sprintf("%05d", i),
+			"app":      app,
+			"username": username,
+			"email":    "user" + strconv.Itoa(i) + "@domain.com",
+			"grade":    float64(randList[i]),
+			"category": float64(category),
+			"active":   i%10 == 0,
+			"big":      fmt.Sprintf("%05d", i) + "/" + strings.Repeat("this is a very long string/", 256),
+		}
+		dataList[i] = docInfo
+		if result := client.CreateDocument(DocumentSpec{DbName: db, CollName: container, PartitionKeyValues: []interface{}{app, username}, DocumentData: docInfo}); result.Error() != nil {
+			t.Fatalf("%s failed: %s", testName, result.Error())
+		} else {
+			totalRu += result.RequestCharge
+		}
+	}
+	// fmt.Printf("\t%s - total RU charged: %0.3f\n", testName+"/Insert", totalRu)
+}
+
+func _initDataSubPartitionsSmallRU(t *testing.T, testName string, client *RestClient, db, container string, numItem int) {
+	client.DeleteDatabase(db)
+	client.CreateDatabase(DatabaseSpec{Id: db, Ru: 400})
+	client.CreateCollection(CollectionSpec{
+		DbName:           db,
+		CollName:         container,
+		PartitionKeyInfo: map[string]interface{}{"paths": []string{"/app", "/username"}, "kind": "MultiHash", "version": 2},
+		UniqueKeyPolicy:  map[string]interface{}{"uniqueKeys": []map[string]interface{}{{"paths": []string{"/email"}}}},
+		Ru:               400,
+	})
+	_initDataSubPartitions(t, testName, client, db, container, numItem)
+}
+
+func _initDataSubPartitionsLargeRU(t *testing.T, testName string, client *RestClient, db, container string, numItem int) {
+	client.DeleteDatabase(db)
+	client.CreateDatabase(DatabaseSpec{Id: db, Ru: 20000})
+	client.CreateCollection(CollectionSpec{
+		DbName:           db,
+		CollName:         container,
+		PartitionKeyInfo: map[string]interface{}{"paths": []string{"/app", "/username"}, "kind": "MultiHash", "version": 2},
+		UniqueKeyPolicy:  map[string]interface{}{"uniqueKeys": []map[string]interface{}{{"paths": []string{"/email"}}}},
+		Ru:               20000,
+	})
+	_initDataSubPartitions(t, testName, client, db, container, numItem)
+}
+
 func _initData(t *testing.T, testName string, client *RestClient, db, container string, numItem int) {
 	totalRu := 0.0
 	randList := make([]int, numItem)

driver.go

@@ -98,7 +98,7 @@ type Driver struct {
 //
 //	AccountEndpoint=<cosmosdb-restapi-endpoint>;AccountKey=<account-key>[;TimeoutMs=<timeout-in-ms>][;Version=<cosmosdb-api-version>][;DefaultDb=<db-name>][;AutoId=<true/false>][;InsecureSkipVerify=<true/false>]
 //
-// If not supplied, default value for TimeoutMs is 10 seconds, Version is defaultApiVersion (which is "2018-12-31"), AutoId is true, and InsecureSkipVerify is false
+// If not supplied, default value for TimeoutMs is 10 seconds, Version is DefaultApiVersion (which is "2020-07-15"), AutoId is true, and InsecureSkipVerify is false
 //
 // - DefaultDb is added since v0.1.1
 // - AutoId is added since v0.1.2

gocosmos.go

@@ -7,7 +7,7 @@ import (
 
 const (
 	// Version of package gocosmos.
-	Version = "0.2.1"
+	Version = "0.3.0"
 )
 
 func goTypeToCosmosDbType(typ reflect.Type) string {

restclient.go

@@ -31,7 +31,13 @@ const (
 	settingVersion            = "VERSION"
 	settingAutoId             = "AUTOID"
 	settingInsecureSkipVerify = "INSECURESKIPVERIFY"
-	defaultApiVersion         = "2018-12-31"
+
+	// DefaultApiVersion holds the default REST API version if not specified in the connection string.
+	//
+	// See: https://learn.microsoft.com/en-us/rest/api/cosmos-db/#supported-rest-api-versions
+	//
+	// @Available since v0.3.0
+	DefaultApiVersion = "2020-07-15"
 )
 
 // NewRestClient constructs a new RestClient instance from the supplied connection string.
@@ -41,7 +47,7 @@ const (
 //
 //	AccountEndpoint=<cosmosdb-restapi-endpoint>;AccountKey=<account-key>[;TimeoutMs=<timeout-in-ms>][;Version=<cosmosdb-api-version>][;AutoId=<true/false>][;InsecureSkipVerify=<true/false>]
 //
-// If not supplied, default value for TimeoutMs is 10 seconds, Version is defaultApiVersion (which is "2018-12-31"), AutoId is true, and InsecureSkipVerify is false
+// If not supplied, default value for TimeoutMs is 10 seconds, Version is DefaultApiVersion (which is "2020-07-15"), AutoId is true, and InsecureSkipVerify is false
 //
 // - AutoId is added since v0.1.2
 // - InsecureSkipVerify is added since v0.1.4
@@ -75,7 +81,7 @@ func NewRestClient(httpClient *http.Client, connStr string) (*RestClient, error)
 	}
 	apiVersion := params[settingVersion]
 	if apiVersion == "" {
-		apiVersion = defaultApiVersion
+		apiVersion = DefaultApiVersion
 	}
 	autoId, err := strconv.ParseBool(params[settingAutoId])
 	if err != nil {
@@ -1194,6 +1200,35 @@ type RespListDb struct {
 	Databases   []DbInfo `json:"Databases"`
 }
 
+// PkInfo holds partitioning configuration settings for a collection.
+//
+// @Available since v0.3.0
+type PkInfo map[string]interface{}
+
+func (pk PkInfo) Kind() string {
+	kind, err := reddo.ToString(pk["kind"])
+	if err == nil {
+		return kind
+	}
+	return ""
+}
+
+func (pk PkInfo) Version() int {
+	version, err := reddo.ToInt(pk["version"])
+	if err == nil {
+		return int(version)
+	}
+	return 0
+}
+
+func (pk PkInfo) Paths() []string {
+	paths, err := reddo.ToSlice(pk["paths"], reddo.TypeString)
+	if err == nil {
+		return paths.([]string)
+	}
+	return nil
+}
+
 // CollInfo captures info of a Cosmos DB collection.
 type CollInfo struct {
 	Id                       string                 `json:"id"`                       // user-generated unique name for the collection
@@ -1207,7 +1242,7 @@ type CollInfo struct {
 	Udfs                     string                 `json:"_udfs"`                    // (system-generated property) _udfs attribute of the collection
 	Conflicts                string                 `json:"_conflicts"`               // (system-generated property) _conflicts attribute of the collection
 	IndexingPolicy           map[string]interface{} `json:"indexingPolicy"`           // indexing policy settings for collection
-	PartitionKey             map[string]interface{} `json:"partitionKey"`             // partitioning configuration settings for collection
+	PartitionKey             PkInfo                 `json:"partitionKey"`             // partitioning configuration settings for collection
 	ConflictResolutionPolicy map[string]interface{} `json:"conflictResolutionPolicy"` // conflict resolution policy settings for collection
 	GeospatialConfig         map[string]interface{} `json:"geospatialConfig"`         // Geo-spatial configuration settings for collection
 }

restclient_collection_test.go

@@ -65,6 +65,54 @@ func TestRestClient_CreateCollection(t *testing.T) {
 	}
 }
 
+func TestRestClient_CreateCollection_SubPartitions(t *testing.T) {
+	name := "TestRestClient_CreateCollection_SubPartitions"
+	client := _newRestClient(t, name)
+
+	dbname := testDb
+	collname := testTable
+	collspecList := []CollectionSpec{
+		// Hierarchical Partition Keys
+		{DbName: dbname, CollName: collname, PartitionKeyInfo: map[string]interface{}{"paths": []string{"/TenantId", "/UserId"}, "kind": "MultiHash", "version": 2}},
+		{DbName: dbname, CollName: collname, MaxRu: 4000, PartitionKeyInfo: map[string]interface{}{"paths": []string{"/TenantId", "/UserId", "/SessionId"}, "kind": "MultiHash", "version": 2}},
+	}
+	for _, collspec := range collspecList {
+		client.DeleteDatabase(dbname)
+		client.CreateDatabase(DatabaseSpec{Id: dbname})
+		var collInfo CollInfo
+		if result := client.CreateCollection(collspec); result.Error() != nil {
+			t.Fatalf("%s failed: %s", name, result.Error())
+		} else if result.Id != collname {
+			t.Fatalf("%s failed: <coll-id> expected %#v but received %#v", name+"/CreateDatabase", collname, result.Id)
+		} else if result.Rid == "" || result.Self == "" || result.Etag == "" || result.Docs == "" ||
+			result.Sprocs == "" || result.Triggers == "" || result.Udfs == "" || result.Conflicts == "" ||
+			result.Ts <= 0 || len(result.IndexingPolicy) == 0 || len(result.PartitionKey) == 0 {
+			t.Fatalf("%s failed: invalid collinfo returned %#v", name, result.CollInfo)
+		} else {
+			collInfo = result.CollInfo
+		}
+
+		if collspec.Ru > 0 || collspec.MaxRu > 0 {
+			if result := client.GetOfferForResource(collInfo.Rid); result.Error() != nil {
+				t.Fatalf("%s failed: %s", name, result.Error())
+			} else {
+				if ru, maxru := result.OfferThroughput(), result.MaxThroughputEverProvisioned(); collspec.Ru > 0 && (collspec.Ru != ru || collspec.Ru != maxru) {
+					t.Fatalf("%s failed: <offer-throughput> expected %#v but expected {ru:%#v, maxru:%#v}", name, collspec.Ru, ru, maxru)
+				}
+				if ru, maxru := result.OfferThroughput(), result.MaxThroughputEverProvisioned(); collspec.MaxRu > 0 && (collspec.MaxRu != ru*10 || collspec.MaxRu != maxru) {
+					t.Fatalf("%s failed: <max-throughput> expected %#v but expected {ru:%#v, maxru:%#v}", name, collspec.MaxRu, ru, maxru)
+				}
+			}
+		}
+
+		if result := client.CreateCollection(collspec); result.CallErr != nil {
+			t.Fatalf("%s failed: %s", name, result.CallErr)
+		} else if result.StatusCode != 409 {
+			t.Fatalf("%s failed: <status-code> expected %#v but received %#v", name, 409, result.StatusCode)
+		}
+	}
+}
+
 func TestRestClient_ChangeOfferCollection(t *testing.T) {
 	name := "TestRestClient_ChangeOfferCollection"
 	client := _newRestClient(t, name)