feat: Ability to explain an executed request (#1188)

- Resolves #326 

- Description: Adds ability to return datapoints / information gathered at every planner step. The information is stored during execution, and gathered post execution.

- Usage: Add `@explain(type: execute) ` after the `query` or `mutation` operation.

- Execute explain request for `query` operation - example:
```
query @Explain(type: execute) {
	Address(groupBy: [country]) {
		country
		_group {
			city
		}
	}
}
```

- Execute explain request for `mutation` operation - example:
```
mutation @Explain(type: execute) {
	update_address(
		ids: ["bae-c8448e47-6cd1-571f-90bd-364acb80da7b"],
		data: "{\"country\": \"USA\"}"
	) {
		country
		city
	}
}
```

client/request/explain.go

@@ -15,5 +15,6 @@ type ExplainType string
 
 // Types of explain requests.
 const (
-	SimpleExplain ExplainType = "simple"
+	SimpleExplain  ExplainType = "simple"
+	ExecuteExplain ExplainType = "execute"
 )

planner/average.go

@@ -26,6 +26,13 @@ type averageNode struct {
 	sumFieldIndex     int
 	countFieldIndex   int
 	virtualFieldIndex int
+
+	execInfo averageExecInfo
+}
+
+type averageExecInfo struct {
+	// Total number of times averageNode was executed.
+	iterations uint64
 }
 
 func (p *Planner) Average(
@@ -64,6 +71,8 @@ func (n *averageNode) Close() error           { return n.plan.Close() }
 func (n *averageNode) Source() planNode       { return n.plan }
 
 func (n *averageNode) Next() (bool, error) {
+	n.execInfo.iterations++
+
 	hasNext, err := n.plan.Next()
 	if err != nil || !hasNext {
 		return hasNext, err
@@ -100,6 +109,17 @@ func (n *averageNode) SetPlan(p planNode) { n.plan = p }
 
 // Explain method returns a map containing all attributes of this node that
 // are to be explained, subscribes / opts-in this node to be an explainablePlanNode.
-func (n *averageNode) Explain() (map[string]any, error) {
-	return map[string]any{}, nil
+func (n *averageNode) Explain(explainType request.ExplainType) (map[string]any, error) {
+	switch explainType {
+	case request.SimpleExplain:
+		return map[string]any{}, nil
+
+	case request.ExecuteExplain:
+		return map[string]any{
+			"iterations": n.execInfo.iterations,
+		}, nil
+
+	default:
+		return nil, ErrUnknownExplainRequestType
+	}
 }

planner/commit.go

@@ -37,6 +37,13 @@ type dagScanNode struct {
 	fetcher      fetcher.HeadFetcher
 	spans        core.Spans
 	commitSelect *mapper.CommitSelect
+
+	execInfo dagScanExecInfo
+}
+
+type dagScanExecInfo struct {
+	// Total number of times dag scan was issued.
+	iterations uint64
 }
 
 func (p *Planner) DAGScan(commitSelect *mapper.CommitSelect) *dagScanNode {
@@ -118,23 +125,21 @@ func (n *dagScanNode) Close() error {
 
 func (n *dagScanNode) Source() planNode { return nil }
 
-// Explain method returns a map containing all attributes of this node that
-// are to be explained, subscribes / opts-in this node to be an explainablePlanNode.
-func (n *dagScanNode) Explain() (map[string]any, error) {
-	explainerMap := map[string]any{}
+func (n *dagScanNode) simpleExplain() (map[string]any, error) {
+	simpleExplainMap := map[string]any{}
 
 	// Add the field attribute to the explanation if it exists.
 	if n.commitSelect.FieldName.HasValue() {
-		explainerMap["field"] = n.commitSelect.FieldName.Value()
+		simpleExplainMap["field"] = n.commitSelect.FieldName.Value()
 	} else {
-		explainerMap["field"] = nil
+		simpleExplainMap["field"] = nil
 	}
 
 	// Add the cid attribute to the explanation if it exists.
 	if n.commitSelect.Cid.HasValue() {
-		explainerMap["cid"] = n.commitSelect.Cid.Value()
+		simpleExplainMap["cid"] = n.commitSelect.Cid.Value()
 	} else {
-		explainerMap["cid"] = nil
+		simpleExplainMap["cid"] = nil
 	}
 
 	// Build the explanation of the spans attribute.
@@ -152,12 +157,31 @@ func (n *dagScanNode) Explain() (map[string]any, error) {
 		}
 	}
 	// Add the built spans attribute, if it was valid.
-	explainerMap[spansLabel] = spansExplainer
+	simpleExplainMap[spansLabel] = spansExplainer
 
-	return explainerMap, nil
+	return simpleExplainMap, nil
+}
+
+// Explain method returns a map containing all attributes of this node that
+// are to be explained, subscribes / opts-in this node to be an explainablePlanNode.
+func (n *dagScanNode) Explain(explainType request.ExplainType) (map[string]any, error) {
+	switch explainType {
+	case request.SimpleExplain:
+		return n.simpleExplain()
+
+	case request.ExecuteExplain:
+		return map[string]any{
+			"iterations": n.execInfo.iterations,
+		}, nil
+
+	default:
+		return nil, ErrUnknownExplainRequestType
+	}
 }
 
 func (n *dagScanNode) Next() (bool, error) {
+	n.execInfo.iterations++
+
 	var currentCid *cid.Cid
 	store := n.planner.txn.DAGstore()
 

planner/count.go

@@ -20,6 +20,7 @@ import (
 	"github.com/sourcenetwork/immutable"
 	"github.com/sourcenetwork/immutable/enumerable"
 
+	"github.com/sourcenetwork/defradb/client/request"
 	"github.com/sourcenetwork/defradb/core"
 	"github.com/sourcenetwork/defradb/planner/mapper"
 )
@@ -33,6 +34,13 @@ type countNode struct {
 
 	virtualFieldIndex int
 	aggregateMapping  []mapper.AggregateTarget
+
+	execInfo countExecInfo
+}
+
+type countExecInfo struct {
+	// Total number of times countNode was executed.
+	iterations uint64
 }
 
 func (p *Planner) Count(field *mapper.Aggregate, host *mapper.Select) (*countNode, error) {
@@ -60,33 +68,50 @@ func (n *countNode) Close() error { return n.plan.Close() }
 
 func (n *countNode) Source() planNode { return n.plan }
 
-// Explain method returns a map containing all attributes of this node that
-// are to be explained, subscribes / opts-in this node to be an explainablePlanNode.
-func (n *countNode) Explain() (map[string]any, error) {
+func (n *countNode) simpleExplain() (map[string]any, error) {
 	sourceExplanations := make([]map[string]any, len(n.aggregateMapping))
 
 	for i, source := range n.aggregateMapping {
-		explainerMap := map[string]any{}
+		simpleExplainMap := map[string]any{}
 
 		// Add the filter attribute if it exists.
 		if source.Filter == nil || source.Filter.ExternalConditions == nil {
-			explainerMap[filterLabel] = nil
+			simpleExplainMap[filterLabel] = nil
 		} else {
-			explainerMap[filterLabel] = source.Filter.ExternalConditions
+			simpleExplainMap[filterLabel] = source.Filter.ExternalConditions
 		}
 
 		// Add the main field name.
-		explainerMap[fieldNameLabel] = source.Field.Name
+		simpleExplainMap[fieldNameLabel] = source.Field.Name
 
-		sourceExplanations[i] = explainerMap
+		sourceExplanations[i] = simpleExplainMap
 	}
 
 	return map[string]any{
 		sourcesLabel: sourceExplanations,
 	}, nil
 }
 
+// Explain method returns a map containing all attributes of this node that
+// are to be explained, subscribes / opts-in this node to be an explainablePlanNode.
+func (n *countNode) Explain(explainType request.ExplainType) (map[string]any, error) {
+	switch explainType {
+	case request.SimpleExplain:
+		return n.simpleExplain()
+
+	case request.ExecuteExplain:
+		return map[string]any{
+			"iterations": n.execInfo.iterations,
+		}, nil
+
+	default:
+		return nil, ErrUnknownExplainRequestType
+	}
+}
+
 func (n *countNode) Next() (bool, error) {
+	n.execInfo.iterations++
+
 	hasValue, err := n.plan.Next()
 	if err != nil || !hasValue {
 		return hasValue, err

planner/create.go

@@ -14,6 +14,7 @@ import (
 	"encoding/json"
 
 	"github.com/sourcenetwork/defradb/client"
+	"github.com/sourcenetwork/defradb/client/request"
 	"github.com/sourcenetwork/defradb/core"
 	"github.com/sourcenetwork/defradb/db/base"
 	"github.com/sourcenetwork/defradb/planner/mapper"
@@ -44,6 +45,13 @@ type createNode struct {
 
 	returned bool
 	results  planNode
+
+	execInfo createExecInfo
+}
+
+type createExecInfo struct {
+	// Total number of times createNode was executed.
+	iterations uint64
 }
 
 func (n *createNode) Kind() string { return "createNode" }
@@ -62,6 +70,8 @@ func (n *createNode) Start() error {
 
 // Next only returns once.
 func (n *createNode) Next() (bool, error) {
+	n.execInfo.iterations++
+
 	if n.err != nil {
 		return false, n.err
 	}
@@ -121,9 +131,7 @@ func (n *createNode) Close() error {
 
 func (n *createNode) Source() planNode { return n.results }
 
-// Explain method returns a map containing all attributes of this node that
-// are to be explained, subscribes / opts-in this node to be an explainablePlanNode.
-func (n *createNode) Explain() (map[string]any, error) {
+func (n *createNode) simpleExplain() (map[string]any, error) {
 	data := map[string]any{}
 	err := json.Unmarshal([]byte(n.newDocStr), &data)
 	if err != nil {
@@ -135,6 +143,23 @@ func (n *createNode) Explain() (map[string]any, error) {
 	}, nil
 }
 
+// Explain method returns a map containing all attributes of this node that
+// are to be explained, subscribes / opts-in this node to be an explainablePlanNode.
+func (n *createNode) Explain(explainType request.ExplainType) (map[string]any, error) {
+	switch explainType {
+	case request.SimpleExplain:
+		return n.simpleExplain()
+
+	case request.ExecuteExplain:
+		return map[string]any{
+			"iterations": n.execInfo.iterations,
+		}, nil
+
+	default:
+		return nil, ErrUnknownExplainRequestType
+	}
+}
+
 func (p *Planner) CreateDoc(parsed *mapper.Mutation) (planNode, error) {
 	results, err := p.Select(&parsed.Select)
 	if err != nil {

planner/delete.go

@@ -12,6 +12,7 @@ package planner
 
 import (
 	"github.com/sourcenetwork/defradb/client"
+	"github.com/sourcenetwork/defradb/client/request"
 	"github.com/sourcenetwork/defradb/core"
 	"github.com/sourcenetwork/defradb/planner/mapper"
 )
@@ -27,9 +28,18 @@ type deleteNode struct {
 
 	filter *mapper.Filter
 	ids    []string
+
+	execInfo deleteExecInfo
+}
+
+type deleteExecInfo struct {
+	// Total number of times deleteNode was executed.
+	iterations uint64
 }
 
 func (n *deleteNode) Next() (bool, error) {
+	n.execInfo.iterations++
+
 	next, err := n.source.Next()
 	if err != nil {
 		return false, err
@@ -74,22 +84,37 @@ func (n *deleteNode) Source() planNode {
 	return n.source
 }
 
-// Explain method returns a map containing all attributes of this node that
-// are to be explained, subscribes / opts-in this node to be an explainablePlanNode.
-func (n *deleteNode) Explain() (map[string]any, error) {
-	explainerMap := map[string]any{}
+func (n *deleteNode) simpleExplain() (map[string]any, error) {
+	simpleExplainMap := map[string]any{}
 
 	// Add the document id(s) that request wants to delete.
-	explainerMap[idsLabel] = n.ids
+	simpleExplainMap[idsLabel] = n.ids
 
 	// Add the filter attribute if it exists, otherwise have it nil.
 	if n.filter == nil || n.filter.ExternalConditions == nil {
-		explainerMap[filterLabel] = nil
+		simpleExplainMap[filterLabel] = nil
 	} else {
-		explainerMap[filterLabel] = n.filter.ExternalConditions
+		simpleExplainMap[filterLabel] = n.filter.ExternalConditions
 	}
 
-	return explainerMap, nil
+	return simpleExplainMap, nil
+}
+
+// Explain method returns a map containing all attributes of this node that
+// are to be explained, subscribes / opts-in this node to be an explainablePlanNode.
+func (n *deleteNode) Explain(explainType request.ExplainType) (map[string]any, error) {
+	switch explainType {
+	case request.SimpleExplain:
+		return n.simpleExplain()
+
+	case request.ExecuteExplain:
+		return map[string]any{
+			"iterations": n.execInfo.iterations,
+		}, nil
+
+	default:
+		return nil, ErrUnknownExplainRequestType
+	}
 }
 
 func (p *Planner) DeleteDocs(parsed *mapper.Mutation) (planNode, error) {

planner/errors.go

@@ -13,8 +13,9 @@ package planner
 import "github.com/sourcenetwork/defradb/errors"
 
 const (
-	errUnknownDependency string = "given field does not exist"
-	errFailedToClosePlan string = "failed to close the plan"
+	errUnknownDependency              string = "given field does not exist"
+	errFailedToClosePlan              string = "failed to close the plan"
+	errFailedToCollectExecExplainInfo string = "failed to collect execution explain information"
 )
 
 var (
@@ -30,6 +31,7 @@ var (
 	ErrMissingChildValue                   = errors.New("expected child value, however none was yielded")
 	ErrUnknownRelationType                 = errors.New("failed sub selection, unknown relation type")
 	ErrUnknownExplainRequestType           = errors.New("can not explain request of unknown type")
+	ErrFailedToCollectExecExplainInfo      = errors.New(errFailedToCollectExecExplainInfo)
 	ErrUnknownDependency                   = errors.New(errUnknownDependency)
 )
 
@@ -40,3 +42,7 @@ func NewErrUnknownDependency(name string) error {
 func NewErrFailedToClosePlan(inner error, location string) error {
 	return errors.Wrap(errFailedToClosePlan, inner, errors.NewKV("Location", location))
 }
+
+func NewErrFailedToCollectExecExplainInfo(inner error) error {
+	return errors.Wrap(errFailedToCollectExecExplainInfo, inner)
+}