Add validation for NAS job in Katib controller

pkg/controller/studyjob/const.go

@@ -17,6 +17,9 @@ const (
 	PyTorchJobWorker = "PyTorchJob"
 	jobTypeNAS       = "NAS"
 	jobTypeHP        = "HP"
+	suggestionNASRL  = "nasrl"
 )
 
 var ValidWorkerKindList = [...]string{DefaultJobWorker, TFJobWorker, PyTorchJobWorker}
+
+var suggestionNASList = []string{suggestionNASRL}

pkg/controller/studyjob/katib_api_util.go

@@ -25,6 +25,7 @@ import (
 )
 
 func initializeStudy(instance *katibv1alpha1.StudyJob) error {
+
 	if instance.Spec.SuggestionSpec.SuggestionAlgorithm == "" {
 		instance.Spec.SuggestionSpec.SuggestionAlgorithm = "random"
 	}
@@ -73,15 +74,15 @@ func initializeStudy(instance *katibv1alpha1.StudyJob) error {
 }
 
 func getStudyConf(instance *katibv1alpha1.StudyJob) (*katibapi.StudyConfig, error) {
-	jobType := getJobType(instance)
+	jobType := getJobType(instance.Spec.SuggestionSpec.SuggestionAlgorithm)
 	if jobType == jobTypeNAS {
 		return populateConfigForNAS(instance)
 	}
 	return populateConfigForHP(instance)
 }
 
-func getJobType(instance *katibv1alpha1.StudyJob) string {
-	if instance.Spec.NasConfig != nil {
+func getJobType(suggestionAlgorithm string) string {
+	if contains(suggestionNASList, suggestionAlgorithm) {
 		return jobTypeNAS
 	}
 	return jobTypeHP

pkg/controller/studyjob/utils.go

@@ -15,6 +15,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"reflect"
 	"strings"
 
 	katibapi "github.com/kubeflow/katib/pkg/api"
@@ -101,6 +102,21 @@ func getWorkerKind(workerSpec *katibv1alpha1.WorkerSpec) (*schema.GroupVersionKi
 }
 
 func validateStudy(instance *katibv1alpha1.StudyJob) error {
+
+	if getJobType(instance.Spec.SuggestionSpec.SuggestionAlgorithm) != jobTypeNAS {
+		//Validate HP job
+		if validJobErr := validateHPJob(instance); validJobErr != nil {
+			instance.Status.Condition = katibv1alpha1.ConditionFailed
+			return validJobErr
+		}
+	} else {
+		//Validate NAS job
+		if validJobErr := validateNASJob(instance); validJobErr != nil {
+			instance.Status.Condition = katibv1alpha1.ConditionFailed
+			return validJobErr
+		}
+	}
+
 	if instance.Spec.SuggestionSpec == nil {
 		return fmt.Errorf("No Spec.SuggestionSpec specified.")
 	}
@@ -155,6 +171,7 @@ func validateStudy(instance *katibv1alpha1.StudyJob) error {
 	if mcjob.GetNamespace() != namespace || mcjob.GetName() != workerID {
 		return fmt.Errorf("Invalid metricsCollector template.")
 	}
+
 	return nil
 }
 
@@ -206,6 +223,86 @@ func contains(l []string, s string) bool {
 	return false
 }
 
+func validateHPJob(instance *katibv1alpha1.StudyJob) error {
+	log.Printf("Validation for the HP job")
+	if instance.Spec.ParameterConfigs == nil {
+		return fmt.Errorf("No Spec.ParameterConfigs specified")
+	}
+	return validateParameterConfigs(instance.Spec.ParameterConfigs, jobTypeHP, instance.Spec.SuggestionSpec.SuggestionAlgorithm)
+
+}
+
+func validateNASJob(instance *katibv1alpha1.StudyJob) error {
+
+	log.Printf("Validation for the NAS job")
+	if instance.Spec.NasConfig == nil {
+		return fmt.Errorf("No Spec.NasConfig specified")
+	}
+
+	if reflect.DeepEqual(instance.Spec.NasConfig.GraphConfig, katibv1alpha1.GraphConfig{}) {
+		return fmt.Errorf("Missing GraphConfig in NasConfig: %v", instance.Spec.NasConfig)
+	}
+
+	if instance.Spec.NasConfig.GraphConfig.InputSize == nil {
+		return fmt.Errorf("Missing InputSize in NasConfig.GraphConfig: %v", instance.Spec.NasConfig.GraphConfig)
+	}
+
+	if instance.Spec.NasConfig.GraphConfig.OutputSize == nil {
+		return fmt.Errorf("Missing OutputSize in NasConfig.GraphConfig: %v", instance.Spec.NasConfig.GraphConfig)
+	}
+
+	if instance.Spec.NasConfig.GraphConfig.NumLayers == 0 && contains(suggestionNASList, instance.Spec.SuggestionSpec.SuggestionAlgorithm) {
+		return fmt.Errorf("Missing NumLayers in NasConfig.GraphConfig: %v", instance.Spec.NasConfig.GraphConfig)
+	}
+
+	if instance.Spec.NasConfig.Operations == nil {
+		return fmt.Errorf("Missing Operations in NasConfig: %v", instance.Spec.NasConfig)
+	}
+
+	for _, op := range instance.Spec.NasConfig.Operations {
+		if op.OperationType == "" {
+			return fmt.Errorf("Missing OperationType in Operation: %v", op)
+		}
+		if op.ParameterConfigs == nil {
+			return fmt.Errorf("Missing ParameterConfig in Operation: %v", op)
+		}
+		return validateParameterConfigs(op.ParameterConfigs, jobTypeNAS, instance.Spec.SuggestionSpec.SuggestionAlgorithm)
+	}
+
+	return nil
+}
+
+func validateParameterConfigs(parameterConfigs []katibv1alpha1.ParameterConfig, jobType string, suggestionAlgorithm string) error {
+	for _, pc := range parameterConfigs {
+		if pc.Name == "" {
+			return fmt.Errorf("Missing Name in ParameterConfig: %v", pc)
+		}
+		if pc.ParameterType == "" {
+			return fmt.Errorf("Missing ParameterType in ParameterConfig: %v", pc)
+		}
+		if reflect.DeepEqual(pc.Feasible, katibv1alpha1.FeasibleSpace{}) {
+			return fmt.Errorf("Missing Feasible in ParameterConfig: %v", pc)
+		}
+		if pc.ParameterType == katibv1alpha1.ParameterTypeCategorical || pc.ParameterType == katibv1alpha1.ParameterTypeDiscrete {
+			if pc.Feasible.List == nil {
+				return fmt.Errorf("Missing List in ParameterConfig.Feasible: %v", pc.Feasible)
+			}
+		}
+		if pc.ParameterType == katibv1alpha1.ParameterTypeDouble || pc.ParameterType == katibv1alpha1.ParameterTypeInt {
+			if pc.Feasible.Min == "" {
+				return fmt.Errorf("Missing Min in ParameterConfig.Feasible: %v", pc.Feasible)
+			}
+			if pc.Feasible.Max == "" {
+				return fmt.Errorf("Missing Max in ParameterConfig.Feasible: %v", pc.Feasible)
+			}
+			if jobType == jobTypeNAS && pc.Feasible.Step == "" && pc.ParameterType == katibv1alpha1.ParameterTypeDouble && contains(suggestionNASList, suggestionAlgorithm) {
+				return fmt.Errorf("Missing Step in ParameterConfig.Feasible for NAS job: %v", pc.Feasible)
+			}
+		}
+	}
+	return nil
+}
+
 func getMyNamespace() string {
 	data, _ := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
 	return strings.TrimSpace(string(data))