chore: code refactoring v2 (#4572)

* stage 1 * fix: Image-Scanner status for failed request (#4513) * retry envs added * json naming * migration for image scan plugin and retry env fix * fix * image scan migration script updated - added new input variables * migration updated * migration updated * name changed * refactor * added default value and change script no * chore: added sql-validator in git-hub action (#4255) * added sql-validator in git-hub action * removed exit commands * edited the grep with whole word * modified * modified comments --------- Co-authored-by: Badal Kumar Prusty <[email protected]> * feat: Adds Copacetic plugin to patch vulnerable images (#4566) * Adds copacetic plugin * copacetic plugin: adds plugin icon and updates url in migration file * Update 211_copacetic_plugin_v1_0_0.up.sql adds copa patch command exit condition * Update 211_copacetic_plugin_v1_0_0.up.sql adds exit 1 if patching multi architecture image * dummy commit 211_copacetic_plugin_v1_0_0.up.sql dummy commit * chore: added sql-validator in git-hub action (#4255) * added sql-validator in git-hub action * removed exit commands * edited the grep with whole word * modified * modified comments --------- Co-authored-by: Badal Kumar Prusty <[email protected]> * updates migration no. --------- Co-authored-by: Badal Kumar <[email protected]> Co-authored-by: Badal Kumar Prusty <[email protected]> * feat: Create Dockerslim plugin and Create EKS Cluster plugin (#4525) * Plugins icon * eks and dockerslim plugin sql up * eks and dockerslim plugin down script * dockerslim plugin after review PR * Changes after review dockerslim * image changes * new image dockerslim * eks creation plugin * Final PR commit, eks and dockerslim plugins * feat: making pre,post,deploy triggers flows idempotent (#4486) * update common-lib * added idempotency code and common-lib version bump * added the optionsal validations pubsub msg pre-processing-hooks logic and added duplicate trigger check as a validation for pre,post,deploy trigger flows * bump common-lib * added callback logger func * sql scripts added * query fix * query fix * fix * refactoring * remove dag exececutor dependency from ciEeventHalndler Sevice * bump common-lib * move context to trigger context * add logs * migration version fix * remove test data * update script number --------- Co-authored-by: Kripansh <[email protected]> * chore: panic middleware and proxy handler (#4546) * initial commit * removed from hyperion * changes * added trace * pr comments * pg timeouts * panic message constant * cleaning up * common lib version * argo assets revert * adding image scanner proxy route * common-lib-update * refactoring: App Store deployment services * fix: sql script added for putting Identifier type=0 for all null values present in generic_note (#4568) * script for initializing null identifier type col to 0(cluster type) * script fix * empty script for 216 * 216 up script revert * fix: 500 fixes second iteration (#4464) * FetchChartNotes return 4xx when installed app or installed app versions not found * GetDeploymentHistory and GetInstalledAppVersion to return 4xx when installed app or installed app versions not found * fixes * adding pg:",discard_unknown_columns" in tables where doesn't exist * fix for Git Sensor response * fix for inception * fix for k8s resource delete * fix for pipeline already exists * fix for pipeline already exists * fix * fix * fix * fix * go mod version changes * merged * common lib main version sync * main merge --------- Co-authored-by: ShashwatDadhich <[email protected]> * feat: added posthog events for cloud provider and version (#4443) * added cloud provider event * added logger for testing * logger removed * common-lib version upgraded * introduced CASBIN_DATABASE env (#4547) * added --reuse-values in troubleshoot (#4577) Co-authored-by: Badal Kumar Prusty <[email protected]> --------- Co-authored-by: nishant <[email protected]> Co-authored-by: ashishdevtron <[email protected]> Co-authored-by: Badal Kumar <[email protected]> Co-authored-by: Badal Kumar Prusty <[email protected]> Co-authored-by: tayalrishabh96 <[email protected]> Co-authored-by: Rupin Solanki <[email protected]> Co-authored-by: Gireesh Naidu <[email protected]> Co-authored-by: Kripansh <[email protected]> Co-authored-by: subhashish-devtron <[email protected]> Co-authored-by: Prakash <[email protected]> Co-authored-by: ShashwatDadhich <[email protected]>
devtron-labs · Jan 22, 2024 · 9549cc2 · 9549cc2
1 parent bb29cbc
commit 9549cc2
Show file tree

Hide file tree

Showing 151 changed files with 3,824 additions and 2,211 deletions.
diff --git a/.github/workflows/pr-issue-validator.yaml b/.github/workflows/pr-issue-validator.yaml
@@ -29,6 +29,10 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v2
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        fetch-depth: 0
+
 
     - name: Validate Issue Reference
       env:
@@ -150,3 +154,84 @@ jobs:
             gh pr edit $PRNUM --remove-label "PR:Ready-to-Review"
             exit 1
         fi
+    - name: Check SQL file format and duplicates
+      shell: bash
+      env: 
+          pr_no: ${{ github.event.pull_request.number }}
+          GH_TOKEN: ${{ github.token }}
+      run: |
+        
+        # Fetch the latest changes from the main branch
+        git fetch origin main
+
+        # Get the list of changed files
+        git diff origin/main...HEAD --name-only > diff
+
+        echo "Changed files:"
+        cat diff
+
+        echo "Changed SQL files-:"
+        # Filter SQL files from the list of changed files
+        awk '/scripts\/sql\//' diff
+
+        # Count the number of changed SQL files in the 'scripts/sql' directory
+        count=$(awk '/scripts\/sql\//' diff | wc -l)
+
+        # Check if no SQL files were changed
+        if [[ $count == "0" ]]; then 
+          echo "No SQL files were added, Exiting from this action."
+          exit 0
+        fi
+
+        # Iterate through each changed SQL file
+        for filename in $(awk '/scripts\/sql\//' diff); do
+          echo "Checking File: $filename"
+          
+          # Check if the SQL file name is in the correct format (i.e., it ends with either '.up.sql' or '.down.sql')
+          if [[ "$filename" =~ \.(up|down)\.sql$ ]]; then
+
+            # Print a message that the file name is in the correct format
+            echo "File name: $filename is in the correct format"
+          else
+            # Print an error message
+            echo "Error: The SQL file name is not in the correct format: $filename."
+
+            # Post a comment on a GitHub pull request with the error message
+            gh pr comment $pr_no --body "The SQL file name: $filename is not in the correct format."
+
+            # Exit the script with a non-zero status code
+            exit 1 
+          fi
+
+          # Navigate to the SQL files directory
+          sql_dir="scripts/sql"
+          echo "Current directory: $(pwd)"
+          cd "$sql_dir"
+          echo "SQL files directory: $(pwd)"
+          
+          # Extract the migration number from the SQL file name
+          migration_no=$(echo "$filename" | cut -d "/" -f 3 | cut -d "_" -f 1)
+          echo "Migration Number: $migration_no"
+          
+          # Count the number of files with the same migration number
+          migration_files_present_of_this_no=$(ls | cut -d "_" -f 1 | grep -w -c "$migration_no")
+
+          # Navigate back to the original directory
+          cd ../..
+          
+          # Check the conditions based on the number of files with the same migration number
+          if [[ $migration_files_present_of_this_no == "2" ]]; then
+            echo "All looks good for this migration number."
+          elif [[ $migration_files_present_of_this_no == "1" ]]; then
+            # Only one file is present for this migration number
+            echo "Only single migration file was present for migration no.: $migration_no. either up or down migration is missing! EXITING"
+            gh pr comment $pr_no --body "Error: Only a single migration file was present for this number: $migration_no."
+            exit 1
+          else 
+            # Migration number is repeated
+            echo "Error: Migration number is repeated."
+            gh pr comment $pr_no --body "Error: The SQL file number: $migration_no is duplicated"
+            exit 1
+          fi
+        done
+
diff --git a/App.go b/App.go
@@ -21,6 +21,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"github.com/devtron-labs/common-lib/middlewares"
 	"log"
 	"net/http"
 	"os"
@@ -101,6 +102,8 @@ func (app *App) Start() {
 	server := &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: authMiddleware.Authorizer(app.sessionManager2, user.WhitelistChecker)(app.MuxRouter.Router)}
 	app.MuxRouter.Router.Use(app.loggingMiddleware.LoggingMiddleware)
 	app.MuxRouter.Router.Use(middleware.PrometheusMiddleware)
+	app.MuxRouter.Router.Use(middlewares.Recovery)
+
 	if tracerProvider != nil {
 		app.MuxRouter.Router.Use(otelmux.Middleware(otel.OTEL_ORCHESTRASTOR_SERVICE_NAME))
 	}

diff --git a/Wire.go b/Wire.go
@@ -22,6 +22,7 @@ package main
 
 import (
 	"github.com/devtron-labs/authenticator/middleware"
+	cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier"
 	pubsub1 "github.com/devtron-labs/common-lib/pubsub-lib"
 	util4 "github.com/devtron-labs/common-lib/utils/k8s"
 	"github.com/devtron-labs/devtron/api/apiToken"
@@ -65,6 +66,7 @@ import (
 	"github.com/devtron-labs/devtron/client/gitSensor"
 	"github.com/devtron-labs/devtron/client/grafana"
 	"github.com/devtron-labs/devtron/client/lens"
+	"github.com/devtron-labs/devtron/client/proxy"
 	"github.com/devtron-labs/devtron/client/telemetry"
 	"github.com/devtron-labs/devtron/internal/sql/repository"
 	app2 "github.com/devtron-labs/devtron/internal/sql/repository/app"
@@ -87,7 +89,6 @@ import (
 	"github.com/devtron-labs/devtron/pkg/appStatus"
 	"github.com/devtron-labs/devtron/pkg/appStore/chartGroup"
 	repository4 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository"
-	appStoreDeploymentFullMode "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode"
 	"github.com/devtron-labs/devtron/pkg/appStore/deployment/service"
 	appStoreDeploymentGitopsTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool"
 	"github.com/devtron-labs/devtron/pkg/appWorkflow"
@@ -144,6 +145,7 @@ func InitializeApp() (*App, error) {
 		sso.SsoConfigWireSet,
 		cluster.ClusterWireSet,
 		dashboard.DashboardWireSet,
+		proxy.ProxyWireSet,
 		client.HelmAppWireSet,
 		k8s.K8sApplicationWireSet,
 		chartRepo.ChartRepositoryWireSet,
@@ -663,6 +665,9 @@ func InitializeApp() (*App, error) {
 		wire.Bind(new(restHandler.TelemetryRestHandler), new(*restHandler.TelemetryRestHandlerImpl)),
 		telemetry.NewPosthogClient,
 
+		cloudProviderIdentifier.NewProviderIdentifierServiceImpl,
+		wire.Bind(new(cloudProviderIdentifier.ProviderIdentifierService), new(*cloudProviderIdentifier.ProviderIdentifierServiceImpl)),
+
 		telemetry.NewTelemetryEventClientImplExtended,
 		wire.Bind(new(telemetry.TelemetryEventClient), new(*telemetry.TelemetryEventClientImplExtended)),
 
@@ -713,8 +718,6 @@ func InitializeApp() (*App, error) {
 		delete2.NewDeleteServiceFullModeImpl,
 		wire.Bind(new(delete2.DeleteServiceFullMode), new(*delete2.DeleteServiceFullModeImpl)),
 
-		appStoreDeploymentFullMode.NewAppStoreDeploymentFullModeServiceImpl,
-		wire.Bind(new(appStoreDeploymentFullMode.AppStoreDeploymentFullModeService), new(*appStoreDeploymentFullMode.AppStoreDeploymentFullModeServiceImpl)),
 		appStoreDeploymentGitopsTool.NewAppStoreDeploymentArgoCdServiceImpl,
 		wire.Bind(new(appStoreDeploymentGitopsTool.AppStoreDeploymentArgoCdService), new(*appStoreDeploymentGitopsTool.AppStoreDeploymentArgoCdServiceImpl)),
 		//	util2.NewGoJsonSchemaCustomFormatChecker,

diff --git a/api/appStore/InstalledAppRestHandler.go b/api/appStore/InstalledAppRestHandler.go
@@ -22,13 +22,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	client "github.com/devtron-labs/devtron/api/helm-app/gRPC"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
 	bean2 "github.com/devtron-labs/devtron/api/bean"
-	client "github.com/devtron-labs/devtron/api/helm-app"
 	openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient"
 	"github.com/devtron-labs/devtron/api/restHandler/common"
 	"github.com/devtron-labs/devtron/client/argocdServer/application"

diff --git a/api/appStore/deployment/AppStoreDeploymentRestHandler.go b/api/appStore/deployment/AppStoreDeploymentRestHandler.go
@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	service2 "github.com/devtron-labs/devtron/api/helm-app/service"
 	"net/http"
 	"strconv"
 	"strings"
@@ -67,15 +68,15 @@ type AppStoreDeploymentRestHandlerImpl struct {
 	appStoreDeploymentService  service.AppStoreDeploymentService
 	appStoreDeploymentServiceC appStoreDeploymentCommon.AppStoreDeploymentCommonService
 	validator                  *validator.Validate
-	helmAppService             client.HelmAppService
+	helmAppService             service2.HelmAppService
 	helmAppRestHandler         client.HelmAppRestHandler
 	argoUserService            argo.ArgoUserService
 	attributesService          attributes.AttributesService
 }
 
 func NewAppStoreDeploymentRestHandlerImpl(Logger *zap.SugaredLogger, userAuthService user.UserService,
 	enforcer casbin.Enforcer, enforcerUtil rbac.EnforcerUtil, enforcerUtilHelm rbac.EnforcerUtilHelm, appStoreDeploymentService service.AppStoreDeploymentService,
-	validator *validator.Validate, helmAppService client.HelmAppService, appStoreDeploymentServiceC appStoreDeploymentCommon.AppStoreDeploymentCommonService,
+	validator *validator.Validate, helmAppService service2.HelmAppService, appStoreDeploymentServiceC appStoreDeploymentCommon.AppStoreDeploymentCommonService,
 	argoUserService argo.ArgoUserService, attributesService attributes.AttributesService) *AppStoreDeploymentRestHandlerImpl {
 	return &AppStoreDeploymentRestHandlerImpl{
 		Logger:                     Logger,

diff --git a/api/appStore/deployment/CommonDeploymentRestHandler.go b/api/appStore/deployment/CommonDeploymentRestHandler.go
@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	service2 "github.com/devtron-labs/devtron/api/helm-app/service"
 	"net/http"
 	"strconv"
 	"time"
@@ -31,7 +32,6 @@ import (
 	"github.com/devtron-labs/devtron/api/restHandler/common"
 	"github.com/devtron-labs/devtron/internal/util"
 	appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean"
-	appStoreDeploymentCommon "github.com/devtron-labs/devtron/pkg/appStore/deployment/common"
 	"github.com/devtron-labs/devtron/pkg/appStore/deployment/service"
 	"github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin"
 	"github.com/devtron-labs/devtron/pkg/auth/user"
@@ -51,35 +51,33 @@ type CommonDeploymentRestHandler interface {
 }
 
 type CommonDeploymentRestHandlerImpl struct {
-	Logger                     *zap.SugaredLogger
-	userAuthService            user.UserService
-	enforcer                   casbin.Enforcer
-	enforcerUtil               rbac.EnforcerUtil
-	enforcerUtilHelm           rbac.EnforcerUtilHelm
-	appStoreDeploymentService  service.AppStoreDeploymentService
-	appStoreDeploymentServiceC appStoreDeploymentCommon.AppStoreDeploymentCommonService
-	validator                  *validator.Validate
-	helmAppService             client.HelmAppService
-	helmAppRestHandler         client.HelmAppRestHandler
-	argoUserService            argo.ArgoUserService
+	Logger                    *zap.SugaredLogger
+	userAuthService           user.UserService
+	enforcer                  casbin.Enforcer
+	enforcerUtil              rbac.EnforcerUtil
+	enforcerUtilHelm          rbac.EnforcerUtilHelm
+	appStoreDeploymentService service.AppStoreDeploymentService
+	validator                 *validator.Validate
+	helmAppService            service2.HelmAppService
+	helmAppRestHandler        client.HelmAppRestHandler
+	argoUserService           argo.ArgoUserService
 }
 
 func NewCommonDeploymentRestHandlerImpl(Logger *zap.SugaredLogger, userAuthService user.UserService,
 	enforcer casbin.Enforcer, enforcerUtil rbac.EnforcerUtil, enforcerUtilHelm rbac.EnforcerUtilHelm, appStoreDeploymentService service.AppStoreDeploymentService,
-	validator *validator.Validate, helmAppService client.HelmAppService, appStoreDeploymentServiceC appStoreDeploymentCommon.AppStoreDeploymentCommonService,
+	validator *validator.Validate, helmAppService service2.HelmAppService,
 	helmAppRestHandler client.HelmAppRestHandler, argoUserService argo.ArgoUserService) *CommonDeploymentRestHandlerImpl {
 	return &CommonDeploymentRestHandlerImpl{
-		Logger:                     Logger,
-		userAuthService:            userAuthService,
-		enforcer:                   enforcer,
-		enforcerUtil:               enforcerUtil,
-		enforcerUtilHelm:           enforcerUtilHelm,
-		appStoreDeploymentService:  appStoreDeploymentService,
-		validator:                  validator,
-		helmAppService:             helmAppService,
-		appStoreDeploymentServiceC: appStoreDeploymentServiceC,
-		helmAppRestHandler:         helmAppRestHandler,
-		argoUserService:            argoUserService,
+		Logger:                    Logger,
+		userAuthService:           userAuthService,
+		enforcer:                  enforcer,
+		enforcerUtil:              enforcerUtil,
+		enforcerUtilHelm:          enforcerUtilHelm,
+		appStoreDeploymentService: appStoreDeploymentService,
+		validator:                 validator,
+		helmAppService:            helmAppService,
+		helmAppRestHandler:        helmAppRestHandler,
+		argoUserService:           argoUserService,
 	}
 }
 func (handler *CommonDeploymentRestHandlerImpl) getAppOfferingMode(installedAppId string, appId string) (string, *appStoreBean.InstallAppVersionDTO, error) {
@@ -91,9 +89,9 @@ func (handler *CommonDeploymentRestHandlerImpl) getAppOfferingMode(installedAppI
 			err = &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: "invalid app id"}
 			return appOfferingMode, installedAppDto, err
 		}
-		installedAppDto, err = handler.appStoreDeploymentServiceC.GetInstalledAppByClusterNamespaceAndName(appIdentifier.ClusterId, appIdentifier.Namespace, appIdentifier.ReleaseName)
+		installedAppDto, err = handler.appStoreDeploymentService.GetInstalledAppByClusterNamespaceAndName(appIdentifier.ClusterId, appIdentifier.Namespace, appIdentifier.ReleaseName)
 		if err != nil {
-			err = &util.ApiError{HttpStatusCode: http.StatusInternalServerError, UserMessage: "unable to find app in database"}
+			err = &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: "unable to find app in database"}
 			return appOfferingMode, installedAppDto, err
 		}
 		// this is the case when hyperion apps does not linked yet
@@ -117,9 +115,9 @@ func (handler *CommonDeploymentRestHandlerImpl) getAppOfferingMode(installedAppI
 			err = &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: "invalid installed app id"}
 			return appOfferingMode, installedAppDto, err
 		}
-		installedAppDto, err = handler.appStoreDeploymentServiceC.GetInstalledAppByInstalledAppId(installedAppId)
+		installedAppDto, err = handler.appStoreDeploymentService.GetInstalledAppByInstalledAppId(installedAppId)
 		if err != nil {
-			err = &util.ApiError{HttpStatusCode: http.StatusInternalServerError, UserMessage: "unable to find app in database"}
+			err = &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: "unable to find app in database"}
 			return appOfferingMode, installedAppDto, err
 		}
 	} else {

diff --git a/api/bean/GitOpsConfig.go b/api/bean/GitOpsConfig.go
@@ -13,6 +13,7 @@ type GitOpsConfigDto struct {
 	BitBucketWorkspaceId string `json:"bitBucketWorkspaceId"`
 	BitBucketProjectKey  string `json:"bitBucketProjectKey"`
 
+	// TODO refactoring: create different struct for internal fields
 	GitRepoName string `json:"gitRepoName"`
 	UserEmailId string `json:"userEmailId"`
 	Description string `json:"description"`

diff --git a/api/cluster/EnvironmentRestHandler.go b/api/cluster/EnvironmentRestHandler.go
@@ -66,7 +66,7 @@ type EnvironmentRestHandlerImpl struct {
 	validator                         *validator.Validate
 	enforcer                          casbin.Enforcer
 	deleteService                     delete2.DeleteService
-	k8sUtil                           *k8s2.K8sUtil
+	k8sUtil                           *k8s2.K8sServiceImpl
 	cfg                               *bean.Config
 }
 
@@ -75,7 +75,7 @@ type ClusterReachableResponse struct {
 	ClusterName      string `json:"clusterName"`
 }
 
-func NewEnvironmentRestHandlerImpl(svc request.EnvironmentService, logger *zap.SugaredLogger, userService user.UserService, validator *validator.Validate, enforcer casbin.Enforcer, deleteService delete2.DeleteService, k8sUtil *k8s2.K8sUtil, k8sCommonService k8s.K8sCommonService) *EnvironmentRestHandlerImpl {
+func NewEnvironmentRestHandlerImpl(svc request.EnvironmentService, logger *zap.SugaredLogger, userService user.UserService, validator *validator.Validate, enforcer casbin.Enforcer, deleteService delete2.DeleteService, k8sUtil *k8s2.K8sServiceImpl, k8sCommonService k8s.K8sCommonService) *EnvironmentRestHandlerImpl {
 	cfg := &bean.Config{}
 	err := env.Parse(cfg)
 	if err != nil {