fix!: apply fixes from feat/fix-raft-bug branch

.github/workflows/discord.yml

@@ -0,0 +1,20 @@
+on:
+    release:
+      types: [published]
+
+  jobs:
+    github-releases-to-discord:
+      runs-on: ubuntu-latest
+      steps:
+        - name: Checkout
+          uses: actions/checkout@v3
+        - name: Github Releases To Discord
+          uses: SethCohen/[email protected]
+          with:
+            webhook_url: ${{ secrets.DISCORD_WEBHOOK_URL }}
+            color: "2105893"
+            username: "Release Changelog"
+            avatar_url: "https://github.com/dgate-io.png"
+            content: "||@everyone||"
+            footer_title: "Changelog"
+            footer_timestamp: true

TODO.md

@@ -7,7 +7,6 @@
   - server management (start-proxy, stop-proxy, restart, status, logs, stats, etc.)
   - cluster management (raft commands, replica commands, etc.) (low priority)
   - other commands (backup, restore, etc.) (low priority)
-  - replace k6 with wrk for performance tests
 
 ## Add Module Tests
 
@@ -16,9 +15,6 @@
   - [ ] - Add option to specify export variables when ambiguous (?)
   - [ ] - check how global variable conflicts are handled
 
-## Start using Pkl
-
-replace dgate server config with pkl.
 
 ## dgate-cli declaritive config
 
@@ -70,10 +66,6 @@ expose metrics for the following:
 - Add Transactions
   - [ ] - Add transactional support for admin API
 
-## DGate Documentation (dgate.io/docs)
-
-Use Docusaurus to create the documentation for DGate.
-
 ## DGate Admin Console (low priority)
 
 Admin Console is a web-based interface that can be used to manage the state of the cluster. Manage resource, view logs, stats, and more. It can also be used to develop and test modules directly in the browser.
@@ -136,14 +128,6 @@ A good example of a bundle would be a bundle that adds support for OAuth2 authen
 Differing from common resource versioning, modules can have multiple versions that can be used at the same time. This can be used to test new versions of modules before deploying them to the cluster.
 
 
-## DGate CLI - argument variable suggestions
-
-For example, if the user types an argument that is not recognized, the CLI can suggest the correct argument by search the available arguments and finding the closest match.
-```
-dgate-cli ns mk my-ns nmae=my-ns
-Variable 'nmae' is not recognized. Did you mean 'name'?
-```
-
 ## DGate CLI - help command show required variables
 
 When the user runs the help command, the CLI should show the required variables for the command. For example, if the user runs `dgate-cli ns mk --help`, the CLI should show the required variables for the `ns mk` command. `name` is a required variable for the `ns mk` command. Also, the CLI should show non-required variables.
@@ -159,4 +143,8 @@ Add stack tracing for typescript modules.
 
 Currently, Raft Implementation is tightly coupled with the Admin API. This makes it difficult to change the Raft Implementation without changing the Admin API. Decouple the Raft Implementation from the Admin API to make it easier to change the Raft Implementation.
 
-## Add Telemetry (sentry, datadog, etc.)
+## Add Telemetry (sentry, datadog, etc.)
+
+## ResourceManager callback for resource changes
+
+Add a callback to the ResourceManager that is called when a resource is changed. This can be used to invalidate caches, update modules, and more.

internal/admin/admin_routes.go

@@ -3,7 +3,6 @@ package admin
 import (
 	"fmt"
 	"log"
-	"net"
 	"net/http"
 	"strings"
 
@@ -75,43 +74,20 @@ func configureRoutes(
 		}
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			if ipList.Len() > 0 {
-				remoteHost, _, err := net.SplitHostPort(r.RemoteAddr)
-				if err != nil {
-					remoteHost = r.RemoteAddr
-				}
-				allowed, err := ipList.Contains(remoteHost)
-				if !allowed && adminConfig.XForwardedForDepth > 0 {
-					xForwardedForIps := r.Header.Values("X-Forwarded-For")
-					count := min(adminConfig.XForwardedForDepth, len(xForwardedForIps))
-					for i := 0; i < count; i++ {
-						allowed, err = ipList.Contains(xForwardedForIps[i])
-						if err != nil {
-							logger.Error("error checking x-forwarded-for ip",
-								zap.Error(err),
-							)
-							if conf.Debug {
-								http.Error(w, "Bad Request: could not parse x-forwarded-for IP address", http.StatusBadRequest)
-							}
-							http.Error(w, "Bad Request", http.StatusBadRequest)
-							return
-						}
-						if allowed {
-							break
-						}
-					}
-				}
-
+				remoteIp := util.GetTrustedIP(r,
+					conf.AdminConfig.XForwardedForDepth)
+				allowed, err := ipList.Contains(remoteIp)
 				if err != nil {
 					if conf.Debug {
 						http.Error(w, err.Error(), http.StatusInternalServerError)
 						return
 					}
-					http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+					http.Error(w, "could not parse X-Forwarded-For IP", http.StatusBadRequest)
 					return
 				}
 				if !allowed {
 					if conf.Debug {
-						http.Error(w, "Unauthorized IP Address: "+remoteHost, http.StatusUnauthorized)
+						http.Error(w, "Unauthorized IP Address: "+remoteIp, http.StatusUnauthorized)
 						return
 					}
 					http.Error(w, "Unauthorized", http.StatusUnauthorized)
@@ -138,24 +114,26 @@ func configureRoutes(
 				} else if adminConfig.KeyAuth.HeaderName != "" {
 					key = r.Header.Get(adminConfig.KeyAuth.HeaderName)
 				} else {
-					key = r.Header.Get("X-API-Key")
+					key = r.Header.Get("X-DGate-Key")
 				}
 				if _, keyFound := keyMap[key]; !keyFound {
 					http.Error(w, "Unauthorized", http.StatusUnauthorized)
 					return
 				}
 			}
-			raftInstance := cs.Raft()
-			if r.Method == http.MethodPut && raftInstance != nil {
-				leader := raftInstance.Leader()
-				if leader == "" {
-					util.JsonError(w, http.StatusServiceUnavailable, "raft: no leader")
-					return
-				}
-				if raftInstance.State() != raft.Leader {
-					r.URL.Host = string(leader)
-					http.Redirect(w, r, r.URL.String(), http.StatusTemporaryRedirect)
-					return
+			if raftInstance := cs.Raft(); raftInstance != nil {
+				if r.Method == http.MethodPut || r.Method == http.MethodDelete {
+					leader := raftInstance.Leader()
+					if leader == "" {
+						// TODO: add a way to wait for a leader with a timeout
+						util.JsonError(w, http.StatusServiceUnavailable, "raft: no leader")
+						return
+					}
+					if raftInstance.State() != raft.Leader {
+						r.URL.Host = string(leader)
+						http.Redirect(w, r, r.URL.String(), http.StatusTemporaryRedirect)
+						return
+					}
 				}
 			}
 
@@ -165,10 +143,14 @@ func configureRoutes(
 
 	server.Get("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "text/plain")
-		w.Header().Set("X-DGate-Raft", fmt.Sprintf("%t", cs.Raft() != nil))
 		w.Header().Set("X-DGate-WatchOnly", fmt.Sprintf("%t", adminConfig.WatchOnly))
 		w.Header().Set("X-DGate-ChangeHash", fmt.Sprintf("%d", cs.ChangeHash()))
-		w.Header().Set("X-DGate-AdminAPI", "true")
+		if raftInstance := cs.Raft(); raftInstance != nil {
+			w.Header().Set(
+				"X-DGate-Raft-State",
+				raftInstance.State().String(),
+			)
+		}
 		w.WriteHeader(http.StatusOK)
 		w.Write([]byte("DGate Admin API"))
 	}))

internal/admin/changestate/testutil/change_state.go

@@ -65,13 +65,8 @@ func (m *MockChangeState) ReloadState(a bool, cls ...*spec.ChangeLog) error {
 	return m.Called(a, cls).Error(0)
 }
 
-// SetReady implements changestate.ChangeState.
-func (m *MockChangeState) SetReady() {
-	m.Called()
-}
-
 // SetupRaft implements changestate.ChangeState.
-func (m *MockChangeState) SetupRaft(*raft.Raft, *raft.Config) {
+func (m *MockChangeState) SetupRaft(*raft.Raft, chan raft.Observation) {
 	m.Called().Error(0)
 }
 

internal/admin/routes/collection_routes.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
-	"time"
 
 	"github.com/dgate-io/chi-router"
 	"github.com/dgate-io/dgate/internal/admin/changestate"
@@ -71,8 +70,7 @@ func ConfigureCollectionAPI(server chi.Router, logger *zap.Logger, cs changestat
 		}
 
 		if repl := cs.Raft(); repl != nil {
-			future := repl.Barrier(time.Second * 5)
-			if err := future.Error(); err != nil {
+			if err := cs.WaitForChanges(); err != nil {
 				util.JsonError(w, http.StatusInternalServerError, err.Error())
 				return
 			}
@@ -279,8 +277,7 @@ func ConfigureCollectionAPI(server chi.Router, logger *zap.Logger, cs changestat
 		}
 
 		if repl := cs.Raft(); repl != nil {
-			future := repl.Barrier(time.Second * 5)
-			if err := future.Error(); err != nil {
+			if err := cs.WaitForChanges(); err != nil {
 				util.JsonError(w, http.StatusInternalServerError, err.Error())
 				return
 			}

internal/admin/routes/domain_routes.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
-	"time"
 
 	"github.com/dgate-io/chi-router"
 	"github.com/dgate-io/dgate/internal/admin/changestate"
@@ -49,8 +48,7 @@ func ConfigureDomainAPI(server chi.Router, logger *zap.Logger, cs changestate.Ch
 		}
 
 		if repl := cs.Raft(); repl != nil {
-			future := repl.Barrier(time.Second * 5)
-			if err := future.Error(); err != nil {
+			if err := cs.WaitForChanges(); err != nil {
 				util.JsonError(w, http.StatusInternalServerError, err.Error())
 				return
 			}

internal/admin/routes/misc_routes.go

@@ -3,7 +3,6 @@ package routes
 import (
 	"encoding/json"
 	"net/http"
-	"time"
 
 	"github.com/dgate-io/chi-router"
 	"github.com/dgate-io/dgate/internal/admin/changestate"
@@ -14,8 +13,7 @@ import (
 func ConfigureChangeLogAPI(server chi.Router, cs changestate.ChangeState, appConfig *config.DGateConfig) {
 	server.Get("/changelog/hash", func(w http.ResponseWriter, r *http.Request) {
 		if repl := cs.Raft(); repl != nil {
-			future := repl.Barrier(time.Second * 5)
-			if err := future.Error(); err != nil {
+			if err := cs.WaitForChanges(); err != nil {
 				util.JsonError(w, http.StatusInternalServerError, err.Error())
 				return
 			}

internal/admin/routes/module_routes.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
-	"time"
 
 	"github.com/dgate-io/chi-router"
 	"github.com/dgate-io/dgate/internal/admin/changestate"
@@ -49,8 +48,7 @@ func ConfigureModuleAPI(server chi.Router, logger *zap.Logger, cs changestate.Ch
 			return
 		}
 		if repl := cs.Raft(); repl != nil {
-			future := repl.Barrier(time.Second * 5)
-			if err := future.Error(); err != nil {
+			if err := cs.WaitForChanges(); err != nil {
 				util.JsonError(w, http.StatusInternalServerError, err.Error())
 				return
 			}

internal/admin/routes/namespace_routes.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
-	"time"
 
 	"github.com/dgate-io/chi-router"
 	"github.com/dgate-io/dgate/internal/admin/changestate"
@@ -43,8 +42,7 @@ func ConfigureNamespaceAPI(server chi.Router, logger *zap.Logger, cs changestate
 		}
 
 		if repl := cs.Raft(); repl != nil {
-			future := repl.Barrier(time.Second * 5)
-			if err := future.Error(); err != nil {
+			if err := cs.WaitForChanges(); err != nil {
 				util.JsonError(w, http.StatusInternalServerError, err.Error())
 				return
 			}

internal/admin/routes/route_routes.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
-	"time"
 
 	"github.com/dgate-io/chi-router"
 	"github.com/dgate-io/dgate/internal/admin/changestate"
@@ -51,8 +50,7 @@ func ConfigureRouteAPI(server chi.Router, logger *zap.Logger, cs changestate.Cha
 		}
 
 		if repl := cs.Raft(); repl != nil {
-			future := repl.Barrier(time.Second * 5)
-			if err := future.Error(); err != nil {
+			if err := cs.WaitForChanges(); err != nil {
 				util.JsonError(w, http.StatusInternalServerError, err.Error())
 				return
 			}

internal/admin/routes/secret_routes.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
-	"time"
 
 	"github.com/dgate-io/chi-router"
 	"github.com/dgate-io/dgate/internal/admin/changestate"
@@ -49,8 +48,7 @@ func ConfigureSecretAPI(server chi.Router, logger *zap.Logger, cs changestate.Ch
 			return
 		}
 		if repl := cs.Raft(); repl != nil {
-			future := repl.Barrier(time.Second * 5)
-			if err := future.Error(); err != nil {
+			if err := cs.WaitForChanges(); err != nil {
 				util.JsonError(w, http.StatusInternalServerError, err.Error())
 				return
 			}

internal/admin/routes/service_routes.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"time"
 
 	"github.com/dgate-io/chi-router"
 	"github.com/dgate-io/dgate/internal/admin/changestate"
@@ -67,8 +66,7 @@ func ConfigureServiceAPI(server chi.Router, logger *zap.Logger, cs changestate.C
 
 		if repl := cs.Raft(); repl != nil {
 			logger.Debug("Waiting for raft barrier")
-			future := repl.Barrier(time.Second * 5)
-			if err := future.Error(); err != nil {
+			if err := cs.WaitForChanges(); err != nil {
 				util.JsonError(w, http.StatusInternalServerError, err.Error())
 				return
 			}

internal/config/config.go

@@ -39,19 +39,22 @@ type (
 	}
 
 	DGateProxyConfig struct {
-		Host                 string                   `koanf:"host"`
-		Port                 int                      `koanf:"port"`
-		TLS                  *DGateTLSConfig          `koanf:"tls"`
-		EnableH2C            bool                     `koanf:"enable_h2c"`
-		EnableHTTP2          bool                     `koanf:"enable_http2"`
-		EnableConsoleLogger  bool                     `koanf:"enable_console_logger"`
-		RedirectHttpsDomains []string                 `koanf:"redirect_https"`
-		AllowedDomains       []string                 `koanf:"allowed_domains"`
-		GlobalHeaders        map[string]string        `koanf:"global_headers"`
-		Transport            DGateHttpTransportConfig `koanf:"client_transport"`
+		Host                     string                   `koanf:"host"`
+		Port                     int                      `koanf:"port"`
+		TLS                      *DGateTLSConfig          `koanf:"tls"`
+		EnableH2C                bool                     `koanf:"enable_h2c"`
+		EnableHTTP2              bool                     `koanf:"enable_http2"`
+		EnableConsoleLogger      bool                     `koanf:"enable_console_logger"`
+		RedirectHttpsDomains     []string                 `koanf:"redirect_https"`
+		AllowedDomains           []string                 `koanf:"allowed_domains"`
+		GlobalHeaders            map[string]string        `koanf:"global_headers"`
+		Transport                DGateHttpTransportConfig `koanf:"client_transport"`
+		DisableXForwardedHeaders bool                     `koanf:"disable_x_forwarded_headers"`
+		StrictMode               bool                     `koanf:"strict_mode"`
+		XForwardedForDepth       int                      `koanf:"x_forwarded_for_depth"`
+
 		// WARN: debug use only
-		InitResources            *DGateResources `koanf:"init_resources"`
-		DisableXForwardedHeaders bool            `koanf:"disable_x_forwarded_headers"`
+		InitResources *DGateResources `koanf:"init_resources"`
 	}
 
 	DGateTestServerConfig struct {