Update status pages

common/status_pages/status_pages.go

@@ -1735,7 +1735,7 @@ var StatusPages = []api.StatusPage{
 		URL:  "http://status.semaphoreci.com",
 		Name: "Semaphore",
 	}, {
-		URL:  "https://status.getsentry.com",
+		URL:  "https://status.sentry.io/",
 		Name: "Sentry",
 	}, {
 		URL:  "https://status.shortcut.com",
@@ -4234,7 +4234,7 @@ var StatusPages = []api.StatusPage{
 		URL:  "https://connectedfleet.michelin.com/en/status",
 		Name: "Michelin",
 	}, {
-		URL:  "https://health.aws.amazon.com/health/status",
+		URL:  "https://status.aws.amazon.com",
 		Name: "Amazon",
 	}, {
 		URL:  "https://status.qualitestgroup.com/Home/Privacy",

scraper/internal/scraper/dbgroomer/dbgroomer.go

@@ -0,0 +1,47 @@
+package dbgroomer
+
+import (
+	"context"
+	"github.com/metoro-io/statusphere/common/db"
+	"github.com/metoro-io/statusphere/common/status_pages"
+	"go.uber.org/zap"
+)
+
+type DbGroomer struct {
+	dbClient *db.DbClient
+	logger   *zap.Logger
+}
+
+func NewDbGroomer(logger *zap.Logger, dbClient *db.DbClient) *DbGroomer {
+	return &DbGroomer{
+		logger:   logger,
+		dbClient: dbClient,
+	}
+}
+
+func (d *DbGroomer) Groom() {
+	go func() {
+		d.logger.Info("grooming status pages")
+		// Delete any status page where it isn't in our local list
+		statusPages := status_pages.StatusPages
+		urls := make(map[string]bool)
+		for _, statusPage := range statusPages {
+			urls[statusPage.URL] = true
+		}
+
+		statusPages, err := d.dbClient.GetAllStatusPages(context.Background())
+		if err != nil {
+			d.logger.Error("failed to get all status pages", zap.Error(err))
+		}
+		for _, statusPage := range statusPages {
+			if _, ok := urls[statusPage.URL]; !ok {
+				d.logger.Info("deleting status page", zap.String("url", statusPage.URL))
+				err := d.dbClient.DeleteStatusPage(context.Background(), statusPage.URL)
+				if err != nil {
+					d.logger.Error("failed to delete status page", zap.Error(err))
+				}
+			}
+		}
+		d.logger.Info("finished grooming status pages")
+	}()
+}