Added dynamic retrieval of course prefix list for coursebook scraping

go.mod

@@ -1,6 +1,6 @@
 module github.com/UTDNebula/api-tools
 
-go 1.23
+go 1.23.0
 
 require (
 	github.com/PuerkitoBio/goquery v1.8.1

scrapers/coursebook.go

@@ -28,37 +28,7 @@ func ScrapeCoursebook(term string, startPrefix string, outDir string) {
 	chromedpCtx, cancel := utils.InitChromeDp()
 	defer cancel()
 
-	// Refresh the token
-	refreshToken(chromedpCtx)
-
-	log.Printf("Finding course prefix nodes...")
-
-	var coursePrefixes []string
-	var coursePrefixNodes []*cdp.Node
-
-	// Get option elements for course prefix dropdown
-	err := chromedp.Run(chromedpCtx,
-		chromedp.Navigate("https://coursebook.utdallas.edu"),
-		chromedp.Nodes("select#combobox_cp option", &coursePrefixNodes, chromedp.ByQueryAll),
-	)
-
-	if err != nil {
-		log.Panic(err)
-	}
-
-	log.Println("Found the course prefix nodes!")
-
-	log.Println("Finding course prefixes...")
-
-	// Remove the first option due to it being empty
-	coursePrefixNodes = coursePrefixNodes[1:]
-
-	// Get the value of each option and append to coursePrefixes
-	for _, node := range coursePrefixNodes {
-		coursePrefixes = append(coursePrefixes, node.AttributeValue("value"))
-	}
-
-	log.Println("Found the course prefixes!")
+	coursePrefixes := utils.GetCoursePrefixes(chromedpCtx)
 
 	// Find index of starting prefix, if one has been given
 	startPrefixIndex := 0

utils/methods.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 
 	"github.com/chromedp/cdproto/network"
+	"github.com/chromedp/cdproto/cdp"
 	"github.com/chromedp/chromedp"
 )
 
@@ -251,3 +252,39 @@ func RetryHTTP(requestCreator func() *http.Request, client *http.Client, retryCa
 	}
 	return res, err
 }
+
+func GetCoursePrefixes(chromedpCtx context.Context) []string {
+	// Refresh the token
+	// refreshToken(chromedpCtx)
+
+	log.Printf("Finding course prefix nodes...")
+
+	var coursePrefixes []string
+	var coursePrefixNodes []*cdp.Node
+
+	// Get option elements for course prefix dropdown
+	err := chromedp.Run(chromedpCtx,
+		chromedp.Navigate("https://coursebook.utdallas.edu"),
+		chromedp.Nodes("select#combobox_cp option", &coursePrefixNodes, chromedp.ByQueryAll),
+	)
+
+	if err != nil {
+		log.Panic(err)
+	}
+
+	log.Println("Found the course prefix nodes!")
+
+	log.Println("Finding course prefixes...")
+
+	// Remove the first option due to it being empty
+	coursePrefixNodes = coursePrefixNodes[1:]
+
+	// Get the value of each option and append to coursePrefixes
+	for _, node := range coursePrefixNodes {
+		coursePrefixes = append(coursePrefixes, node.AttributeValue("value"))
+	}
+
+	log.Println("Found the course prefixes!")
+
+	return coursePrefixes
+}