Adjust uploader code for IdWrapper removal

UTDNebula · Apr 24, 2024 · 75eaabb · 75eaabb
1 parent efb0f9c
commit 75eaabb
Show file tree

Hide file tree

Showing 10 changed files with 426 additions and 309 deletions.
diff --git a/go.mod b/go.mod
@@ -4,32 +4,55 @@ go 1.19
 
 require (
 	github.com/PuerkitoBio/goquery v1.8.1
-	github.com/UTDNebula/nebula-api/api v0.0.0-20231204040332-adccdc08b203
-	github.com/chromedp/cdproto v0.0.0-20240116100315-4a0ec5e4c400
-	github.com/chromedp/chromedp v0.9.3
+	github.com/UTDNebula/nebula-api/api v0.0.0-20240423212728-2ef02f280c6c
+	github.com/chromedp/cdproto v0.0.0-20240421230201-ab917191657d
+	github.com/chromedp/chromedp v0.9.5
 	github.com/joho/godotenv v1.5.1
 	go.mongodb.org/mongo-driver v1.15.0
 )
 
 require (
 	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/bytedance/sonic v1.11.5 // indirect
+	github.com/bytedance/sonic/loader v0.1.1 // indirect
 	github.com/chromedp/sysutil v1.0.0 // indirect
+	github.com/cloudwego/base64x v0.1.3 // indirect
+	github.com/cloudwego/iasm v0.2.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/gin-gonic/gin v1.9.1 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.19.0 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
-	github.com/gobwas/ws v1.3.0 // indirect
-	github.com/golang/snappy v0.0.1 // indirect
-	github.com/google/go-cmp v0.5.5 // indirect
+	github.com/gobwas/ws v1.3.2 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/gorilla/schema v1.3.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/montanaflynn/stats v0.7.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.1 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.12 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
-	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	golang.org/x/crypto v0.17.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
+	golang.org/x/arch v0.7.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
diff --git a/go.sum b/go.sum
diff --git a/parser/courseParser.go b/parser/courseParser.go
@@ -48,7 +48,7 @@ func parseCourse(courseNum string, session schema.AcademicSession, rowInfo map[s
 
 	course = &schema.Course{}
 
-	course.Id = schema.IdWrapper(primitive.NewObjectID().Hex())
+	course.Id = primitive.NewObjectID()
 	course.Course_number = idMatches[2]
 	course.Subject_prefix = idMatches[1]
 	course.Title = rowInfo["Course Title:"]

diff --git a/parser/parser.go b/parser/parser.go
@@ -8,22 +8,23 @@ import (
 	"time"
 
 	"github.com/UTDNebula/api-tools/utils"
+	"go.mongodb.org/mongo-driver/bson/primitive"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/UTDNebula/nebula-api/api/schema"
 )
 
 // Main dictionaries for mapping unique keys to the actual data
-var Sections = make(map[schema.IdWrapper]*schema.Section)
+var Sections = make(map[primitive.ObjectID]*schema.Section)
 var Courses = make(map[string]*schema.Course)
 var Professors = make(map[string]*schema.Professor)
 
 // Auxilliary dictionaries for mapping the generated ObjectIDs to the keys used in the above maps, used for validation purposes
-var CourseIDMap = make(map[schema.IdWrapper]string)
-var ProfessorIDMap = make(map[schema.IdWrapper]string)
+var CourseIDMap = make(map[primitive.ObjectID]string)
+var ProfessorIDMap = make(map[primitive.ObjectID]string)
 
 // Requisite parser closures associated with courses
-var ReqParsers = make(map[schema.IdWrapper]func())
+var ReqParsers = make(map[primitive.ObjectID]func())
 
 // Grade mappings for section grade distributions, mapping is MAP[SEMESTER] -> MAP[SUBJECT + NUMBER + SECTION] -> GRADE DISTRIBUTION
 var GradeMap map[string]map[string][]int

diff --git a/parser/professorParser.go b/parser/professorParser.go
@@ -8,10 +8,10 @@ import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
-func parseProfessors(sectionId schema.IdWrapper, rowInfo map[string]string, classInfo map[string]string) []schema.IdWrapper {
+func parseProfessors(sectionId primitive.ObjectID, rowInfo map[string]string, classInfo map[string]string) []primitive.ObjectID {
 	professorText := rowInfo["Instructor(s):"]
 	professorMatches := personRegexp.FindAllStringSubmatch(professorText, -1)
-	var profRefs []schema.IdWrapper = make([]schema.IdWrapper, 0, len(professorMatches))
+	var profRefs []primitive.ObjectID = make([]primitive.ObjectID, 0, len(professorMatches))
 	for _, match := range professorMatches {
 
 		nameStr := utils.TrimWhitespace(match[1])
@@ -35,12 +35,12 @@ func parseProfessors(sectionId schema.IdWrapper, rowInfo map[string]string, clas
 		}
 
 		prof = &schema.Professor{}
-		prof.Id = schema.IdWrapper(primitive.NewObjectID().Hex())
+		prof.Id = primitive.NewObjectID()
 		prof.First_name = firstName
 		prof.Last_name = lastName
 		prof.Titles = []string{utils.TrimWhitespace(match[2])}
 		prof.Email = utils.TrimWhitespace(match[3])
-		prof.Sections = []schema.IdWrapper{sectionId}
+		prof.Sections = []primitive.ObjectID{sectionId}
 		profRefs = append(profRefs, prof.Id)
 		Professors[profKey] = prof
 		ProfessorIDMap[prof.Id] = profKey

diff --git a/parser/sectionParser.go b/parser/sectionParser.go
@@ -22,7 +22,7 @@ func parseSection(courseRef *schema.Course, classNum string, syllabusURI string,
 
 	section := &schema.Section{}
 
-	section.Id = schema.IdWrapper(primitive.NewObjectID().Hex())
+	section.Id = primitive.NewObjectID()
 	section.Section_number = idMatches[1]
 	section.Course_reference = courseRef.Id
 

diff --git a/scrapers/events.go b/scrapers/events.go
@@ -307,7 +307,7 @@ func ScrapeEvents(outDir string) {
 		utils.VPrintf("Scraped contact phone info: %s", contactInformationPhone)
 
 		events = append(events, schema.Event{
-			Id:                 schema.IdWrapper(primitive.NewObjectID().Hex()),
+			Id:                 primitive.NewObjectID(),
 			Summary:            summary,
 			Location:           location,
 			StartTime:          dateTimeStart,

diff --git a/scrapers/organizations.go b/scrapers/organizations.go
@@ -227,7 +227,7 @@ func parseCsvRecord(ctx context.Context, entry []string) (*schema.Organization,
 		utils.VPrintf("Error retrieving image for %s: %v", entry[0], err)
 	}
 	return &schema.Organization{
-		Id:             schema.IdWrapper(primitive.NewObjectID().Hex()),
+		Id:             primitive.NewObjectID(),
 		Title:          entry[0],
 		Categories:     parseCategories(entry[1]),
 		Description:    entry[2],

diff --git a/scrapers/profiles.go b/scrapers/profiles.go
@@ -275,7 +275,7 @@ func ScrapeProfiles(outDir string) {
 		utils.VPrintf("Parsed list! #: %s, Office: %v", phoneNumber, office)
 
 		professors = append(professors, schema.Professor{
-			Id:           schema.IdWrapper(primitive.NewObjectID().Hex()),
+			Id:           primitive.NewObjectID(),
 			First_name:   firstName,
 			Last_name:    lastName,
 			Titles:       titles,
@@ -285,7 +285,7 @@ func ScrapeProfiles(outDir string) {
 			Profile_uri:  link,
 			Image_uri:    imageUri,
 			Office_hours: []schema.Meeting{},
-			Sections:     []schema.IdWrapper{},
+			Sections:     []primitive.ObjectID{},
 		})
 
 		utils.VPrintf("Scraped profile for %s %s!", firstName, lastName)