Created toolkit uploader

go.mod

@@ -4,22 +4,55 @@ go 1.19
 
 require (
 	github.com/PuerkitoBio/goquery v1.8.1
-	github.com/UTDNebula/nebula-api/api v0.0.0-20231204040332-adccdc08b203
-	github.com/chromedp/cdproto v0.0.0-20240116100315-4a0ec5e4c400
-	github.com/chromedp/chromedp v0.9.3
+	github.com/UTDNebula/nebula-api/api v0.0.0-20240423212728-2ef02f280c6c
+	github.com/chromedp/cdproto v0.0.0-20240421230201-ab917191657d
+	github.com/chromedp/chromedp v0.9.5
 	github.com/joho/godotenv v1.5.1
-	go.mongodb.org/mongo-driver v1.13.0
+	go.mongodb.org/mongo-driver v1.15.0
 )
 
 require (
 	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/bytedance/sonic v1.11.5 // indirect
+	github.com/bytedance/sonic/loader v0.1.1 // indirect
 	github.com/chromedp/sysutil v1.0.0 // indirect
+	github.com/cloudwego/base64x v0.1.3 // indirect
+	github.com/cloudwego/iasm v0.2.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/gin-gonic/gin v1.9.1 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.19.0 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
-	github.com/gobwas/ws v1.3.0 // indirect
-	github.com/google/go-cmp v0.5.5 // indirect
+	github.com/gobwas/ws v1.3.2 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/gorilla/schema v1.3.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.14.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/montanaflynn/stats v0.7.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.1 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.12 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
+	golang.org/x/arch v0.7.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

parser/courseParser.go

@@ -48,7 +48,7 @@ func parseCourse(courseNum string, session schema.AcademicSession, rowInfo map[s
 
 	course = &schema.Course{}
 
-	course.Id = schema.IdWrapper(primitive.NewObjectID().Hex())
+	course.Id = primitive.NewObjectID()
 	course.Course_number = idMatches[2]
 	course.Subject_prefix = idMatches[1]
 	course.Title = rowInfo["Course Title:"]

parser/parser.go

@@ -8,22 +8,23 @@ import (
 	"time"
 
 	"github.com/UTDNebula/api-tools/utils"
+	"go.mongodb.org/mongo-driver/bson/primitive"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/UTDNebula/nebula-api/api/schema"
 )
 
 // Main dictionaries for mapping unique keys to the actual data
-var Sections = make(map[schema.IdWrapper]*schema.Section)
+var Sections = make(map[primitive.ObjectID]*schema.Section)
 var Courses = make(map[string]*schema.Course)
 var Professors = make(map[string]*schema.Professor)
 
 // Auxilliary dictionaries for mapping the generated ObjectIDs to the keys used in the above maps, used for validation purposes
-var CourseIDMap = make(map[schema.IdWrapper]string)
-var ProfessorIDMap = make(map[schema.IdWrapper]string)
+var CourseIDMap = make(map[primitive.ObjectID]string)
+var ProfessorIDMap = make(map[primitive.ObjectID]string)
 
 // Requisite parser closures associated with courses
-var ReqParsers = make(map[schema.IdWrapper]func())
+var ReqParsers = make(map[primitive.ObjectID]func())
 
 // Grade mappings for section grade distributions, mapping is MAP[SEMESTER] -> MAP[SUBJECT + NUMBER + SECTION] -> GRADE DISTRIBUTION
 var GradeMap map[string]map[string][]int

parser/professorParser.go

@@ -8,10 +8,10 @@ import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
-func parseProfessors(sectionId schema.IdWrapper, rowInfo map[string]string, classInfo map[string]string) []schema.IdWrapper {
+func parseProfessors(sectionId primitive.ObjectID, rowInfo map[string]string, classInfo map[string]string) []primitive.ObjectID {
 	professorText := rowInfo["Instructor(s):"]
 	professorMatches := personRegexp.FindAllStringSubmatch(professorText, -1)
-	var profRefs []schema.IdWrapper = make([]schema.IdWrapper, 0, len(professorMatches))
+	var profRefs []primitive.ObjectID = make([]primitive.ObjectID, 0, len(professorMatches))
 	for _, match := range professorMatches {
 
 		nameStr := utils.TrimWhitespace(match[1])
@@ -35,12 +35,12 @@ func parseProfessors(sectionId schema.IdWrapper, rowInfo map[string]string, clas
 		}
 
 		prof = &schema.Professor{}
-		prof.Id = schema.IdWrapper(primitive.NewObjectID().Hex())
+		prof.Id = primitive.NewObjectID()
 		prof.First_name = firstName
 		prof.Last_name = lastName
 		prof.Titles = []string{utils.TrimWhitespace(match[2])}
 		prof.Email = utils.TrimWhitespace(match[3])
-		prof.Sections = []schema.IdWrapper{sectionId}
+		prof.Sections = []primitive.ObjectID{sectionId}
 		profRefs = append(profRefs, prof.Id)
 		Professors[profKey] = prof
 		ProfessorIDMap[prof.Id] = profKey

parser/sectionParser.go

@@ -22,7 +22,7 @@ func parseSection(courseRef *schema.Course, classNum string, syllabusURI string,
 
 	section := &schema.Section{}
 
-	section.Id = schema.IdWrapper(primitive.NewObjectID().Hex())
+	section.Id = primitive.NewObjectID()
 	section.Section_number = idMatches[1]
 	section.Course_reference = courseRef.Id
 

scrapers/events.go

@@ -307,7 +307,7 @@ func ScrapeEvents(outDir string) {
 		utils.VPrintf("Scraped contact phone info: %s", contactInformationPhone)
 
 		events = append(events, schema.Event{
-			Id:                 schema.IdWrapper(primitive.NewObjectID().Hex()),
+			Id:                 primitive.NewObjectID(),
 			Summary:            summary,
 			Location:           location,
 			StartTime:          dateTimeStart,

scrapers/organizations.go

@@ -227,7 +227,7 @@ func parseCsvRecord(ctx context.Context, entry []string) (*schema.Organization,
 		utils.VPrintf("Error retrieving image for %s: %v", entry[0], err)
 	}
 	return &schema.Organization{
-		Id:             schema.IdWrapper(primitive.NewObjectID().Hex()),
+		Id:             primitive.NewObjectID(),
 		Title:          entry[0],
 		Categories:     parseCategories(entry[1]),
 		Description:    entry[2],

scrapers/profiles.go

@@ -275,7 +275,7 @@ func ScrapeProfiles(outDir string) {
 		utils.VPrintf("Parsed list! #: %s, Office: %v", phoneNumber, office)
 
 		professors = append(professors, schema.Professor{
-			Id:           schema.IdWrapper(primitive.NewObjectID().Hex()),
+			Id:           primitive.NewObjectID(),
 			First_name:   firstName,
 			Last_name:    lastName,
 			Titles:       titles,
@@ -285,7 +285,7 @@ func ScrapeProfiles(outDir string) {
 			Profile_uri:  link,
 			Image_uri:    imageUri,
 			Office_hours: []schema.Meeting{},
-			Sections:     []schema.IdWrapper{},
+			Sections:     []primitive.ObjectID{},
 		})
 
 		utils.VPrintf("Scraped profile for %s %s!", firstName, lastName)

uploader/database.go

@@ -4,7 +4,7 @@
 
 package uploader
 
-/* import (
+import (
 	//"go.mongodb.org/mongo-driver/bson"
 	//"go.mongodb.org/mongo-driver/bson/primitive"
 	"context"
@@ -17,18 +17,15 @@ package uploader
 )
 
 func connectDB() *mongo.Client {
-	client, err := mongo.NewClient(options.Client().ApplyURI(getEnvMongoURI()))
-	if err != nil {
-		log.Panic("Unable to create MongoDB client")
-		os.Exit(1)
-	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	err = client.Connect(ctx)
+	opts := options.Client().ApplyURI(getEnvMongoURI())
+
+	client, err := mongo.Connect(ctx, opts)
 	if err != nil {
-		log.Panic("Unable to connect to database")
+		log.Panic("Unable to create MongoDB client and connect to database")
 		os.Exit(1)
 	}
 
@@ -56,4 +53,4 @@ func getEnvMongoURI() string {
 		os.Exit(1)
 	}
 	return uri
-} */
+}

uploader/uploader.go

@@ -4,15 +4,23 @@
 
 package uploader
 
-/*
 import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"time"
 
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
-	"github.com/joho/godotenv"
 
+	"github.com/UTDNebula/nebula-api/api/schema"
+	"github.com/joho/godotenv"
 )
 
 //  It's important to note that all of the files must be updated/uploaded TOGETHER!
@@ -22,23 +30,147 @@ import (
 //  Also note that this uploader assumes that the collection names match the names of these files, which they should.
 //  If the names of these collections ever change, the file names should be updated accordingly.
 
-var filesToUpload []string = []string{"courses.json", "professors.json", "sections.json"}
-*/
+var filesToUpload [3]string = [3]string{"courses.json", "professors.json", "sections.json"}
+
 func Upload(inDir string, replace bool) {
-	/*
-		//Load env vars
-		if err := godotenv.Load(); err != nil {
+
+	//Load env vars
+	if err := godotenv.Load(); err != nil {
 		log.Panic("Error loading .env file")
-		}
-		//Connect to mongo
-		client := connectDB()
-		for _, path := range(filesToUpload) {
-		//Open data file for reading
-		fptr, err := os.Open(path)
+	}
+
+	//Connect to mongo
+	client := connectDB()
+
+	// Get 5 minute context
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+
+	for _, path := range filesToUpload {
+
+		// Open data file for reading
+		fptr, err := os.Open(fmt.Sprintf("%s/"+path, inDir))
 		if err != nil {
-		panic(err)
+			log.Panic(err)
 		}
+
 		defer fptr.Close()
+
+		switch path {
+		case "courses.json":
+			UploadData[schema.Course](client, ctx, fptr, replace)
+		case "professors.json":
+			UploadData[schema.Professor](client, ctx, fptr, replace)
+		case "sections.json":
+			UploadData[schema.Section](client, ctx, fptr, replace)
 		}
-	*/
+	}
+
+}
+
+// Generic upload function to upload parsed JSON data to the Mongo database
+// Make sure that the name of the file being parsed matches with the name of the collection you are uploading to!
+// For example, your file should be named courses.json if you want to upload courses
+// As of right now, courses, professors, and sections are available to upload.
+func UploadData[T any](client *mongo.Client, ctx context.Context, fptr *os.File, replace bool) {
+	fileName := fptr.Name()[strings.LastIndex(fptr.Name(), "/")+1 : len(fptr.Name())-5]
+	log.Println("Uploading " + fileName + ".json ...")
+
+	// Decode documents from file
+	var docs []T
+	decoder := json.NewDecoder(fptr)
+	err := decoder.Decode(&docs)
+	if err != nil {
+		log.Panic(err)
+	}
+
+	if replace {
+
+		// Get collection
+		collection := getCollection(client, fileName)
+
+		// Delete all documents from collection
+		_, err := collection.DeleteMany(ctx, bson.D{})
+		if err != nil {
+			log.Panic(err)
+		}
+
+		// Convert your documents to []interface{}
+		docsInterface := make([]interface{}, len(docs))
+		for i := range docs {
+			docsInterface[i] = docs[i]
+		}
+
+		// Add all documents decoded from the file into the collection
+		opts := options.InsertMany().SetOrdered(false)
+		_, err = collection.InsertMany(ctx, docsInterface, opts)
+		if err != nil {
+			log.Panic(err)
+		}
+
+	} else {
+		// If a temp collection already exists, drop it
+		tempCollection := getCollection(client, "temp")
+		err = tempCollection.Drop(ctx)
+		if err != nil {
+			log.Panic(err)
+		}
+
+		// Create a temporary collection
+		err := client.Database("combinedDB").CreateCollection(ctx, "temp")
+		if err != nil {
+			log.Panic(err)
+		}
+
+		// Get the temporary collection
+		tempCollection = getCollection(client, "temp")
+
+		// Convert your documents to []interface{}
+		docsInterface := make([]interface{}, len(docs))
+		for i := range docs {
+			docsInterface[i] = docs[i]
+		}
+
+		// Add all documents decoded from the file into the temporary collection
+		opts := options.InsertMany().SetOrdered(false)
+		_, err = tempCollection.InsertMany(ctx, docsInterface, opts)
+		if err != nil {
+			log.Panic(err)
+		}
+
+		// Create a merge aggregate pipeline
+		// Matched documents from the temporary collection will replace matched documents from the Mongo collection
+		// Unmatched documents from the temporary collection will be inserted into the Mongo collection
+		var matchFilters []string
+		switch fileName {
+		case "courses":
+			matchFilters = []string{"catalog_year", "course_number", "subject_prefix"}
+		case "professors":
+			matchFilters = []string{"first_name", "last_name"}
+		case "sections":
+			matchFilters = []string{"section_number", "course_reference", "academic_session"}
+		default:
+			log.Panic("Unrecognizable filename: " + fileName)
+		}
+
+		// The documents will be added/merged into the collection with the same name as the file
+		// The filters for the merge aggregate pipeline are based on the file name
+		mergeStage := bson.D{primitive.E{Key: "$merge", Value: bson.D{primitive.E{Key: "into", Value: fileName}, primitive.E{Key: "on", Value: matchFilters}, primitive.E{Key: "whenMatched", Value: "replace"}, primitive.E{Key: "whenNotMatched", Value: "insert"}}}}
+
+		// Execute aggregate pipeline
+		_, err = tempCollection.Aggregate(ctx, mongo.Pipeline{mergeStage})
+		if err != nil {
+			log.Panic(err)
+		}
+
+		// Drop the temporary collection
+		err = tempCollection.Drop(ctx)
+		if err != nil {
+			log.Panic(err)
+		}
+	}
+
+	log.Println("Done uploading " + fileName + ".json!")
+
+	defer fptr.Close()
 }