From 75eaabbf25950773ff53c00cf89f6604baab35ed Mon Sep 17 00:00:00 2001 From: jpahm <20374744+jpahm@users.noreply.github.com> Date: Wed, 24 Apr 2024 18:31:00 -0500 Subject: [PATCH] Adjust uploader code for IdWrapper removal --- go.mod | 49 +++- go.sum | 132 ++++++++-- parser/courseParser.go | 2 +- parser/parser.go | 9 +- parser/professorParser.go | 8 +- parser/sectionParser.go | 2 +- scrapers/events.go | 2 +- scrapers/organizations.go | 2 +- scrapers/profiles.go | 4 +- uploader/uploader.go | 525 ++++++++++++++++++++------------------ 10 files changed, 426 insertions(+), 309 deletions(-) diff --git a/go.mod b/go.mod index 5391f81..8137c5c 100644 --- a/go.mod +++ b/go.mod @@ -4,32 +4,55 @@ go 1.19 require ( github.com/PuerkitoBio/goquery v1.8.1 - github.com/UTDNebula/nebula-api/api v0.0.0-20231204040332-adccdc08b203 - github.com/chromedp/cdproto v0.0.0-20240116100315-4a0ec5e4c400 - github.com/chromedp/chromedp v0.9.3 + github.com/UTDNebula/nebula-api/api v0.0.0-20240423212728-2ef02f280c6c + github.com/chromedp/cdproto v0.0.0-20240421230201-ab917191657d + github.com/chromedp/chromedp v0.9.5 github.com/joho/godotenv v1.5.1 go.mongodb.org/mongo-driver v1.15.0 ) require ( github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/bytedance/sonic v1.11.5 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/chromedp/sysutil v1.0.0 // indirect + github.com/cloudwego/base64x v0.1.3 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gin-gonic/gin v1.9.1 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.19.0 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect - github.com/gobwas/ws v1.3.0 // indirect - github.com/golang/snappy v0.0.1 // indirect - github.com/google/go-cmp v0.5.5 // indirect + github.com/gobwas/ws v1.3.2 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/gorilla/schema v1.3.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.13.6 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.1 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.15.0 // indirect + github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect + golang.org/x/arch v0.7.0 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 28f64a4..bbecc99 100644 --- a/go.sum +++ b/go.sum @@ -1,69 +1,138 @@ github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= -github.com/UTDNebula/nebula-api/api v0.0.0-20231204040332-adccdc08b203 h1:Q2LpqHQs7Ghf+qfFjxqwDQRQSzUXiTI6QC37j8Jb2QI= -github.com/UTDNebula/nebula-api/api v0.0.0-20231204040332-adccdc08b203/go.mod h1:/qhwfZIy/fvEDMklOqynJ34K2H2Ml40bcu9A/EB2AfU= +github.com/UTDNebula/nebula-api/api v0.0.0-20240423212728-2ef02f280c6c h1:v2jc53nEy7aOpqqgf0NlVc7yL5mK3b1c5MAU/NPaS0k= +github.com/UTDNebula/nebula-api/api v0.0.0-20240423212728-2ef02f280c6c/go.mod h1:JysMkqwHCAS7iRuaS5YgNDtau+URfPY6XQ0xUvCG7+g= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= -github.com/chromedp/cdproto v0.0.0-20231011050154-1d073bb38998/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= -github.com/chromedp/cdproto v0.0.0-20240116100315-4a0ec5e4c400 h1:mHR3reslmE6J351eW8TgB/BPT+B9OzMxLe7dPa5WYSQ= -github.com/chromedp/cdproto v0.0.0-20240116100315-4a0ec5e4c400/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= -github.com/chromedp/chromedp v0.9.3 h1:Wq58e0dZOdHsxaj9Owmfcf+ibtpYN1N0FWVbaxa/esg= -github.com/chromedp/chromedp v0.9.3/go.mod h1:NipeUkUcuzIdFbBP8eNNvl9upcceOfWzoJn6cRe4ksA= +github.com/bytedance/sonic v1.11.5 h1:G00FYjjqll5iQ1PYXynbg/hyzqBqavH8Mo9/oTopd9k= +github.com/bytedance/sonic v1.11.5/go.mod h1:X2PC2giUdj/Cv2lliWFLk6c/DUQok5rViJSemeB0wDw= +github.com/bytedance/sonic/loader v0.1.0/go.mod h1:UmRT+IRTGKz/DAkzcEGzyVqQFJ7H9BqwBO3pm9H/+HY= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/cdproto v0.0.0-20240421230201-ab917191657d h1:x9d0XwRV3aWw1gAZtv0LrI39U+Efjp0mtyXRyikGb9Y= +github.com/chromedp/cdproto v0.0.0-20240421230201-ab917191657d/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/chromedp v0.9.5 h1:viASzruPJOiThk7c5bueOUY91jGLJVximoEMGoH93rg= +github.com/chromedp/chromedp v0.9.5/go.mod h1:D4I2qONslauw/C7INoCir1BJkSwBYMyZgx8X276z3+Y= github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= +github.com/cloudwego/base64x v0.1.3 h1:b5J/l8xolB7dyDTTmhJP2oTs5LdrjyrUFuNxdfq5hAg= +github.com/cloudwego/base64x v0.1.3/go.mod h1:1+1K5BUHIQzyapgpF7LwvOGAEDicKtt1umPV+aN8pi8= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= +github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.3.0 h1:sbeU3Y4Qzlb+MOzIe6mQGf7QR4Hkv6ZD0qhGkBFL2O0= -github.com/gobwas/ws v1.3.0/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gobwas/ws v1.3.2 h1:zlnbNHxumkRvfPWgfXu8RBwyNR1x8wh9cf5PTOCqs9Q= +github.com/gobwas/ws v1.3.2/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/gorilla/schema v1.3.0 h1:rbciOzXAx3IB8stEFnfTwO3sYa6EWlQk79XdyustPDA= +github.com/gorilla/schema v1.3.0/go.mod h1:Dg5SSm5PV60mhF2NFaTV1xuYYj8tV8NOPRo4FggUMnM= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/pelletier/go-toml/v2 v2.2.1 h1:9TA9+T8+8CUCO2+WYnDLCgrYi9+omqKXyjDtosvtEhg= +github.com/pelletier/go-toml/v2 v2.2.1/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= +github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= +golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -71,8 +140,9 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -89,4 +159,12 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/parser/courseParser.go b/parser/courseParser.go index db3427d..484e232 100644 --- a/parser/courseParser.go +++ b/parser/courseParser.go @@ -48,7 +48,7 @@ func parseCourse(courseNum string, session schema.AcademicSession, rowInfo map[s course = &schema.Course{} - course.Id = schema.IdWrapper(primitive.NewObjectID().Hex()) + course.Id = primitive.NewObjectID() course.Course_number = idMatches[2] course.Subject_prefix = idMatches[1] course.Title = rowInfo["Course Title:"] diff --git a/parser/parser.go b/parser/parser.go index 58718dd..1c371ae 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -8,22 +8,23 @@ import ( "time" "github.com/UTDNebula/api-tools/utils" + "go.mongodb.org/mongo-driver/bson/primitive" "github.com/PuerkitoBio/goquery" "github.com/UTDNebula/nebula-api/api/schema" ) // Main dictionaries for mapping unique keys to the actual data -var Sections = make(map[schema.IdWrapper]*schema.Section) +var Sections = make(map[primitive.ObjectID]*schema.Section) var Courses = make(map[string]*schema.Course) var Professors = make(map[string]*schema.Professor) // Auxilliary dictionaries for mapping the generated ObjectIDs to the keys used in the above maps, used for validation purposes -var CourseIDMap = make(map[schema.IdWrapper]string) -var ProfessorIDMap = make(map[schema.IdWrapper]string) +var CourseIDMap = make(map[primitive.ObjectID]string) +var ProfessorIDMap = make(map[primitive.ObjectID]string) // Requisite parser closures associated with courses -var ReqParsers = make(map[schema.IdWrapper]func()) +var ReqParsers = make(map[primitive.ObjectID]func()) // Grade mappings for section grade distributions, mapping is MAP[SEMESTER] -> MAP[SUBJECT + NUMBER + SECTION] -> GRADE DISTRIBUTION var GradeMap map[string]map[string][]int diff --git a/parser/professorParser.go b/parser/professorParser.go index d310992..2184074 100644 --- a/parser/professorParser.go +++ b/parser/professorParser.go @@ -8,10 +8,10 @@ import ( "go.mongodb.org/mongo-driver/bson/primitive" ) -func parseProfessors(sectionId schema.IdWrapper, rowInfo map[string]string, classInfo map[string]string) []schema.IdWrapper { +func parseProfessors(sectionId primitive.ObjectID, rowInfo map[string]string, classInfo map[string]string) []primitive.ObjectID { professorText := rowInfo["Instructor(s):"] professorMatches := personRegexp.FindAllStringSubmatch(professorText, -1) - var profRefs []schema.IdWrapper = make([]schema.IdWrapper, 0, len(professorMatches)) + var profRefs []primitive.ObjectID = make([]primitive.ObjectID, 0, len(professorMatches)) for _, match := range professorMatches { nameStr := utils.TrimWhitespace(match[1]) @@ -35,12 +35,12 @@ func parseProfessors(sectionId schema.IdWrapper, rowInfo map[string]string, clas } prof = &schema.Professor{} - prof.Id = schema.IdWrapper(primitive.NewObjectID().Hex()) + prof.Id = primitive.NewObjectID() prof.First_name = firstName prof.Last_name = lastName prof.Titles = []string{utils.TrimWhitespace(match[2])} prof.Email = utils.TrimWhitespace(match[3]) - prof.Sections = []schema.IdWrapper{sectionId} + prof.Sections = []primitive.ObjectID{sectionId} profRefs = append(profRefs, prof.Id) Professors[profKey] = prof ProfessorIDMap[prof.Id] = profKey diff --git a/parser/sectionParser.go b/parser/sectionParser.go index dd694a6..316cd13 100644 --- a/parser/sectionParser.go +++ b/parser/sectionParser.go @@ -22,7 +22,7 @@ func parseSection(courseRef *schema.Course, classNum string, syllabusURI string, section := &schema.Section{} - section.Id = schema.IdWrapper(primitive.NewObjectID().Hex()) + section.Id = primitive.NewObjectID() section.Section_number = idMatches[1] section.Course_reference = courseRef.Id diff --git a/scrapers/events.go b/scrapers/events.go index 28ad666..5af3af8 100644 --- a/scrapers/events.go +++ b/scrapers/events.go @@ -307,7 +307,7 @@ func ScrapeEvents(outDir string) { utils.VPrintf("Scraped contact phone info: %s", contactInformationPhone) events = append(events, schema.Event{ - Id: schema.IdWrapper(primitive.NewObjectID().Hex()), + Id: primitive.NewObjectID(), Summary: summary, Location: location, StartTime: dateTimeStart, diff --git a/scrapers/organizations.go b/scrapers/organizations.go index 7c02cb2..b741aa0 100644 --- a/scrapers/organizations.go +++ b/scrapers/organizations.go @@ -227,7 +227,7 @@ func parseCsvRecord(ctx context.Context, entry []string) (*schema.Organization, utils.VPrintf("Error retrieving image for %s: %v", entry[0], err) } return &schema.Organization{ - Id: schema.IdWrapper(primitive.NewObjectID().Hex()), + Id: primitive.NewObjectID(), Title: entry[0], Categories: parseCategories(entry[1]), Description: entry[2], diff --git a/scrapers/profiles.go b/scrapers/profiles.go index 8b6b5d5..867ead5 100644 --- a/scrapers/profiles.go +++ b/scrapers/profiles.go @@ -275,7 +275,7 @@ func ScrapeProfiles(outDir string) { utils.VPrintf("Parsed list! #: %s, Office: %v", phoneNumber, office) professors = append(professors, schema.Professor{ - Id: schema.IdWrapper(primitive.NewObjectID().Hex()), + Id: primitive.NewObjectID(), First_name: firstName, Last_name: lastName, Titles: titles, @@ -285,7 +285,7 @@ func ScrapeProfiles(outDir string) { Profile_uri: link, Image_uri: imageUri, Office_hours: []schema.Meeting{}, - Sections: []schema.IdWrapper{}, + Sections: []primitive.ObjectID{}, }) utils.VPrintf("Scraped profile for %s %s!", firstName, lastName) diff --git a/uploader/uploader.go b/uploader/uploader.go index 32304bb..46a6326 100644 --- a/uploader/uploader.go +++ b/uploader/uploader.go @@ -41,276 +41,291 @@ func Upload(inDir string, replace bool) { //Connect to mongo client := connectDB() - // Get context - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + // Get 5 minute context + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() for _, path := range filesToUpload { + // Open data file for reading fptr, err := os.Open(fmt.Sprintf("%s/"+path, inDir)) if err != nil { log.Panic(err) } + defer fptr.Close() + switch path { case "courses.json": - log.Println("Uploading courses.json ...") - - // Decode courses from courses.json - var courses []schema.Course - decoder := json.NewDecoder(fptr) - err = decoder.Decode(&courses) - if err != nil { - log.Panic(err) - } - - if replace { - - // Get collection - collection := getCollection(client, "courses") - - // Delete all documents from collection - _, err := collection.DeleteMany(ctx, bson.D{}) - if err != nil { - log.Panic(err) - } - - // Convert your courses to []interface{} - courseDocs := make([]interface{}, len(courses)) - for i := range courses { - courseDocs[i] = courses[i] - } - - // Add all documents decoded from courses.json into the temporary collection - opts := options.InsertMany().SetOrdered(false) - _, err = collection.InsertMany(ctx, courseDocs, opts) - if err != nil { - log.Panic(err) - } - - } else { - - // If a temp collection already exists, drop it - tempCollection := getCollection(client, "temp") - err = tempCollection.Drop(ctx) - if err != nil { - log.Panic(err) - } - - // Create a temporary collection - err := client.Database("combinedDB").CreateCollection(ctx, "temp") - if err != nil { - log.Panic(err) - } - - // Get the temporary collection - tempCollection = getCollection(client, "temp") - - // Convert your courses to []interface{} - courseDocs := make([]interface{}, len(courses)) - for i := range courses { - courseDocs[i] = courses[i] - } - - // Add all documents decoded from courses.json into the temporary collection - opts := options.InsertMany().SetOrdered(false) - _, err = tempCollection.InsertMany(ctx, courseDocs, opts) - if err != nil { - log.Panic(err) - } - - // Create a merge aggregate pipeline - // Matched documents from the temporary collection will replace matched documents from the Mongo collection - // Unmatched documents from the temporary collection will be inserted into the Mongo collection - mergeStage := bson.D{primitive.E{Key: "$merge", Value: bson.D{primitive.E{Key: "into", Value: "courses"}, primitive.E{Key: "on", Value: [3]string{"catalog_year", "course_number", "subject_prefix"}}, primitive.E{Key: "whenMatched", Value: "replace"}, primitive.E{Key: "whenNotMatched", Value: "insert"}}}} - - // Execute aggregate pipeline - _, err = tempCollection.Aggregate(ctx, mongo.Pipeline{mergeStage}) - if err != nil { - log.Panic(err) - } - - // Drop the temporary collection - err = tempCollection.Drop(ctx) - if err != nil { - log.Panic(err) - } - } - - log.Println("Done uploading courses.json!") - + uploadCourses(client, ctx, fptr, replace) case "professors.json": - log.Println("Uploading professors.json ...") - - // Decode courses from professors.json - var professors []schema.Professor - decoder := json.NewDecoder(fptr) - err = decoder.Decode(&professors) - if err != nil { - log.Panic(err) - } - - if replace { - - // Get collection - collection := getCollection(client, "professors") - - // Delete all documents from collection - _, err := collection.DeleteMany(ctx, bson.D{}) - if err != nil { - log.Panic(err) - } - - // Convert your professors to []interface{} - professorsDocs := make([]interface{}, len(professors)) - for i := range professors { - professorsDocs[i] = professors[i] - } - - // Add all documents decoded from professors.json into the temporary collection - opts := options.InsertMany().SetOrdered(false) - _, err = collection.InsertMany(ctx, professorsDocs, opts) - if err != nil { - log.Panic(err) - } - - } else { - - // If a temp collection already exists, drop it - tempCollection := getCollection(client, "temp") - err = tempCollection.Drop(ctx) - if err != nil { - log.Panic(err) - } - - // Create a temporary collection - err := client.Database("combinedDB").CreateCollection(ctx, "temp") - if err != nil { - log.Panic(err) - } - - // Get the temporary collection - tempCollection = getCollection(client, "temp") - - // Convert your professors to []interface{} - professorsDocs := make([]interface{}, len(professors)) - for i := range professors { - professorsDocs[i] = professors[i] - } - - // Add all documents decoded from professors.json into the temporary collection - opts := options.InsertMany().SetOrdered(false) - _, err = tempCollection.InsertMany(ctx, professorsDocs, opts) - if err != nil { - log.Panic(err) - } - - // Create a merge aggregate pipeline - // Matched documents from the temporary collection will replace matched documents from the Mongo collection - // Unmatched documents from the temporary collection will be inserted into the Mongo collection - mergeStage := bson.D{primitive.E{Key: "$merge", Value: bson.D{primitive.E{Key: "into", Value: "professors"}, primitive.E{Key: "on", Value: [2]string{"first_name", "last_name"}}, primitive.E{Key: "whenMatched", Value: "replace"}, primitive.E{Key: "whenNotMatched", Value: "insert"}}}} - - // Execute aggregate pipeline - _, err = tempCollection.Aggregate(ctx, mongo.Pipeline{mergeStage}) - if err != nil { - log.Panic(err) - } - - // Drop the temporary collection - err = tempCollection.Drop(ctx) - if err != nil { - log.Panic(err) - } - } - - log.Println("Done uploading professors.json!") - + uploadProfessors(client, ctx, fptr, replace) case "sections.json": - log.Println("Uploading sections.json ...") - - // Decode courses from sections.json - var sections []schema.Section - decoder := json.NewDecoder(fptr) - err = decoder.Decode(§ions) - if err != nil { - log.Panic(err) - } - - if replace { - - // Get collection - collection := getCollection(client, "sections") - - // Delete all documents from collection - _, err := collection.DeleteMany(ctx, bson.D{}) - if err != nil { - log.Panic(err) - } - - // Convert your sections to []interface{} - sectionsDocs := make([]interface{}, len(sections)) - for i := range sections { - sectionsDocs[i] = sections[i] - } - - // Add all documents decoded from sections.json into the temporary collection - opts := options.InsertMany().SetOrdered(false) - _, err = collection.InsertMany(ctx, sectionsDocs, opts) - if err != nil { - log.Panic(err) - } - } else { - // If a temp collection already exists, drop it - tempCollection := getCollection(client, "temp") - err = tempCollection.Drop(ctx) - if err != nil { - log.Panic(err) - } - - // Create a temporary collection - err := client.Database("combinedDB").CreateCollection(ctx, "temp") - if err != nil { - log.Panic(err) - } - - // Get the temporary collection - tempCollection = getCollection(client, "temp") - - // Convert your sections to []interface{} - sectionsDocs := make([]interface{}, len(sections)) - for i := range sections { - sectionsDocs[i] = sections[i] - } - - // Add all documents decoded from professors.json into the temporary collection - opts := options.InsertMany().SetOrdered(false) - _, err = tempCollection.InsertMany(ctx, sectionsDocs, opts) - if err != nil { - log.Panic(err) - } - - // Create a merge aggregate pipeline - // Matched documents from the temporary collection will replace matched documents from the Mongo collection - // Unmatched documents from the temporary collection will be inserted into the Mongo collection - mergeStage := bson.D{primitive.E{Key: "$merge", Value: bson.D{primitive.E{Key: "into", Value: "sections"}, primitive.E{Key: "on", Value: [3]string{"section_number", "course_reference", "academic_session"}}, primitive.E{Key: "whenMatched", Value: "replace"}, primitive.E{Key: "whenNotMatched", Value: "insert"}}}} - - // Execute aggregate pipeline - _, err = tempCollection.Aggregate(ctx, mongo.Pipeline{mergeStage}) - if err != nil { - log.Panic(err) - } - - // Drop the temporary collection - err = tempCollection.Drop(ctx) - if err != nil { - log.Panic(err) - } - } - - log.Println("Done uploading sections.json!") + uploadSections(client, ctx, fptr, replace) } + } - defer fptr.Close() +} + +func uploadCourses(client *mongo.Client, ctx context.Context, fptr *os.File, replace bool) { + log.Println("Uploading courses.json ...") + + // Decode courses from courses.json + var courses []schema.Course + decoder := json.NewDecoder(fptr) + err := decoder.Decode(&courses) + if err != nil { + log.Panic(err) + } + + if replace { + + // Get collection + collection := getCollection(client, "courses") + + // Delete all documents from collection + _, err := collection.DeleteMany(ctx, bson.D{}) + if err != nil { + log.Panic(err) + } + + // Convert your courses to []interface{} + courseDocs := make([]interface{}, len(courses)) + for i := range courses { + courseDocs[i] = courses[i] + } + + // Add all documents decoded from courses.json into the collection + opts := options.InsertMany().SetOrdered(false) + _, err = collection.InsertMany(ctx, courseDocs, opts) + if err != nil { + log.Panic(err) + } + + } else { + + // If a temp collection already exists, drop it + tempCollection := getCollection(client, "temp") + err = tempCollection.Drop(ctx) + if err != nil { + log.Panic(err) + } + + // Create a temporary collection + err := client.Database("combinedDB").CreateCollection(ctx, "temp") + if err != nil { + log.Panic(err) + } + + // Get the temporary collection + tempCollection = getCollection(client, "temp") + + // Convert your courses to []interface{} + courseDocs := make([]interface{}, len(courses)) + for i := range courses { + courseDocs[i] = courses[i] + } + + // Add all documents decoded from courses.json into the temporary collection + opts := options.InsertMany().SetOrdered(false) + _, err = tempCollection.InsertMany(ctx, courseDocs, opts) + + if err != nil { + log.Panic(err) + } + + // Create a merge aggregate pipeline + // Matched documents from the temporary collection will replace matched documents from the Mongo collection + // Unmatched documents from the temporary collection will be inserted into the Mongo collection + mergeStage := bson.D{primitive.E{Key: "$merge", Value: bson.D{primitive.E{Key: "into", Value: "courses"}, primitive.E{Key: "on", Value: [3]string{"catalog_year", "course_number", "subject_prefix"}}, primitive.E{Key: "whenMatched", Value: "replace"}, primitive.E{Key: "whenNotMatched", Value: "insert"}}}} + + // Execute aggregate pipeline + _, err = tempCollection.Aggregate(ctx, mongo.Pipeline{mergeStage}) + if err != nil { + log.Panic(err) + } + + // Drop the temporary collection + err = tempCollection.Drop(ctx) + if err != nil { + log.Panic(err) + } + } + + log.Println("Done uploading courses.json!") +} + +func uploadProfessors(client *mongo.Client, ctx context.Context, fptr *os.File, replace bool) { + log.Println("Uploading professors.json ...") + + // Decode courses from professors.json + var professors []schema.Professor + decoder := json.NewDecoder(fptr) + err := decoder.Decode(&professors) + if err != nil { + log.Panic(err) } + if replace { + + // Get collection + collection := getCollection(client, "professors") + + // Delete all documents from collection + _, err := collection.DeleteMany(ctx, bson.D{}) + if err != nil { + log.Panic(err) + } + + // Convert your professors to []interface{} + professorsDocs := make([]interface{}, len(professors)) + for i := range professors { + professorsDocs[i] = professors[i] + } + + // Add all documents decoded from professors.json into the collection + opts := options.InsertMany().SetOrdered(false) + _, err = collection.InsertMany(ctx, professorsDocs, opts) + if err != nil { + log.Panic(err) + } + + } else { + + // If a temp collection already exists, drop it + tempCollection := getCollection(client, "temp") + err = tempCollection.Drop(ctx) + if err != nil { + log.Panic(err) + } + + // Create a temporary collection + err := client.Database("combinedDB").CreateCollection(ctx, "temp") + if err != nil { + log.Panic(err) + } + + // Get the temporary collection + tempCollection = getCollection(client, "temp") + + // Convert your professors to []interface{} + professorsDocs := make([]interface{}, len(professors)) + for i := range professors { + professorsDocs[i] = professors[i] + } + + // Add all documents decoded from professors.json into the temporary collection + opts := options.InsertMany().SetOrdered(false) + _, err = tempCollection.InsertMany(ctx, professorsDocs, opts) + if err != nil { + log.Panic(err) + } + + // Create a merge aggregate pipeline + // Matched documents from the temporary collection will replace matched documents from the Mongo collection + // Unmatched documents from the temporary collection will be inserted into the Mongo collection + mergeStage := bson.D{primitive.E{Key: "$merge", Value: bson.D{primitive.E{Key: "into", Value: "professors"}, primitive.E{Key: "on", Value: [2]string{"first_name", "last_name"}}, primitive.E{Key: "whenMatched", Value: "replace"}, primitive.E{Key: "whenNotMatched", Value: "insert"}}}} + + // Execute aggregate pipeline + _, err = tempCollection.Aggregate(ctx, mongo.Pipeline{mergeStage}) + if err != nil { + log.Panic(err) + } + + // Drop the temporary collection + err = tempCollection.Drop(ctx) + if err != nil { + log.Panic(err) + } + } + + log.Println("Done uploading professors.json!") +} + +func uploadSections(client *mongo.Client, ctx context.Context, fptr *os.File, replace bool) { + log.Println("Uploading sections.json ...") + + // Decode courses from sections.json + var sections []schema.Section + decoder := json.NewDecoder(fptr) + err := decoder.Decode(§ions) + if err != nil { + log.Panic(err) + } + + if replace { + + // Get collection + collection := getCollection(client, "sections") + + // Delete all documents from collection + _, err := collection.DeleteMany(ctx, bson.D{}) + if err != nil { + log.Panic(err) + } + + // Convert your sections to []interface{} + sectionsDocs := make([]interface{}, len(sections)) + for i := range sections { + sectionsDocs[i] = sections[i] + } + + // Add all documents decoded from sections.json into the collection + opts := options.InsertMany().SetOrdered(false) + _, err = collection.InsertMany(ctx, sectionsDocs, opts) + if err != nil { + log.Panic(err) + } + + } else { + // If a temp collection already exists, drop it + tempCollection := getCollection(client, "temp") + err = tempCollection.Drop(ctx) + if err != nil { + log.Panic(err) + } + + // Create a temporary collection + err := client.Database("combinedDB").CreateCollection(ctx, "temp") + if err != nil { + log.Panic(err) + } + + // Get the temporary collection + tempCollection = getCollection(client, "temp") + + // Convert your sections to []interface{} + sectionsDocs := make([]interface{}, len(sections)) + for i := range sections { + sectionsDocs[i] = sections[i] + } + + // Add all documents decoded from professors.json into the temporary collection + opts := options.InsertMany().SetOrdered(false) + _, err = tempCollection.InsertMany(ctx, sectionsDocs, opts) + if err != nil { + log.Panic(err) + } + + // Create a merge aggregate pipeline + // Matched documents from the temporary collection will replace matched documents from the Mongo collection + // Unmatched documents from the temporary collection will be inserted into the Mongo collection + mergeStage := bson.D{primitive.E{Key: "$merge", Value: bson.D{primitive.E{Key: "into", Value: "sections"}, primitive.E{Key: "on", Value: [3]string{"section_number", "course_reference", "academic_session"}}, primitive.E{Key: "whenMatched", Value: "replace"}, primitive.E{Key: "whenNotMatched", Value: "insert"}}}} + + // Execute aggregate pipeline + _, err = tempCollection.Aggregate(ctx, mongo.Pipeline{mergeStage}) + if err != nil { + log.Panic(err) + } + + // Drop the temporary collection + err = tempCollection.Drop(ctx) + if err != nil { + log.Panic(err) + } + } + + log.Println("Done uploading sections.json!") + + defer fptr.Close() }