Skip to content

Commit 5ac7669

Browse files
heartggheartgg
authored
Merge pull request #30 from demonlexe/feature/skedge-14-fix-rmp-fetching-inconsistency-issue
SKEDGE-14 #comment convert to async/await instead of .then() logics, organizing awaits
2 parents 1226dd0 + 517598e commit 5ac7669

File tree

3 files changed

+126
-43
lines changed

3 files changed

+126
-43
lines changed

‎src/data/config.ts

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import type { RMPRatingInterface } from "~data/interfaces";
12
export const HEADERS = {
23
"Authorization": "Basic dGVzdDp0ZXN0",
34
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
@@ -18,6 +19,7 @@ export const NEBULA_FETCH_OPTIONS = {
1819
}
1920

2021
export const SCHOOL_ID = "1273"
22+
export const RMP_GRAPHQL_URL = "https://www.ratemyprofessors.com/graphql";
2123

2224
function unRegister(key: string) {
2325
let newVar = "";
@@ -27,5 +29,4 @@ function unRegister(key: string) {
2729
newVar = newVar.concat(String.fromCharCode(a));
2830
}
2931
return newVar;
30-
}
31-
32+
}

‎src/data/fetchFromRmp.ts

+89-40
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
1-
import { HEADERS, PROFESSOR_QUERY } from "~data/config";
1+
import { HEADERS, PROFESSOR_QUERY, RMP_GRAPHQL_URL } from "~data/config";
2+
import type { RMPRatingInterface } from "~data/interfaces";
3+
4+
function reportError(context, err) {
5+
console.error("Error in " + context + ": " + err);
6+
}
27

38
function getProfessorUrl(professorName: string, schoolId: string): string {
49
return `https://www.ratemyprofessors.com/search/professors/${schoolId}?q=${encodeURIComponent(professorName)}}`
@@ -15,16 +20,24 @@ function getProfessorIds(texts: string[], professorNames: string[]): string[] {
1520
const professorIds = []
1621
const lowerCaseProfessorNames = professorNames.map(name => name.toLowerCase())
1722
texts.forEach(text => {
18-
let matched = false;
19-
const regex = /"legacyId":(\d+).*?"firstName":"(.*?)","lastName":"(.*?)"/g;
20-
for (const match of text.matchAll(regex)) {
21-
console.log(match[2].split(' ')[0].toLowerCase() + " " + match[3].toLowerCase())
22-
if (lowerCaseProfessorNames.includes(match[2].split(' ')[0].toLowerCase() + " " + match[3].toLowerCase())) {
23-
professorIds.push(match[1]);
24-
matched = true;
23+
let pendingMatch = null;
24+
const regex = /"legacyId":(\d+).*?"numRatings":(\d+).*?"firstName":"(.*?)","lastName":"(.*?)"/g;
25+
let allMatches: string[] = text.match(regex);
26+
let highestNumRatings = 0;
27+
28+
if (allMatches) {
29+
for (const fullMatch of allMatches) {
30+
for (const match of fullMatch.matchAll(regex)) {
31+
console.log(match[3].split(' ')[0].toLowerCase() + " " + match[4].toLowerCase() + " ")
32+
let numRatings = parseInt(match[2]);
33+
if (lowerCaseProfessorNames.includes(match[3].split(' ')[0].toLowerCase() + " " + match[4].toLowerCase()) && numRatings >= highestNumRatings) {
34+
pendingMatch = match[1];
35+
}
36+
}
2537
}
2638
}
27-
if (!matched) professorIds.push(null)
39+
40+
professorIds.push(pendingMatch);
2841
})
2942
return professorIds
3043
}
@@ -46,48 +59,84 @@ function getGraphQlUrlProps(professorIds: string[]) {
4659
return graphQlUrlProps
4760
}
4861

49-
function fetchWithGraphQl(graphQlUrlProps: any[], resolve) {
50-
const graphqlUrl = "https://www.ratemyprofessors.com/graphql";
62+
function wait(delay){
63+
return new Promise((resolve) => setTimeout(resolve, delay));
64+
}
5165

52-
Promise.all(graphQlUrlProps.map(u=>fetch(graphqlUrl, u)))
53-
.then(responses => Promise.all(responses.map(res => res.json())))
54-
.then(ratings => {
55-
for (let i = 0; i < ratings.length; i++) {
56-
if (ratings[i] != null && ratings[i].hasOwnProperty("data") && ratings[i]["data"].hasOwnProperty("node")) {
57-
ratings[i] = ratings[i]["data"]["node"];
58-
}
66+
function fetchRetry(url: string, delay: number, tries: number, fetchOptions) {
67+
function onError(err){
68+
let triesLeft: number = tries - 1;
69+
if(!triesLeft){
70+
throw err;
71+
}
72+
return wait(delay).then(() => fetchRetry(url, delay, triesLeft, fetchOptions));
73+
}
74+
return fetch(url,fetchOptions).catch(onError);
75+
}
76+
77+
// If using orderedFetchOpts, make sure that it is an array and that the index of the fetch options corresponds to the index of the response in the responses array.
78+
async function validateResponses(responses: any[], orderedFetchOpts: any[]) {
79+
for (const [key, value] of Object.entries(responses)) {
80+
let notOk = value?.status !== 200;
81+
if (notOk && value && value.url) {
82+
let details = {
83+
status: value.status,
84+
statusText: value.statusText,
85+
redirected: value.redirected,
86+
url: value.url
5987
}
60-
console.log(ratings)
61-
resolve(ratings)
62-
})
88+
reportError("validateResponses", "Status not OK for fetch request. Details are: "+JSON.stringify(details));
89+
let fetchOptions = orderedFetchOpts[key] || {}; // If we don't have fetch options, we just use an empty object.
90+
responses[key] = await fetchRetry(value?.url, 200, 3, fetchOptions);
91+
}
92+
};
93+
return responses;
94+
}
95+
96+
export async function fetchWithGraphQl(graphQlUrlProps: any[]) {
97+
try {
98+
let responses = await validateResponses(await Promise.all(graphQlUrlProps.map(u=>fetch(RMP_GRAPHQL_URL, u))),graphQlUrlProps);
99+
// We now have all the responses. So, we consider all the responses, and collect the ratings.
100+
let ratings: RMPRatingInterface [] = await Promise.all(responses.map(res => res.json()));
101+
for (let i = 0; i < ratings.length; i++) {
102+
if (ratings[i] != null && ratings[i].hasOwnProperty("data") && ratings[i]["data"].hasOwnProperty("node")) {
103+
ratings[i] = ratings[i]["data"]["node"];
104+
}
105+
}
106+
return ratings;
107+
}
108+
catch (err) {
109+
reportError("fetchWithGraphQl",err);
110+
return [];
111+
}
63112
}
64113

65114
export interface RmpRequest {
66115
professorNames: string[],
67116
schoolId: string
68117
}
69-
export function requestProfessorsFromRmp(request: RmpRequest): Promise<RMPInterface[]> {
70-
return new Promise((resolve, reject) => {
71-
72-
// make a list of urls for promises
73-
const professorUrls = getProfessorUrls(request.professorNames, request.schoolId)
118+
export async function requestProfessorsFromRmp(request: RmpRequest): Promise<RMPInterface[]> {
74119

75-
// fetch professor ids from each url
76-
Promise.all(professorUrls.map(u=>fetch(u)))
77-
.then(responses => Promise.all(responses.map(res => res.text())))
78-
.then(texts => {
79-
const professorIds = getProfessorIds(texts, request.professorNames)
120+
// make a list of urls for promises
121+
const professorUrls = getProfessorUrls(request.professorNames, request.schoolId)
122+
123+
// fetch professor ids from each url
124+
try {
125+
let responses = await validateResponses(await Promise.all(professorUrls.map(u=>(fetch(u)))), []);
126+
127+
let texts = await Promise.all(responses.map(res => res.text()));
128+
const professorIds = getProfessorIds(texts, request.professorNames)
80129

81-
// create fetch objects for each professor id
82-
const graphQlUrlProps = getGraphQlUrlProps(professorIds)
130+
// create fetch objects for each professor id
131+
const graphQlUrlProps = getGraphQlUrlProps(professorIds)
83132

84-
// fetch professor info by id with graphQL
85-
fetchWithGraphQl(graphQlUrlProps, resolve)
86-
}
87-
).catch(error => {
88-
reject(error);
89-
});
90-
})
133+
// fetch professor info by id with graphQL
134+
let professors = await fetchWithGraphQl(graphQlUrlProps);
135+
return professors;
136+
} catch (error) {
137+
reportError("requestProfessorsFromRmp", error);
138+
return [];
139+
};
91140
}
92141

93142
interface RMPInterface {

‎src/data/interfaces.ts

+34-1
Original file line numberDiff line numberDiff line change
@@ -85,4 +85,37 @@ export interface SectionInterface {
8585
section_number: string;
8686
syllabus_uri: string;
8787
teaching_assistants: any[];
88-
};
88+
};
89+
90+
export interface CourseCodeInterface {
91+
courseCount: number,
92+
courseName: string
93+
};
94+
export interface RatingsDistributionInterface {
95+
r1: number,
96+
r2: number,
97+
r3: number,
98+
r4: number,
99+
r5: number,
100+
total: number
101+
};
102+
export interface TeacherRatingTag {
103+
tagCount: number,
104+
tagName: string
105+
}
106+
107+
export interface RMPRatingInterface {
108+
avgDifficulty: number,
109+
avgRating: number,
110+
courseCodes: CourseCodeInterface [],
111+
department: string,
112+
firstName: string,
113+
lastName: string,
114+
legacyId: number,
115+
numRatings: number,
116+
ratingsDistribution: RatingsDistributionInterface,
117+
school: {id: string},
118+
teacherRatingTags: TeacherRatingTag [],
119+
wouldTakeAgainPercent: number
120+
121+
}

0 commit comments

Comments
 (0)