rankings: fetch profiles faster and improvements

rankings/.gitignore

@@ -1,2 +1,3 @@
 .env
-output/
+output/
+cache/

rankings/export.go

@@ -2,20 +2,30 @@ package main
 
 import (
 	"encoding/json"
+	"log"
 	"os"
 )
 
-func exportAll(spRankings, mpRankings, overallRankings *[]*Player) {
-	sp, _ := os.Create("./output/sp.json")
-	spRankingsOut, _ := json.Marshal(*spRankings)
+func exportAll(spRankings, mpRankings, overallRankings []*Player) {
+	err := os.Mkdir("./output", 0775)
+	if err != nil && !os.IsExist(err) {
+		log.Fatalln(err.Error())
+	}
+
+	sp, err := os.Create("./output/sp.json")
+	if err != nil {
+		log.Fatalln(err.Error())
+	}
+
+	spRankingsOut, _ := json.Marshal(spRankings)
 	sp.Write(spRankingsOut)
 	sp.Close()
 	mp, _ := os.Create("./output/mp.json")
-	mpRankingsOut, _ := json.Marshal(*mpRankings)
+	mpRankingsOut, _ := json.Marshal(mpRankings)
 	mp.Write(mpRankingsOut)
 	mp.Close()
 	overall, _ := os.Create("./output/overall.json")
-	overallRankingsOut, _ := json.Marshal(*overallRankings)
+	overallRankingsOut, _ := json.Marshal(overallRankings)
 	overall.Write(overallRankingsOut)
 	overall.Close()
 }

rankings/fetch.go

@@ -9,9 +9,10 @@ import (
 	"net/http"
 	"os"
 	"strconv"
+	"strings"
 )
 
-func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *map[string]*Player {
+func fetchLeaderboard(records []Record, overrides map[string]map[string]int, useCache bool) map[string]*Player {
 	log.Println("fetching leaderboard")
 	players := map[string]*Player{}
 	// first init players map with records from portal gun and doors
@@ -20,8 +21,8 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
 	end := 5000
 
 	for fetchAnotherPage {
-		portalGunEntries := fetchRecordsFromMap(47459, 0, 5000)
-		fetchAnotherPage = portalGunEntries.needsAnotherPage(&(*records)[0])
+		portalGunEntries := fetchRecordsFromMap(47459, 0, 5000, useCache)
+		fetchAnotherPage = portalGunEntries.needsAnotherPage(&records[0])
 		if fetchAnotherPage {
 			start = end + 1
 			end = start + 5000
@@ -49,8 +50,8 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
 	end = 5000
 
 	for fetchAnotherPage {
-		doorsEntries := fetchRecordsFromMap(47740, start, end)
-		fetchAnotherPage = doorsEntries.needsAnotherPage(&(*records)[51])
+		doorsEntries := fetchRecordsFromMap(47740, start, end, useCache)
+		fetchAnotherPage = doorsEntries.needsAnotherPage(&records[51])
 		if fetchAnotherPage {
 			start = end + 1
 			end = start + 5000
@@ -83,7 +84,7 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
 		}
 	}
 
-	for _, record := range *records {
+	for _, record := range records {
 		if record.MapID == 47459 || record.MapID == 47740 {
 			continue
 		}
@@ -93,7 +94,7 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
 		end := 5000
 
 		for fetchAnotherPage {
-			entries := fetchRecordsFromMap(record.MapID, start, end)
+			entries := fetchRecordsFromMap(record.MapID, start, end, useCache)
 			fetchAnotherPage = entries.needsAnotherPage(&record)
 			if fetchAnotherPage {
 				start = end + 1
@@ -106,11 +107,11 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
 				}
 				score := entry.Score
 				if entry.Score < record.MapWR {
-					_, ok := (*overrides)[entry.SteamID]
+					_, ok := overrides[entry.SteamID]
 					if ok {
-						_, ok := (*overrides)[entry.SteamID][strconv.Itoa(record.MapID)]
+						_, ok := overrides[entry.SteamID][strconv.Itoa(record.MapID)]
 						if ok {
-							score = (*overrides)[entry.SteamID][strconv.Itoa(record.MapID)]
+							score = overrides[entry.SteamID][strconv.Itoa(record.MapID)]
 						} else {
 							continue // ban
 						}
@@ -136,28 +137,60 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
 		}
 
 	}
-	return &players
+	return players
 }
 
-func fetchRecordsFromMap(mapID int, start int, end int) *Leaderboard {
-	resp, err := http.Get(fmt.Sprintf("https://steamcommunity.com/stats/Portal2/leaderboards/%d?xml=1&start=%d&end=%d", mapID, start, end))
+func fetchRecordsFromMap(mapID int, start int, end int, useCache bool) *Leaderboard {
+	var filename string
+	if useCache {
+		filename := fmt.Sprintf("./cache/lb_%d_%d_%d.xml", mapID, start, end)
+		log.Println("from cache", filename)
+		file, _ := os.ReadFile(filename)
+		if file != nil {
+			leaderboard := Leaderboard{}
+			err := xml.Unmarshal(file, &leaderboard)
+			if err != nil {
+				log.Fatalln("failed to unmarshal cache.", err.Error())
+			}
+			return &leaderboard
+		}
+	}
+
+	url := fmt.Sprintf("https://steamcommunity.com/stats/Portal2/leaderboards/%d?xml=1&start=%d&end=%d", mapID, start, end)
+	resp, err := http.Get(url)
+	log.Println("fetched", url, ":", resp.StatusCode)
 	if err != nil {
-		log.Fatalln(err.Error())
+		log.Fatalln("failed to fetch leaderboard.", err.Error())
 	}
 	respBytes, err := io.ReadAll(resp.Body)
 	if err != nil {
-		log.Fatalln(err.Error())
+		log.Fatalln("failed to read leadeboard body.", err.Error())
 	}
 	leaderboard := Leaderboard{}
 	err = xml.Unmarshal(respBytes, &leaderboard)
 	if err != nil {
-		log.Fatalln(err.Error())
+		log.Println(string(respBytes))
+		log.Fatalln("failed to unmarshal leaderboard.", err.Error())
 	}
+
+	if useCache {
+		if err = os.WriteFile(filename, respBytes, 0644); err != nil {
+			log.Fatalln("failed write to file.", err.Error())
+		}
+	}
+
 	return &leaderboard
 }
 
-func fetchPlayerInfo(player *Player) {
-	url := fmt.Sprintf("http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v2/?key=%s&steamids=%s", os.Getenv("API_KEY"), player.SteamID)
+func fetchPlayerInfo(players []*Player) {
+	log.Println("fetching info for", len(players), "players")
+
+	ids := make([]string, len(players))
+	for _, player := range players {
+		ids = append(ids, player.SteamID)
+	}
+
+	url := fmt.Sprintf("http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v2/?key=%s&steamids=%s", os.Getenv("API_KEY"), strings.Join(ids, ","))
 	resp, err := http.Get(url)
 	if err != nil {
 		log.Fatalln(err.Error())
@@ -167,6 +200,7 @@ func fetchPlayerInfo(player *Player) {
 		log.Fatalln(err.Error())
 	}
 	type PlayerSummary struct {
+		SteamID     string `json:"steamid"`
 		PersonaName string `json:"personaname"`
 		AvatarFull  string `json:"avatarfull"`
 	}
@@ -180,6 +214,13 @@ func fetchPlayerInfo(player *Player) {
 	if err := json.Unmarshal(body, &data); err != nil {
 		log.Fatalln(err.Error())
 	}
-	player.AvatarLink = data.Response.Players[0].AvatarFull
-	player.Username = data.Response.Players[0].PersonaName
+
+	for _, profile := range data.Response.Players {
+		for _, player := range players {
+			if player.SteamID == profile.SteamID {
+				player.AvatarLink = profile.AvatarFull
+				player.Username = profile.PersonaName
+			}
+		}
+	}
 }

rankings/filter.go

@@ -2,11 +2,12 @@ package main
 
 import (
 	"log"
+	"math"
 	"sort"
 )
 
-func filterRankings(spRankings, mpRankings, overallRankings *[]*Player, players *map[string]*Player) {
-	for k, p := range *players {
+func filterRankings(spRankings, mpRankings, overallRankings *[]*Player, players map[string]*Player) {
+	for k, p := range players {
 		if p.SpIterations == 51 {
 			*spRankings = append(*spRankings, p)
 		}
@@ -18,13 +19,14 @@ func filterRankings(spRankings, mpRankings, overallRankings *[]*Player, players
 			*overallRankings = append(*overallRankings, p)
 		}
 		if p.SpIterations < 51 && p.MpIterations < 48 {
-			delete(*players, k)
+			delete(players, k)
 		}
 	}
 
-	log.Println("getting player summaries")
-	for _, v := range *players {
-		fetchPlayerInfo(v)
+	log.Println("getting player summaries for", len(players), "players")
+
+	for _, chunk := range chunkMap(players, 100) {
+		fetchPlayerInfo(chunk)
 	}
 
 	log.Println("sorting the ranks")
@@ -91,3 +93,26 @@ func filterRankings(spRankings, mpRankings, overallRankings *[]*Player, players
 		(*overallRankings)[idx].OverallRank = rank
 	}
 }
+
+func chunkMap[T any](m map[string]*T, chunkSize int) [][]*T {
+	chunks := make([][]*T, 0, int(math.Ceil(float64(len(m))/float64(chunkSize))))
+	chunk := make([]*T, 0, chunkSize)
+
+	count := 0
+	for _, player := range m {
+		chunk = append(chunk, player)
+		count++
+
+		if count == chunkSize {
+			chunks = append(chunks, chunk)
+			chunk = make([]*T, 0, chunkSize)
+			count = 0
+		}
+	}
+
+	if len(chunk) > 0 {
+		chunks = append(chunks, chunk)
+	}
+
+	return chunks
+}

rankings/main.go

@@ -10,11 +10,33 @@ import (
 	"github.com/robfig/cron/v3"
 )
 
+var useCache = false
+
 func main() {
 	err := godotenv.Load()
 	if err != nil {
 		log.Fatalln("Error loading .env file:", err.Error())
 	}
+
+	runNow := false
+	for _, arg := range os.Args {
+		if arg == "-n" || arg == "--now" {
+			runNow = true
+			continue
+		}
+		if arg == "-c" || arg == "--cache" {
+			useCache = true
+			continue
+		}
+	}
+
+	useCache = useCache && runNow
+
+	if runNow {
+		run()
+		return
+	}
+
 	c := cron.New()
 	_, err = c.AddFunc("0 0 * * *", run)
 	if err != nil {
@@ -29,17 +51,24 @@ func main() {
 
 func run() {
 	log.Println("started job")
+
 	records := readRecords()
+	log.Println("loaded", len(records), "records")
+
 	overrides := readOverrides()
-	players := fetchLeaderboard(records, overrides)
+	log.Println("loaded", len(overrides), "player overrides")
+
+	players := fetchLeaderboard(records, overrides, useCache)
 
 	spRankings := []*Player{}
 	mpRankings := []*Player{}
 	overallRankings := []*Player{}
 
-	log.Println("filtering rankings")
+	log.Println("filtering rankings for", len(players), "players")
 	filterRankings(&spRankings, &mpRankings, &overallRankings, players)
 
-	log.Println("exporting jsons")
-	exportAll(&spRankings, &mpRankings, &overallRankings)
+	log.Println("exporting jsons for", len(players), "players")
+	exportAll(spRankings, mpRankings, overallRankings)
+
+	log.Println("done")
 }

rankings/prefetch.go

@@ -7,7 +7,7 @@ import (
 	"os"
 )
 
-func readRecords() *[]Record {
+func readRecords() []Record {
 	recordsFile, err := os.Open("./input/records.json")
 	if err != nil {
 		log.Fatalln(err.Error())
@@ -22,10 +22,10 @@ func readRecords() *[]Record {
 	if err != nil {
 		log.Fatalln(err.Error())
 	}
-	return &records
+	return records
 }
 
-func readOverrides() *map[string]map[string]int {
+func readOverrides() map[string]map[string]int {
 	overridesFile, err := os.Open("./input/overrides.json")
 	if err != nil {
 		log.Fatalln(err.Error())
@@ -40,5 +40,5 @@ func readOverrides() *map[string]map[string]int {
 	if err != nil {
 		log.Fatalln(err.Error())
 	}
-	return &overrides
+	return overrides
 }