feat/rankings: fetch profiles faster and improvements (#234)v1.0.2

1 files changed, 61 insertions, 20 deletions

diff --git a/rankings/fetch.go b/rankings/fetch.go
index ee5d5bb..cf04e81 100644
--- a/rankings/fetch.go
+++ b/rankings/fetch.go
@@ -9,9 +9,10 @@ import (
         "net/http"
         "os"
         "strconv"
+        "strings"
 )
 
-func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *map[string]*Player {
+func fetchLeaderboard(records []Record, overrides map[string]map[string]int, useCache bool) map[string]*Player {
         log.Println("fetching leaderboard")
         players := map[string]*Player{}
         // first init players map with records from portal gun and doors
@@ -20,8 +21,8 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
         end := 5000
 
         for fetchAnotherPage {
-                portalGunEntries := fetchRecordsFromMap(47459, 0, 5000)
-                fetchAnotherPage = portalGunEntries.needsAnotherPage(&(*records)[0])
+                portalGunEntries := fetchRecordsFromMap(47459, 0, 5000, useCache)
+                fetchAnotherPage = portalGunEntries.needsAnotherPage(&records[0])
                 if fetchAnotherPage {
                         start = end + 1
                         end = start + 5000
@@ -49,8 +50,8 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
         end = 5000
 
         for fetchAnotherPage {
-                doorsEntries := fetchRecordsFromMap(47740, start, end)
-                fetchAnotherPage = doorsEntries.needsAnotherPage(&(*records)[51])
+                doorsEntries := fetchRecordsFromMap(47740, start, end, useCache)
+                fetchAnotherPage = doorsEntries.needsAnotherPage(&records[51])
                 if fetchAnotherPage {
                         start = end + 1
                         end = start + 5000
@@ -83,7 +84,7 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
                 }
         }
 
-        for _, record := range *records {
+        for _, record := range records {
                 if record.MapID == 47459 || record.MapID == 47740 {
                         continue
                 }
@@ -93,7 +94,7 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
                 end := 5000
 
                 for fetchAnotherPage {
-                        entries := fetchRecordsFromMap(record.MapID, start, end)
+                        entries := fetchRecordsFromMap(record.MapID, start, end, useCache)
                         fetchAnotherPage = entries.needsAnotherPage(&record)
                         if fetchAnotherPage {
                                 start = end + 1
@@ -106,11 +107,11 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
                                 }
                                 score := entry.Score
                                 if entry.Score < record.MapWR {
-                                        _, ok := (*overrides)[entry.SteamID]
+                                        _, ok := overrides[entry.SteamID]
                                         if ok {
-                                                _, ok := (*overrides)[entry.SteamID][strconv.Itoa(record.MapID)]
+                                                _, ok := overrides[entry.SteamID][strconv.Itoa(record.MapID)]
                                                 if ok {
-                                                        score = (*overrides)[entry.SteamID][strconv.Itoa(record.MapID)]
+                                                        score = overrides[entry.SteamID][strconv.Itoa(record.MapID)]
                                                 } else {
                                                         continue // ban
                                                 }
@@ -136,28 +137,60 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
                 }
 
         }
-        return &players
+        return players
 }
 
-func fetchRecordsFromMap(mapID int, start int, end int) *Leaderboard {
-        resp, err := http.Get(fmt.Sprintf("https://steamcommunity.com/stats/Portal2/leaderboards/%d?xml=1&start=%d&end=%d", mapID, start, end))
+func fetchRecordsFromMap(mapID int, start int, end int, useCache bool) *Leaderboard {
+        var filename string
+        if useCache {
+                filename := fmt.Sprintf("./cache/lb_%d_%d_%d.xml", mapID, start, end)
+                log.Println("from cache", filename)
+                file, _ := os.ReadFile(filename)
+                if file != nil {
+                        leaderboard := Leaderboard{}
+                        err := xml.Unmarshal(file, &leaderboard)
+                        if err != nil {
+                                log.Fatalln("failed to unmarshal cache.", err.Error())
+                        }
+                        return &leaderboard
+                }
+        }
+
+        url := fmt.Sprintf("https://steamcommunity.com/stats/Portal2/leaderboards/%d?xml=1&start=%d&end=%d", mapID, start, end)
+        resp, err := http.Get(url)
+        log.Println("fetched", url, ":", resp.StatusCode)
         if err != nil {
-                log.Fatalln(err.Error())
+                log.Fatalln("failed to fetch leaderboard.", err.Error())
         }
         respBytes, err := io.ReadAll(resp.Body)
         if err != nil {
-                log.Fatalln(err.Error())
+                log.Fatalln("failed to read leadeboard body.", err.Error())
         }
         leaderboard := Leaderboard{}
         err = xml.Unmarshal(respBytes, &leaderboard)
         if err != nil {
-                log.Fatalln(err.Error())
+                log.Println(string(respBytes))
+                log.Fatalln("failed to unmarshal leaderboard.", err.Error())
         }
+
+        if useCache {
+                if err = os.WriteFile(filename, respBytes, 0644); err != nil {
+                        log.Fatalln("failed write to file.", err.Error())
+                }
+        }
+
         return &leaderboard
 }
 
-func fetchPlayerInfo(player *Player) {
-        url := fmt.Sprintf("http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v2/?key=%s&steamids=%s", os.Getenv("API_KEY"), player.SteamID)
+func fetchPlayerInfo(players []*Player) {
+        log.Println("fetching info for", len(players), "players")
+
+        ids := make([]string, len(players))
+        for _, player := range players {
+                ids = append(ids, player.SteamID)
+        }
+
+        url := fmt.Sprintf("http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v2/?key=%s&steamids=%s", os.Getenv("API_KEY"), strings.Join(ids, ","))
         resp, err := http.Get(url)
         if err != nil {
                 log.Fatalln(err.Error())
@@ -167,6 +200,7 @@ func fetchPlayerInfo(player *Player) {
                 log.Fatalln(err.Error())
         }
         type PlayerSummary struct {
+                SteamID     string `json:"steamid"`
                 PersonaName string `json:"personaname"`
                 AvatarFull  string `json:"avatarfull"`
         }
@@ -180,6 +214,13 @@ func fetchPlayerInfo(player *Player) {
         if err := json.Unmarshal(body, &data); err != nil {
                 log.Fatalln(err.Error())
         }
-        player.AvatarLink = data.Response.Players[0].AvatarFull
-        player.Username = data.Response.Players[0].PersonaName
+
+        for _, profile := range data.Response.Players {
+                for _, player := range players {
+                        if player.SteamID == profile.SteamID {
+                                player.AvatarLink = profile.AvatarFull
+                                player.Username = profile.PersonaName
+                        }
+                }
+        }
 }