feat/rankings: fetch profiles faster and improvements (#234)v1.0.2

6 files changed, 146 insertions, 40 deletions

diff --git a/rankings/.gitignore b/rankings/.gitignore
index 764d23d..48d3792 100644
--- a/rankings/.gitignore
+++ b/rankings/.gitignore
@@ -1,2 +1,3 @@
 .env
-output/
\ No newline at end of file
+output/
+cache/
diff --git a/rankings/export.go b/rankings/export.go
index 20dfebe..cdb9213 100644
--- a/rankings/export.go
+++ b/rankings/export.go
@@ -2,20 +2,30 @@ package main
 
 import (
         "encoding/json"
+        "log"
         "os"
 )
 
-func exportAll(spRankings, mpRankings, overallRankings *[]*Player) {
-        sp, _ := os.Create("./output/sp.json")
-        spRankingsOut, _ := json.Marshal(*spRankings)
+func exportAll(spRankings, mpRankings, overallRankings []*Player) {
+        err := os.Mkdir("./output", 0775)
+        if err != nil && !os.IsExist(err) {
+                log.Fatalln(err.Error())
+        }
+
+        sp, err := os.Create("./output/sp.json")
+        if err != nil {
+                log.Fatalln(err.Error())
+        }
+
+        spRankingsOut, _ := json.Marshal(spRankings)
         sp.Write(spRankingsOut)
         sp.Close()
         mp, _ := os.Create("./output/mp.json")
-        mpRankingsOut, _ := json.Marshal(*mpRankings)
+        mpRankingsOut, _ := json.Marshal(mpRankings)
         mp.Write(mpRankingsOut)
         mp.Close()
         overall, _ := os.Create("./output/overall.json")
-        overallRankingsOut, _ := json.Marshal(*overallRankings)
+        overallRankingsOut, _ := json.Marshal(overallRankings)
         overall.Write(overallRankingsOut)
         overall.Close()
 }
diff --git a/rankings/fetch.go b/rankings/fetch.go
index ee5d5bb..cf04e81 100644
--- a/rankings/fetch.go
+++ b/rankings/fetch.go
@@ -9,9 +9,10 @@ import (
         "net/http"
         "os"
         "strconv"
+        "strings"
 )
 
-func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *map[string]*Player {
+func fetchLeaderboard(records []Record, overrides map[string]map[string]int, useCache bool) map[string]*Player {
         log.Println("fetching leaderboard")
         players := map[string]*Player{}
         // first init players map with records from portal gun and doors
@@ -20,8 +21,8 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
         end := 5000
 
         for fetchAnotherPage {
-                portalGunEntries := fetchRecordsFromMap(47459, 0, 5000)
-                fetchAnotherPage = portalGunEntries.needsAnotherPage(&(*records)[0])
+                portalGunEntries := fetchRecordsFromMap(47459, 0, 5000, useCache)
+                fetchAnotherPage = portalGunEntries.needsAnotherPage(&records[0])
                 if fetchAnotherPage {
                         start = end + 1
                         end = start + 5000
@@ -49,8 +50,8 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
         end = 5000
 
         for fetchAnotherPage {
-                doorsEntries := fetchRecordsFromMap(47740, start, end)
-                fetchAnotherPage = doorsEntries.needsAnotherPage(&(*records)[51])
+                doorsEntries := fetchRecordsFromMap(47740, start, end, useCache)
+                fetchAnotherPage = doorsEntries.needsAnotherPage(&records[51])
                 if fetchAnotherPage {
                         start = end + 1
                         end = start + 5000
@@ -83,7 +84,7 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
                 }
         }
 
-        for _, record := range *records {
+        for _, record := range records {
                 if record.MapID == 47459 || record.MapID == 47740 {
                         continue
                 }
@@ -93,7 +94,7 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
                 end := 5000
 
                 for fetchAnotherPage {
-                        entries := fetchRecordsFromMap(record.MapID, start, end)
+                        entries := fetchRecordsFromMap(record.MapID, start, end, useCache)
                         fetchAnotherPage = entries.needsAnotherPage(&record)
                         if fetchAnotherPage {
                                 start = end + 1
@@ -106,11 +107,11 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
                                 }
                                 score := entry.Score
                                 if entry.Score < record.MapWR {
-                                        _, ok := (*overrides)[entry.SteamID]
+                                        _, ok := overrides[entry.SteamID]
                                         if ok {
-                                                _, ok := (*overrides)[entry.SteamID][strconv.Itoa(record.MapID)]
+                                                _, ok := overrides[entry.SteamID][strconv.Itoa(record.MapID)]
                                                 if ok {
-                                                        score = (*overrides)[entry.SteamID][strconv.Itoa(record.MapID)]
+                                                        score = overrides[entry.SteamID][strconv.Itoa(record.MapID)]
                                                 } else {
                                                         continue // ban
                                                 }
@@ -136,28 +137,60 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *
                 }
 
         }
-        return &players
+        return players
 }
 
-func fetchRecordsFromMap(mapID int, start int, end int) *Leaderboard {
-        resp, err := http.Get(fmt.Sprintf("https://steamcommunity.com/stats/Portal2/leaderboards/%d?xml=1&start=%d&end=%d", mapID, start, end))
+func fetchRecordsFromMap(mapID int, start int, end int, useCache bool) *Leaderboard {
+        var filename string
+        if useCache {
+                filename := fmt.Sprintf("./cache/lb_%d_%d_%d.xml", mapID, start, end)
+                log.Println("from cache", filename)
+                file, _ := os.ReadFile(filename)
+                if file != nil {
+                        leaderboard := Leaderboard{}
+                        err := xml.Unmarshal(file, &leaderboard)
+                        if err != nil {
+                                log.Fatalln("failed to unmarshal cache.", err.Error())
+                        }
+                        return &leaderboard
+                }
+        }
+
+        url := fmt.Sprintf("https://steamcommunity.com/stats/Portal2/leaderboards/%d?xml=1&start=%d&end=%d", mapID, start, end)
+        resp, err := http.Get(url)
+        log.Println("fetched", url, ":", resp.StatusCode)
         if err != nil {
-                log.Fatalln(err.Error())
+                log.Fatalln("failed to fetch leaderboard.", err.Error())
         }
         respBytes, err := io.ReadAll(resp.Body)
         if err != nil {
-                log.Fatalln(err.Error())
+                log.Fatalln("failed to read leadeboard body.", err.Error())
         }
         leaderboard := Leaderboard{}
         err = xml.Unmarshal(respBytes, &leaderboard)
         if err != nil {
-                log.Fatalln(err.Error())
+                log.Println(string(respBytes))
+                log.Fatalln("failed to unmarshal leaderboard.", err.Error())
         }
+
+        if useCache {
+                if err = os.WriteFile(filename, respBytes, 0644); err != nil {
+                        log.Fatalln("failed write to file.", err.Error())
+                }
+        }
+
         return &leaderboard
 }
 
-func fetchPlayerInfo(player *Player) {
-        url := fmt.Sprintf("http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v2/?key=%s&steamids=%s", os.Getenv("API_KEY"), player.SteamID)
+func fetchPlayerInfo(players []*Player) {
+        log.Println("fetching info for", len(players), "players")
+
+        ids := make([]string, len(players))
+        for _, player := range players {
+                ids = append(ids, player.SteamID)
+        }
+
+        url := fmt.Sprintf("http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v2/?key=%s&steamids=%s", os.Getenv("API_KEY"), strings.Join(ids, ","))
         resp, err := http.Get(url)
         if err != nil {
                 log.Fatalln(err.Error())
@@ -167,6 +200,7 @@ func fetchPlayerInfo(player *Player) {
                 log.Fatalln(err.Error())
         }
         type PlayerSummary struct {
+                SteamID     string `json:"steamid"`
                 PersonaName string `json:"personaname"`
                 AvatarFull  string `json:"avatarfull"`
         }
@@ -180,6 +214,13 @@ func fetchPlayerInfo(player *Player) {
         if err := json.Unmarshal(body, &data); err != nil {
                 log.Fatalln(err.Error())
         }
-        player.AvatarLink = data.Response.Players[0].AvatarFull
-        player.Username = data.Response.Players[0].PersonaName
+
+        for _, profile := range data.Response.Players {
+                for _, player := range players {
+                        if player.SteamID == profile.SteamID {
+                                player.AvatarLink = profile.AvatarFull
+                                player.Username = profile.PersonaName
+                        }
+                }
+        }
 }
diff --git a/rankings/filter.go b/rankings/filter.go
index 1d7233b..2af7911 100644
--- a/rankings/filter.go
+++ b/rankings/filter.go
@@ -2,11 +2,12 @@ package main
 
 import (
         "log"
+        "math"
         "sort"
 )
 
-func filterRankings(spRankings, mpRankings, overallRankings *[]*Player, players *map[string]*Player) {
-        for k, p := range *players {
+func filterRankings(spRankings, mpRankings, overallRankings *[]*Player, players map[string]*Player) {
+        for k, p := range players {
                 if p.SpIterations == 51 {
                         *spRankings = append(*spRankings, p)
                 }
@@ -18,13 +19,14 @@ func filterRankings(spRankings, mpRankings, overallRankings *[]*Player, players
                         *overallRankings = append(*overallRankings, p)
                 }
                 if p.SpIterations < 51 && p.MpIterations < 48 {
-                        delete(*players, k)
+                        delete(players, k)
                 }
         }
 
-        log.Println("getting player summaries")
-        for _, v := range *players {
-                fetchPlayerInfo(v)
+        log.Println("getting player summaries for", len(players), "players")
+
+        for _, chunk := range chunkMap(players, 100) {
+                fetchPlayerInfo(chunk)
         }
 
         log.Println("sorting the ranks")
@@ -91,3 +93,26 @@ func filterRankings(spRankings, mpRankings, overallRankings *[]*Player, players
                 (*overallRankings)[idx].OverallRank = rank
         }
 }
+
+func chunkMap[T any](m map[string]*T, chunkSize int) [][]*T {
+        chunks := make([][]*T, 0, int(math.Ceil(float64(len(m))/float64(chunkSize))))
+        chunk := make([]*T, 0, chunkSize)
+
+        count := 0
+        for _, player := range m {
+                chunk = append(chunk, player)
+                count++
+
+                if count == chunkSize {
+                        chunks = append(chunks, chunk)
+                        chunk = make([]*T, 0, chunkSize)
+                        count = 0
+                }
+        }
+
+        if len(chunk) > 0 {
+                chunks = append(chunks, chunk)
+        }
+
+        return chunks
+}
diff --git a/rankings/main.go b/rankings/main.go
index dfafb0c..552f058 100644
--- a/rankings/main.go
+++ b/rankings/main.go
@@ -10,11 +10,33 @@ import (
         "github.com/robfig/cron/v3"
 )
 
+var useCache = false
+
 func main() {
         err := godotenv.Load()
         if err != nil {
                 log.Fatalln("Error loading .env file:", err.Error())
         }
+
+        runNow := false
+        for _, arg := range os.Args {
+                if arg == "-n" || arg == "--now" {
+                        runNow = true
+                        continue
+                }
+                if arg == "-c" || arg == "--cache" {
+                        useCache = true
+                        continue
+                }
+        }
+
+        useCache = useCache && runNow
+
+        if runNow {
+                run()
+                return
+        }
+
         c := cron.New()
         _, err = c.AddFunc("0 0 * * *", run)
         if err != nil {
@@ -29,17 +51,24 @@ func main() {
 
 func run() {
         log.Println("started job")
+
         records := readRecords()
+        log.Println("loaded", len(records), "records")
+
         overrides := readOverrides()
-        players := fetchLeaderboard(records, overrides)
+        log.Println("loaded", len(overrides), "player overrides")
+
+        players := fetchLeaderboard(records, overrides, useCache)
 
         spRankings := []*Player{}
         mpRankings := []*Player{}
         overallRankings := []*Player{}
 
-        log.Println("filtering rankings")
+        log.Println("filtering rankings for", len(players), "players")
         filterRankings(&spRankings, &mpRankings, &overallRankings, players)
 
-        log.Println("exporting jsons")
-        exportAll(&spRankings, &mpRankings, &overallRankings)
+        log.Println("exporting jsons for", len(players), "players")
+        exportAll(spRankings, mpRankings, overallRankings)
+
+        log.Println("done")
 }
diff --git a/rankings/prefetch.go b/rankings/prefetch.go
index 487a76f..a559b26 100644
--- a/rankings/prefetch.go
+++ b/rankings/prefetch.go
@@ -7,7 +7,7 @@ import (
         "os"
 )
 
-func readRecords() *[]Record {
+func readRecords() []Record {
         recordsFile, err := os.Open("./input/records.json")
         if err != nil {
                 log.Fatalln(err.Error())
@@ -22,10 +22,10 @@ func readRecords() *[]Record {
         if err != nil {
                 log.Fatalln(err.Error())
         }
-        return &records
+        return records
 }
 
-func readOverrides() *map[string]map[string]int {
+func readOverrides() map[string]map[string]int {
         overridesFile, err := os.Open("./input/overrides.json")
         if err != nil {
                 log.Fatalln(err.Error())
@@ -40,5 +40,5 @@ func readOverrides() *map[string]map[string]int {
         if err != nil {
                 log.Fatalln(err.Error())
         }
-        return &overrides
+        return overrides
 }