From 5fb47b69d895fcbe98fc714b47057b0051387e05 Mon Sep 17 00:00:00 2001 From: NeKz Date: Sat, 16 Nov 2024 08:24:03 +0100 Subject: feat/rankings: fetch profiles faster and improvements (#234) --- rankings/fetch.go | 81 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 61 insertions(+), 20 deletions(-) (limited to 'rankings/fetch.go') diff --git a/rankings/fetch.go b/rankings/fetch.go index ee5d5bb..cf04e81 100644 --- a/rankings/fetch.go +++ b/rankings/fetch.go @@ -9,9 +9,10 @@ import ( "net/http" "os" "strconv" + "strings" ) -func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *map[string]*Player { +func fetchLeaderboard(records []Record, overrides map[string]map[string]int, useCache bool) map[string]*Player { log.Println("fetching leaderboard") players := map[string]*Player{} // first init players map with records from portal gun and doors @@ -20,8 +21,8 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) * end := 5000 for fetchAnotherPage { - portalGunEntries := fetchRecordsFromMap(47459, 0, 5000) - fetchAnotherPage = portalGunEntries.needsAnotherPage(&(*records)[0]) + portalGunEntries := fetchRecordsFromMap(47459, 0, 5000, useCache) + fetchAnotherPage = portalGunEntries.needsAnotherPage(&records[0]) if fetchAnotherPage { start = end + 1 end = start + 5000 @@ -49,8 +50,8 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) * end = 5000 for fetchAnotherPage { - doorsEntries := fetchRecordsFromMap(47740, start, end) - fetchAnotherPage = doorsEntries.needsAnotherPage(&(*records)[51]) + doorsEntries := fetchRecordsFromMap(47740, start, end, useCache) + fetchAnotherPage = doorsEntries.needsAnotherPage(&records[51]) if fetchAnotherPage { start = end + 1 end = start + 5000 @@ -83,7 +84,7 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) * } } - for _, record := range *records { + for _, record := range records { if record.MapID == 47459 || record.MapID == 47740 { continue } @@ -93,7 +94,7 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) * end := 5000 for fetchAnotherPage { - entries := fetchRecordsFromMap(record.MapID, start, end) + entries := fetchRecordsFromMap(record.MapID, start, end, useCache) fetchAnotherPage = entries.needsAnotherPage(&record) if fetchAnotherPage { start = end + 1 @@ -106,11 +107,11 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) * } score := entry.Score if entry.Score < record.MapWR { - _, ok := (*overrides)[entry.SteamID] + _, ok := overrides[entry.SteamID] if ok { - _, ok := (*overrides)[entry.SteamID][strconv.Itoa(record.MapID)] + _, ok := overrides[entry.SteamID][strconv.Itoa(record.MapID)] if ok { - score = (*overrides)[entry.SteamID][strconv.Itoa(record.MapID)] + score = overrides[entry.SteamID][strconv.Itoa(record.MapID)] } else { continue // ban } @@ -136,28 +137,60 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) * } } - return &players + return players } -func fetchRecordsFromMap(mapID int, start int, end int) *Leaderboard { - resp, err := http.Get(fmt.Sprintf("https://steamcommunity.com/stats/Portal2/leaderboards/%d?xml=1&start=%d&end=%d", mapID, start, end)) +func fetchRecordsFromMap(mapID int, start int, end int, useCache bool) *Leaderboard { + var filename string + if useCache { + filename := fmt.Sprintf("./cache/lb_%d_%d_%d.xml", mapID, start, end) + log.Println("from cache", filename) + file, _ := os.ReadFile(filename) + if file != nil { + leaderboard := Leaderboard{} + err := xml.Unmarshal(file, &leaderboard) + if err != nil { + log.Fatalln("failed to unmarshal cache.", err.Error()) + } + return &leaderboard + } + } + + url := fmt.Sprintf("https://steamcommunity.com/stats/Portal2/leaderboards/%d?xml=1&start=%d&end=%d", mapID, start, end) + resp, err := http.Get(url) + log.Println("fetched", url, ":", resp.StatusCode) if err != nil { - log.Fatalln(err.Error()) + log.Fatalln("failed to fetch leaderboard.", err.Error()) } respBytes, err := io.ReadAll(resp.Body) if err != nil { - log.Fatalln(err.Error()) + log.Fatalln("failed to read leadeboard body.", err.Error()) } leaderboard := Leaderboard{} err = xml.Unmarshal(respBytes, &leaderboard) if err != nil { - log.Fatalln(err.Error()) + log.Println(string(respBytes)) + log.Fatalln("failed to unmarshal leaderboard.", err.Error()) } + + if useCache { + if err = os.WriteFile(filename, respBytes, 0644); err != nil { + log.Fatalln("failed write to file.", err.Error()) + } + } + return &leaderboard } -func fetchPlayerInfo(player *Player) { - url := fmt.Sprintf("http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v2/?key=%s&steamids=%s", os.Getenv("API_KEY"), player.SteamID) +func fetchPlayerInfo(players []*Player) { + log.Println("fetching info for", len(players), "players") + + ids := make([]string, len(players)) + for _, player := range players { + ids = append(ids, player.SteamID) + } + + url := fmt.Sprintf("http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v2/?key=%s&steamids=%s", os.Getenv("API_KEY"), strings.Join(ids, ",")) resp, err := http.Get(url) if err != nil { log.Fatalln(err.Error()) @@ -167,6 +200,7 @@ func fetchPlayerInfo(player *Player) { log.Fatalln(err.Error()) } type PlayerSummary struct { + SteamID string `json:"steamid"` PersonaName string `json:"personaname"` AvatarFull string `json:"avatarfull"` } @@ -180,6 +214,13 @@ func fetchPlayerInfo(player *Player) { if err := json.Unmarshal(body, &data); err != nil { log.Fatalln(err.Error()) } - player.AvatarLink = data.Response.Players[0].AvatarFull - player.Username = data.Response.Players[0].PersonaName + + for _, profile := range data.Response.Players { + for _, player := range players { + if player.SteamID == profile.SteamID { + player.AvatarLink = profile.AvatarFull + player.Username = profile.PersonaName + } + } + } } -- cgit v1.2.3