From 5fb47b69d895fcbe98fc714b47057b0051387e05 Mon Sep 17 00:00:00 2001 From: NeKz Date: Sat, 16 Nov 2024 08:24:03 +0100 Subject: feat/rankings: fetch profiles faster and improvements (#234) --- rankings/.gitignore | 3 +- rankings/export.go | 20 +++++++++---- rankings/fetch.go | 81 +++++++++++++++++++++++++++++++++++++++------------- rankings/filter.go | 37 ++++++++++++++++++++---- rankings/main.go | 37 +++++++++++++++++++++--- rankings/prefetch.go | 8 +++--- 6 files changed, 146 insertions(+), 40 deletions(-) diff --git a/rankings/.gitignore b/rankings/.gitignore index 764d23d..48d3792 100644 --- a/rankings/.gitignore +++ b/rankings/.gitignore @@ -1,2 +1,3 @@ .env -output/ \ No newline at end of file +output/ +cache/ diff --git a/rankings/export.go b/rankings/export.go index 20dfebe..cdb9213 100644 --- a/rankings/export.go +++ b/rankings/export.go @@ -2,20 +2,30 @@ package main import ( "encoding/json" + "log" "os" ) -func exportAll(spRankings, mpRankings, overallRankings *[]*Player) { - sp, _ := os.Create("./output/sp.json") - spRankingsOut, _ := json.Marshal(*spRankings) +func exportAll(spRankings, mpRankings, overallRankings []*Player) { + err := os.Mkdir("./output", 0775) + if err != nil && !os.IsExist(err) { + log.Fatalln(err.Error()) + } + + sp, err := os.Create("./output/sp.json") + if err != nil { + log.Fatalln(err.Error()) + } + + spRankingsOut, _ := json.Marshal(spRankings) sp.Write(spRankingsOut) sp.Close() mp, _ := os.Create("./output/mp.json") - mpRankingsOut, _ := json.Marshal(*mpRankings) + mpRankingsOut, _ := json.Marshal(mpRankings) mp.Write(mpRankingsOut) mp.Close() overall, _ := os.Create("./output/overall.json") - overallRankingsOut, _ := json.Marshal(*overallRankings) + overallRankingsOut, _ := json.Marshal(overallRankings) overall.Write(overallRankingsOut) overall.Close() } diff --git a/rankings/fetch.go b/rankings/fetch.go index ee5d5bb..cf04e81 100644 --- a/rankings/fetch.go +++ b/rankings/fetch.go @@ -9,9 +9,10 @@ import ( "net/http" "os" "strconv" + "strings" ) -func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) *map[string]*Player { +func fetchLeaderboard(records []Record, overrides map[string]map[string]int, useCache bool) map[string]*Player { log.Println("fetching leaderboard") players := map[string]*Player{} // first init players map with records from portal gun and doors @@ -20,8 +21,8 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) * end := 5000 for fetchAnotherPage { - portalGunEntries := fetchRecordsFromMap(47459, 0, 5000) - fetchAnotherPage = portalGunEntries.needsAnotherPage(&(*records)[0]) + portalGunEntries := fetchRecordsFromMap(47459, 0, 5000, useCache) + fetchAnotherPage = portalGunEntries.needsAnotherPage(&records[0]) if fetchAnotherPage { start = end + 1 end = start + 5000 @@ -49,8 +50,8 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) * end = 5000 for fetchAnotherPage { - doorsEntries := fetchRecordsFromMap(47740, start, end) - fetchAnotherPage = doorsEntries.needsAnotherPage(&(*records)[51]) + doorsEntries := fetchRecordsFromMap(47740, start, end, useCache) + fetchAnotherPage = doorsEntries.needsAnotherPage(&records[51]) if fetchAnotherPage { start = end + 1 end = start + 5000 @@ -83,7 +84,7 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) * } } - for _, record := range *records { + for _, record := range records { if record.MapID == 47459 || record.MapID == 47740 { continue } @@ -93,7 +94,7 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) * end := 5000 for fetchAnotherPage { - entries := fetchRecordsFromMap(record.MapID, start, end) + entries := fetchRecordsFromMap(record.MapID, start, end, useCache) fetchAnotherPage = entries.needsAnotherPage(&record) if fetchAnotherPage { start = end + 1 @@ -106,11 +107,11 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) * } score := entry.Score if entry.Score < record.MapWR { - _, ok := (*overrides)[entry.SteamID] + _, ok := overrides[entry.SteamID] if ok { - _, ok := (*overrides)[entry.SteamID][strconv.Itoa(record.MapID)] + _, ok := overrides[entry.SteamID][strconv.Itoa(record.MapID)] if ok { - score = (*overrides)[entry.SteamID][strconv.Itoa(record.MapID)] + score = overrides[entry.SteamID][strconv.Itoa(record.MapID)] } else { continue // ban } @@ -136,28 +137,60 @@ func fetchLeaderboard(records *[]Record, overrides *map[string]map[string]int) * } } - return &players + return players } -func fetchRecordsFromMap(mapID int, start int, end int) *Leaderboard { - resp, err := http.Get(fmt.Sprintf("https://steamcommunity.com/stats/Portal2/leaderboards/%d?xml=1&start=%d&end=%d", mapID, start, end)) +func fetchRecordsFromMap(mapID int, start int, end int, useCache bool) *Leaderboard { + var filename string + if useCache { + filename := fmt.Sprintf("./cache/lb_%d_%d_%d.xml", mapID, start, end) + log.Println("from cache", filename) + file, _ := os.ReadFile(filename) + if file != nil { + leaderboard := Leaderboard{} + err := xml.Unmarshal(file, &leaderboard) + if err != nil { + log.Fatalln("failed to unmarshal cache.", err.Error()) + } + return &leaderboard + } + } + + url := fmt.Sprintf("https://steamcommunity.com/stats/Portal2/leaderboards/%d?xml=1&start=%d&end=%d", mapID, start, end) + resp, err := http.Get(url) + log.Println("fetched", url, ":", resp.StatusCode) if err != nil { - log.Fatalln(err.Error()) + log.Fatalln("failed to fetch leaderboard.", err.Error()) } respBytes, err := io.ReadAll(resp.Body) if err != nil { - log.Fatalln(err.Error()) + log.Fatalln("failed to read leadeboard body.", err.Error()) } leaderboard := Leaderboard{} err = xml.Unmarshal(respBytes, &leaderboard) if err != nil { - log.Fatalln(err.Error()) + log.Println(string(respBytes)) + log.Fatalln("failed to unmarshal leaderboard.", err.Error()) } + + if useCache { + if err = os.WriteFile(filename, respBytes, 0644); err != nil { + log.Fatalln("failed write to file.", err.Error()) + } + } + return &leaderboard } -func fetchPlayerInfo(player *Player) { - url := fmt.Sprintf("http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v2/?key=%s&steamids=%s", os.Getenv("API_KEY"), player.SteamID) +func fetchPlayerInfo(players []*Player) { + log.Println("fetching info for", len(players), "players") + + ids := make([]string, len(players)) + for _, player := range players { + ids = append(ids, player.SteamID) + } + + url := fmt.Sprintf("http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v2/?key=%s&steamids=%s", os.Getenv("API_KEY"), strings.Join(ids, ",")) resp, err := http.Get(url) if err != nil { log.Fatalln(err.Error()) @@ -167,6 +200,7 @@ func fetchPlayerInfo(player *Player) { log.Fatalln(err.Error()) } type PlayerSummary struct { + SteamID string `json:"steamid"` PersonaName string `json:"personaname"` AvatarFull string `json:"avatarfull"` } @@ -180,6 +214,13 @@ func fetchPlayerInfo(player *Player) { if err := json.Unmarshal(body, &data); err != nil { log.Fatalln(err.Error()) } - player.AvatarLink = data.Response.Players[0].AvatarFull - player.Username = data.Response.Players[0].PersonaName + + for _, profile := range data.Response.Players { + for _, player := range players { + if player.SteamID == profile.SteamID { + player.AvatarLink = profile.AvatarFull + player.Username = profile.PersonaName + } + } + } } diff --git a/rankings/filter.go b/rankings/filter.go index 1d7233b..2af7911 100644 --- a/rankings/filter.go +++ b/rankings/filter.go @@ -2,11 +2,12 @@ package main import ( "log" + "math" "sort" ) -func filterRankings(spRankings, mpRankings, overallRankings *[]*Player, players *map[string]*Player) { - for k, p := range *players { +func filterRankings(spRankings, mpRankings, overallRankings *[]*Player, players map[string]*Player) { + for k, p := range players { if p.SpIterations == 51 { *spRankings = append(*spRankings, p) } @@ -18,13 +19,14 @@ func filterRankings(spRankings, mpRankings, overallRankings *[]*Player, players *overallRankings = append(*overallRankings, p) } if p.SpIterations < 51 && p.MpIterations < 48 { - delete(*players, k) + delete(players, k) } } - log.Println("getting player summaries") - for _, v := range *players { - fetchPlayerInfo(v) + log.Println("getting player summaries for", len(players), "players") + + for _, chunk := range chunkMap(players, 100) { + fetchPlayerInfo(chunk) } log.Println("sorting the ranks") @@ -91,3 +93,26 @@ func filterRankings(spRankings, mpRankings, overallRankings *[]*Player, players (*overallRankings)[idx].OverallRank = rank } } + +func chunkMap[T any](m map[string]*T, chunkSize int) [][]*T { + chunks := make([][]*T, 0, int(math.Ceil(float64(len(m))/float64(chunkSize)))) + chunk := make([]*T, 0, chunkSize) + + count := 0 + for _, player := range m { + chunk = append(chunk, player) + count++ + + if count == chunkSize { + chunks = append(chunks, chunk) + chunk = make([]*T, 0, chunkSize) + count = 0 + } + } + + if len(chunk) > 0 { + chunks = append(chunks, chunk) + } + + return chunks +} diff --git a/rankings/main.go b/rankings/main.go index dfafb0c..552f058 100644 --- a/rankings/main.go +++ b/rankings/main.go @@ -10,11 +10,33 @@ import ( "github.com/robfig/cron/v3" ) +var useCache = false + func main() { err := godotenv.Load() if err != nil { log.Fatalln("Error loading .env file:", err.Error()) } + + runNow := false + for _, arg := range os.Args { + if arg == "-n" || arg == "--now" { + runNow = true + continue + } + if arg == "-c" || arg == "--cache" { + useCache = true + continue + } + } + + useCache = useCache && runNow + + if runNow { + run() + return + } + c := cron.New() _, err = c.AddFunc("0 0 * * *", run) if err != nil { @@ -29,17 +51,24 @@ func main() { func run() { log.Println("started job") + records := readRecords() + log.Println("loaded", len(records), "records") + overrides := readOverrides() - players := fetchLeaderboard(records, overrides) + log.Println("loaded", len(overrides), "player overrides") + + players := fetchLeaderboard(records, overrides, useCache) spRankings := []*Player{} mpRankings := []*Player{} overallRankings := []*Player{} - log.Println("filtering rankings") + log.Println("filtering rankings for", len(players), "players") filterRankings(&spRankings, &mpRankings, &overallRankings, players) - log.Println("exporting jsons") - exportAll(&spRankings, &mpRankings, &overallRankings) + log.Println("exporting jsons for", len(players), "players") + exportAll(spRankings, mpRankings, overallRankings) + + log.Println("done") } diff --git a/rankings/prefetch.go b/rankings/prefetch.go index 487a76f..a559b26 100644 --- a/rankings/prefetch.go +++ b/rankings/prefetch.go @@ -7,7 +7,7 @@ import ( "os" ) -func readRecords() *[]Record { +func readRecords() []Record { recordsFile, err := os.Open("./input/records.json") if err != nil { log.Fatalln(err.Error()) @@ -22,10 +22,10 @@ func readRecords() *[]Record { if err != nil { log.Fatalln(err.Error()) } - return &records + return records } -func readOverrides() *map[string]map[string]int { +func readOverrides() map[string]map[string]int { overridesFile, err := os.Open("./input/overrides.json") if err != nil { log.Fatalln(err.Error()) @@ -40,5 +40,5 @@ func readOverrides() *map[string]map[string]int { if err != nil { log.Fatalln(err.Error()) } - return &overrides + return overrides } -- cgit v1.2.3