I’ve been keeping track of my models performances in an Excel spreadsheet, but I found it quite tedious to copy the data from the website, so today I wrote some code to get the data from numerapi in the way I wanted, and thought I’d share it in case it helps anyone else.
from tqdm.notebook import tqdm
import datetime
import pytz
import pandas as pd
import numpy as np
from numerapi import NumerAPI, utils
import math
from collections import defaultdict
class CachingNumerAPIWithPercentiles(NumerAPI):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.round_details_cache = {}
self.daily_submissions_performance_with_percentiles_cache = {}
self.competition_cache = None
def round_details(self, round_num):
if not self.round_details_cache.get(round_num):
self.round_details_cache[round_num] = super().round_details(round_num)
return self.round_details_cache[round_num]
def daily_submissions_performance_with_percentiles(self, username):
if not self.daily_submissions_performance_with_percentiles_cache.get(username):
# this is the same as NumerAPI.daily_submissions_performance but we include the percentiles in the query
query = """
query($username: String!) {
v2UserProfile(username: $username) {
dailySubmissionPerformances {
date
roundNumber
correlation
corrPercentile
mmc
mmcPercentile
fnc
fncPercentile
correlationWithMetamodel
}
}
}
"""
arguments = {'username': username}
data = self.raw_query(query, arguments)['data']['v2UserProfile']
performances = data['dailySubmissionPerformances']
# convert strings to python objects
for perf in performances:
utils.replace(perf, "date", utils.parse_datetime_string)
# remove useless items
performances = [p for p in performances if any([p['correlation'], p['fnc'], p['mmc']])]
self.daily_submissions_performance_with_percentiles_cache[username] = performances
return self.daily_submissions_performance_with_percentiles_cache[username]
def get_competitions(self):
if not self.competition_cache:
self.competition_cache = super().get_competitions()
return self.competition_cache
def round_summary(self, round_num):
for some_round_summmary in self.get_competitions():
if round_num == some_round_summmary["number"]:
return some_round_summmary
class AdvancedNumerAPI(object):
def __init__(self, napicache=None):
self.napicache = napicache or CachingNumerAPIWithPercentiles()
def get_current_scored_round(self):
current_round = self.napicache.get_current_round()
if not self.napicache.round_details(current_round):
# if there are no details the round does not have any scored values yet
current_round-=1
return current_round
def is_round_resolved(self, round_num):
return self.napicache.round_summary(round_num)["resolvedStaking"]
def get_user_names_for_rounds(self, round_min, round_max=None, disable_tqdm=False):
round_max = round_max or self.get_current_scored_round()
usernames = set()
for round_num in tqdm(range(round_min, round_max+1), disable=disable_tqdm):
for round_detail in self.napicache.round_details(round_num):
usernames.add(round_detail["username"])
return usernames
def get_user_performance_by_round(self, round_min, round_max=None, model_names=None, disable_tqdm=False):
round_max = round_max or self.get_current_scored_round()
model_names = model_names or self.get_user_names_for_rounds(round_min, round_max, disable_tqdm=True)
user_performance_by_round = defaultdict(dict)
for username in tqdm(model_names, disable=disable_tqdm):
user_daily_submissions_performance = self.napicache.daily_submissions_performance_with_percentiles(username)
for daily_submission_performance in user_daily_submissions_performance:
round_number = daily_submission_performance["roundNumber"]
if round_number < round_min or round_number > round_max:
continue
date_of_performance = daily_submission_performance["date"]
prev_date_of_performance = user_performance_by_round[round_number][username]["date"] if user_performance_by_round[round_number].get(username) else datetime.datetime.min.replace(tzinfo=pytz.UTC)
if date_of_performance > prev_date_of_performance:
user_performance_by_round[round_number][username] = daily_submission_performance
return user_performance_by_round
def get_leaderboard_for_round(self, round_num=None, by="correlation", model_names=None, pct_format=True, disable_tqdm=False):
round_num = round_num or self.get_current_scored_round()
round_leaderboard = pd.DataFrame.from_dict(self.get_user_performance_by_round(round_num, round_num, model_names=model_names, disable_tqdm=disable_tqdm)[round_num], orient="index")
round_leaderboard.sort_values(by=by, inplace=True, ascending=by=="correlationWithMetamodel")
round_leaderboard.drop(["date","roundNumber"], inplace=True, axis=1)
for column in ["corrPercentile", "mmcPercentile", "fncPercentile"]:
round_leaderboard[column] = round_leaderboard[column]*100
if pct_format:
round_leaderboard[column] = pd.Series(["{:.1f}%".format(value) for value in round_leaderboard[column].values],index=round_leaderboard[column].index)
round_leaderboard = round_leaderboard.reindex(sorted(round_leaderboard.columns), axis=1)
return round_leaderboard
def get_percentile_ranks_for_rounds(self, model_names, round_min, round_max=None, by="correlation", colour_if_resolved=True, pct_format=True, disable_tqdm=False):
assert(by in ["mmc", "fnc", "corr"])
round_max = round_max or self.get_current_scored_round()
percentile_ranks_by_round = {}
for round_num in tqdm(range(round_min, round_max+1), disable=disable_tqdm):
leaderboard_for_round = self.get_leaderboard_for_round(round_num, model_names=model_names, pct_format=False, disable_tqdm=True)
percentiles_for_round = {}
average_percentiles_for_round = []
for model_name in model_names:
if model_name not in leaderboard_for_round.index:
percentiles_for_round[model_name] = ""
else:
average_percentiles_for_round.append(leaderboard_for_round.loc[model_name]["{}Percentile".format(by)])
percentiles_for_round[model_name] = leaderboard_for_round.loc[model_name]["{}Percentile".format(by)]
# capital A so that we handle the edge case of the user "average" :)
percentiles_for_round["Average"] = np.mean(average_percentiles_for_round)
if pct_format:
for model_name in model_names+["Average"]:
if percentiles_for_round[model_name]!="":
percentiles_for_round[model_name] = "{:.1f}%".format(percentiles_for_round[model_name])
percentile_ranks_by_round[round_num] = percentiles_for_round
percentiles_df = pd.DataFrame.from_dict(percentile_ranks_by_round, orient="index")
if colour_if_resolved:
def green_if_resolved_else_orange(row):
colour = "green" if self.is_round_resolved(row.name) else "orange"
return ["color: {}".format(colour) for _ in range(len(row))]
percentiles_df = percentiles_df.style.apply(green_if_resolved_else_orange, axis=1)
return percentiles_df
This allows you to get a pandas DataFrame of the percentile ranks of all your models for CORR like so:
anapi = AdvancedNumerAPI()
model_names = ["ml_is_lyf"]+["ml_is_lyf_{}".format(module_num) for module_num in range(1,14)] # list of your model names
round_min = 251 # round you want the table to start at
anapi.get_percentile_ranks_for_rounds(model_names, round_min, by="corr")
You can also get them for MMC like so:
anapi.get_percentile_ranks_for_rounds(model_names, round_min, by="mmc")
For rounds that haven’t resolved the percentiles are just the latest in your daily scores. These queries each only take a few seconds to run, so its very quick to re-run it if you want to see your daily scores.
The percentiles generated in the tables above are exactly the same as if you go and hover over your CORR/MMC for each round on your profile. But I think its nicer to have them all-together as its easier to compare your models.
I also included an average column, which is just the average of your percentile ranks in the round, as I think this gives you a better picture of how you’re doing in the tournament.
I think it is also nice to be able to look at the performance of your models by round, which you can do like so:
anapi.get_leaderboard_for_round(256, model_names=model_names)
By default, they are sorted by correlation, but you can use the “by” argument to sort by any column, e.g.
anapi.get_leaderboard_for_round(256, by="mmc", model_names=model_names)
You can also get the leaderboard for all participants in the round if you don’t specify any model names, this takes over 40 minutes to build though as you have to make a query for every participant in the round.
anapi.get_leaderboard_for_round(256)
Here’s a Google Colab notebook with the code ready to go with the examples above:
Let me know if you find any bugs in the code or have any questions on it, hope this is useful to others 