\begin{table*}[htp]
\resizebox{\textwidth}{!}{
\begin{tabular}{lrrrrrrr}
\toprule
Model/adapter & EM & Observed inference time (s) & # eval & # train & truncated & # prompt tokens & # output tokens \\
\midrule
Phi-3 (14B) & 0.48 & 0.3811429190635681 & 100.0 & 5.0 &  & 406.65 & 1.0 \\
Phi-3 (7B) & 0.44 & 0.5047230005264283 & 100.0 & 5.0 &  & 373.44 & 1.0 \\
DBRX Instruct & 0.34 & 0.4316913342475891 & 100.0 & 5.0 &  & 366.44 & 1.0 \\
DeepSeek LLM Chat (67B) & 0.44 & 0.4850481009483337 & 100.0 & 5.0 &  & 382.07 & 1.0 \\
Falcon (40B) & 0.31 & 1.445729100704193 & 100.0 & 5.0 &  & 396.92 & 1.0 \\
Falcon (7B) & 0.26 & 0.4339600944519043 & 100.0 & 5.0 &  & 396.92 & 1.0 \\
Gemma 2 Instruct (27B) & 0.44 & 0.3288047504425049 & 100.0 & 5.0 &  & 383.97 & 1.0 \\
Gemma 2 Instruct (9B) & 0.42 & 0.2989117908477783 & 100.0 & 5.0 &  & 383.97 & 1.0 \\
Gemma (7B) & 0.28 & 0.27131984949111937 & 100.0 & 5.0 &  & 375.97 & 1.0 \\
Llama 2 (13B) & 0.28 & 0.37155511140823366 & 100.0 & 5.0 &  & 397.65 & 1.0 \\
Llama 2 (70B) & 0.31 & 0.46531004428863526 & 100.0 & 5.0 &  & 397.65 & 1.0 \\
Llama 2 (7B) & 0.3 & 0.314219696521759 & 100.0 & 5.0 &  & 397.65 & 1.0 \\
Llama 3 (70B) & 0.43 & 0.3868687057495117 & 100.0 & 5.0 &  & 366.43 & 1.0 \\
Llama 3 (8B) & 0.33 & 0.30905162572860717 & 100.0 & 5.0 &  & 366.43 & 1.0 \\
Llama 3.1 Instruct Turbo (405B) & 0.7 & 0.4640246653556824 & 100.0 & 5.0 &  & 373.43 & 1.0 \\
Llama 3.1 Instruct Turbo (70B) & 0.55 & 45.250502264499666 & 100.0 & 5.0 &  & 373.43 & 1.0 \\
Llama 3.1 Instruct Turbo (8B) & 0.26 & 0.28381933450698854 & 100.0 & 5.0 &  & 373.43 & 1.0 \\
LLaMA (65B) & 0.34 & 4.3549476957321165 & 100.0 & 5.0 &  & 397.65 & 1.0 \\
Mistral Instruct v0.3 (7B) & 0.27 & 0.32117165088653565 & 100.0 & 5.0 &  & 411.44 & 1.0 \\
Mistral v0.1 (7B) & 0.31 & 0.28625457048416136 & 100.0 & 5.0 &  & 402.44 & 1.0 \\
Mixtral (8x22B) & 0.48 & 0.31304038524627686 & 100.0 & 5.0 &  & 402.44 & 1.0 \\
Mixtral (8x7B 32K seqlen) & 0.38 & 0.3562413024902344 & 100.0 & 5.0 &  & 402.44 & 1.0 \\
OLMo (7B) & 0.26 & 0.309316143989563 & 100.0 & 5.0 &  & 358.76 & 1.0 \\
Phi-2 & 0.31 & 0.2925554180145264 & 100.0 & 5.0 &  & 371.38 & 1.0 \\
Qwen1.5 Chat (110B) & 0.57 & 0.22966567754745484 & 100.0 & 5.0 &  & 378.19 & 1.0 \\
Qwen1.5 (14B) & 0.4 & 0.28459527969360354 & 100.0 & 5.0 &  & 371.19 & 1.0 \\
Qwen1.5 (32B) & 0.4 & 0.33740817070007323 & 100.0 & 5.0 &  & 371.19 & 1.0 \\
Qwen1.5 (72B) & 0.44 & 0.3480935263633728 & 100.0 & 5.0 &  & 371.19 & 1.0 \\
Qwen1.5 (7B) & 0.39 & 0.28086970567703246 & 100.0 & 5.0 &  & 371.19 & 1.0 \\
Qwen2 Instruct (72B) & 0.67 & 0.3948828268051148 & 100.0 & 5.0 &  & 378.19 & 1.0 \\
Arctic Instruct & 0.31 & 0.2929831385612488 & 100.0 & 5.0 &  & 406.65 & 1.0 \\
Yi (34B) & 0.4 & 0.8523464965820312 & 100.0 & 5.0 &  & 383.67 & 1.0 \\
Yi (6B) & 0.3 & 0.3231867814064026 & 100.0 & 5.0 &  & 383.67 & 1.0 \\
Jurassic-2 Grande (17B) & 0.25 & 0.5552291941642761 & 100.0 & 5.0 &  & 324.26 & 1.0 \\
Jurassic-2 Jumbo (178B) & 0.25 & 0.6931413793563843 & 100.0 & 5.0 &  & 324.26 & 1.0 \\
Jamba Instruct & 0.36 & 0.27479029655456544 & 100.0 & 5.0 &  & 331.26 &  \\
Jamba 1.5 Mini & 0.33 & 0.783083221912384 & 100.0 & 5.0 &  & 397.58 &  \\
Jamba 1.5 Large & 0.53 & 0.9326767182350159 & 100.0 & 5.0 &  & 397.58 &  \\
Luminous Base (13B) & 0.22 & 0.6191291689872742 & 100.0 & 5.0 &  & 360.75 & 1.0 \\
Luminous Extended (30B) & 0.21 & 0.6899165153503418 & 100.0 & 5.0 &  & 360.75 & 1.0 \\
Luminous Supreme (70B) & 0.25 & 0.824960114955902 & 100.0 & 5.0 &  & 360.75 & 1.0 \\
Claude v1.3 & 0.35 & 1.2275683069229126 & 100.0 & 5.0 &  & 435.26 & 1.0 \\
Claude Instant 1.2 & 0.37 & 0.5904157018661499 & 100.0 & 5.0 &  & 435.26 & 1.0 \\
Claude 2.0 & 0.38 & 1.64589448928833 & 100.0 & 5.0 &  & 435.26 & 1.0 \\
Claude 2.1 & 0.4 & 2.043452892303467 & 100.0 & 5.0 &  & 435.26 & 1.0 \\
Claude 3 Haiku (20240307) & 0.42 & 0.6928385472297669 & 100.0 & 5.0 &  & 370.26 & 1.0 \\
Claude 3 Sonnet (20240229) & 0.39 & 1.2476251411437989 & 100.0 & 5.0 &  & 370.26 & 1.0 \\
Claude 3 Opus (20240229) & 0.64 & 4.182226595878601 & 100.0 & 5.0 &  & 370.26 & 1.0 \\
Claude 3.5 Sonnet (20240620) & 0.75 & 0.7789034700393677 & 100.0 & 5.0 &  & 379.26 & 1.0 \\
Command & 0.27 & 0.8208299922943115 & 100.0 & 5.0 &  & 372.75 & 1.0 \\
Command Light & 0.25 & 0.418369448184967 & 100.0 & 5.0 &  & 372.75 & 1.0 \\
Command R & 0.33 & 0.1620460057258606 & 100.0 & 5.0 &  & 397.66 &  \\
Command R Plus & 0.21 & 0.2603452730178833 & 100.0 & 5.0 &  & 397.66 &  \\
Gemini 1.0 Pro (002) & 0.27 & 0.39701672077178957 & 100.0 & 5.0 &  & 383.97 &  \\
Gemini 1.5 Pro (001) & 0.75 & 0.6589885497093201 & 100.0 & 5.0 &  & 383.97 &  \\
Gemini 1.5 Flash (001) & 0.58 & 0.595533971786499 & 100.0 & 5.0 &  & 383.97 &  \\
PaLM-2 (Bison) & 0.39 & 1.0166235256195069 & 100.0 & 5.0 &  & 360.7 & 1.0 \\
PaLM-2 (Unicorn) & 0.53 & 1.2228304648399353 & 100.0 & 5.0 &  & 360.7 & 1.0 \\
Yi Large (Preview) & 0.6 & 0.718058660030365 & 100.0 & 5.0 &  & 396.67 & 1.0 \\
Mistral Small (2402) & 0.26 & 1.3102962040901185 & 100.0 & 5.0 &  & 402.44 & 1.0 \\
Mistral Medium (2312) & 0.32 & 3.1207306599617004 & 100.0 & 5.0 &  & 402.44 & 0.99 \\
Mistral Large (2402) & 0.45 & 1.4797466564178468 & 100.0 & 5.0 &  & 402.44 & 1.0 \\
Mistral Large 2 (2407) & 0.7 & 0.7317730689048767 & 100.0 & 5.0 &  & 412.44 & 1.0 \\
Mistral NeMo (2402) & 0.29 & 0.6429726719856262 & 100.0 & 5.0 &  & 377.89 & 1.0 \\
GPT-3.5 (text-davinci-003) & 0.3 & 0.19926663398742675 & 100.0 & 5.0 &  & 371.38 & 1.0 \\
GPT-3.5 (text-davinci-002) & 0.26 & 0.17565438747406006 & 100.0 & 5.0 &  & 371.38 & 1.0 \\
GPT-3.5 Turbo (0613) & 0.38 & 0.17670444011688233 & 100.0 & 5.0 &  & 366.44 & 1.0 \\
GPT-4 Turbo (1106 preview) & 0.53 & 0.42504594564437864 & 100.0 & 5.0 &  & 366.44 & 1.0 \\
GPT-4 (0613) & 0.63 & 0.39332568168640136 & 100.0 & 5.0 &  & 366.44 & 1.0 \\
GPT-4 Turbo (2024-04-09) & 0.56 & 0.539907853603363 & 100.0 & 5.0 &  & 373.44 & 1.0 \\
GPT-4o (2024-05-13) & 0.64 & 0.39040852546691895 & 100.0 & 5.0 &  & 374.53 & 1.0 \\
GPT-4o mini (2024-07-18) & 0.42 & 0.29186195611953736 & 100.0 & 5.0 &  & 374.53 & 1.0 \\
Palmyra X V2 (33B) & 0.37 & 0.4616232466697693 & 100.0 & 5.0 &  & 371.38 & 1.0 \\
Palmyra X V3 (72B) & 0.53 & 0.6195793676376343 & 100.0 & 5.0 &  & 371.38 & 1.0 \\
Palmyra X-004 & 0.75 & 0.7220739269256592 & 100.0 & 5.0 &  & 373.43 & 1.0 \\
\bottomrule
\end{tabular}}
\caption{Results for mmlu_subject:abstract_algebra (mmlu)}
\label{fig:mmlu_subject:abstract_algebra (mmlu)}
\end{table*}