\begin{table*}[htp]
\resizebox{\textwidth}{!}{
\begin{tabular}{lrrrrrrr}
\toprule
Model/adapter & EM & Observed inference time (s) & # eval & # train & truncated & # prompt tokens & # output tokens \\
\midrule
Gemma 2 (27B) & 0.9666666666666667 & 1.5516301684909397 & 90.0 & 5.0 &  & 430.06666666666666 & 1.0 \\
Gemma 2 (9B) & 0.9111111111111111 & 0.539949173397488 & 90.0 & 5.0 &  & 430.06666666666666 & 1.0 \\
Phi-3 (14B) & 0.9666666666666667 & 0.447341611650255 & 90.0 & 5.0 &  & 488.6111111111111 & 1.0 \\
Phi-3 (7B) & 0.9666666666666667 & 0.5761785348256429 & 90.0 & 5.0 &  & 422.65555555555557 & 1.0 \\
DBRX Instruct & 0.93 & 0.3899818444252014 & 100.0 & 5.0 &  & 415.79 & 1.0 \\
DeepSeek LLM Chat (67B) & 0.91 & 0.5037446546554566 & 100.0 & 5.0 &  & 453.51 & 1.0 \\
Falcon (40B) & 0.79 & 1.2640673851966857 & 100.0 & 5.0 &  & 449.19 & 1.0 \\
Falcon (7B) & 0.39 & 0.493820219039917 & 100.0 & 5.0 &  & 449.19 & 1.0 \\
Gemma (7B) & 0.87 & 0.29293906927108765 & 100.0 & 5.0 &  & 430.2 & 1.0 \\
Llama 2 (13B) & 0.84 & 0.3795260763168335 & 100.0 & 5.0 &  & 479.81 & 1.0 \\
Llama 2 (70B) & 0.92 & 0.5028035902976989 & 100.0 & 5.0 &  & 479.81 & 1.0 \\
Llama 2 (7B) & 0.63 & 0.3407774615287781 & 100.0 & 5.0 &  & 479.81 & 1.0 \\
Llama 3 (70B) & 0.94 & 0.4027411961555481 & 100.0 & 5.0 &  & 415.79 & 1.0 \\
Llama 3 (8B) & 0.88 & 0.29950841665267947 & 100.0 & 5.0 &  & 415.79 & 1.0 \\
Llama 3.1 Instruct Turbo (405B) & 0.9666666666666667 & 0.4321868922975328 & 90.0 & 5.0 &  & 422.65555555555557 & 1.0 \\
Llama 3.1 Instruct Turbo (70B) & 0.9222222222222223 & 1.5527344465255737 & 90.0 & 5.0 &  & 422.65555555555557 & 1.0 \\
Llama 3.1 Instruct Turbo (8B) & 0.8666666666666667 & 0.5649965524673461 & 90.0 & 5.0 &  & 422.65555555555557 & 1.0 \\
LLaMA (65B) & 0.89 & 2.6358231592178343 & 100.0 & 5.0 &  & 479.81 & 1.0 \\
Mistral Instruct v0.3 (7B) & 0.79 & 0.48650413513183594 & 100.0 & 5.0 &  & 482.19 & 1.0 \\
Mistral v0.1 (7B) & 0.85 & 0.2914116883277893 & 100.0 & 5.0 &  & 473.19 & 1.0 \\
Mixtral (8x22B) & 0.95 & 0.3477613878250122 & 100.0 & 5.0 &  & 473.19 & 1.0 \\
Mixtral (8x7B 32K seqlen) & 0.93 & 0.3552170705795288 & 100.0 & 5.0 &  & 473.19 & 1.0 \\
OLMo (7B) & 0.26 & 0.31185237407684324 & 100.0 & 5.0 &  & 444.08 & 1.0 \\
Phi-2 & 0.78 & 0.2921306538581848 & 100.0 & 5.0 &  & 433.12 & 1.0 \\
Qwen1.5 Chat (110B) & 0.87 & 0.22928016662597656 & 100.0 & 5.0 &  & 428.16 & 1.0 \\
Qwen1.5 (14B) & 0.87 & 0.31888857364654544 & 100.0 & 5.0 &  & 421.16 & 1.0 \\
Qwen1.5 (32B) & 0.91 & 0.3429260540008545 & 100.0 & 5.0 &  & 421.16 & 1.0 \\
Qwen1.5 (72B) & 0.94 & 0.3515354657173157 & 100.0 & 5.0 &  & 421.16 & 1.0 \\
Qwen1.5 (7B) & 0.84 & 0.28910151720046995 & 100.0 & 5.0 &  & 421.16 & 1.0 \\
Qwen2 Instruct (72B) & 0.94 & 0.19451653003692626 & 100.0 & 5.0 &  & 428.16 & 1.0 \\
Arctic Instruct & 0.88 & 0.31651313543319703 & 100.0 & 5.0 &  & 488.81 & 1.0 \\
Yi (34B) & 0.91 & 0.5111375832557679 & 100.0 & 5.0 &  & 458.53 & 1.0 \\
Yi (6B) & 0.87 & 0.33008920669555664 & 100.0 & 5.0 &  & 458.53 & 1.0 \\
Jurassic-2 Jumbo (178B) & 0.83 & 0.9200221586227417 & 100.0 & 5.0 &  & 321.12 & 1.0 \\
Jurassic-2 Grande (17B) & 0.77 & 0.5652387118339539 & 100.0 & 5.0 &  & 321.12 & 1.0 \\
Jamba Instruct & 0.9333333333333333 & 0.2732248650656806 & 90.0 & 5.0 &  & 328.1777777777778 &  \\
Luminous Base (13B) & 0.23 & 0.6281867814064026 & 100.0 & 5.0 &  & 440.72 & 1.0 \\
Luminous Extended (30B) & 0.2 & 0.7145060086250306 & 100.0 & 5.0 &  & 440.72 & 1.0 \\
Luminous Supreme (70B) & 0.5 & 0.885556857585907 & 100.0 & 5.0 &  & 440.72 & 1.0 \\
Claude v1.3 & 0.93 & 1.2937625908851624 & 100.0 & 5.0 &  & 520.25 & 1.0 \\
Claude Instant 1.2 & 0.9 & 0.6128408885002137 & 100.0 & 5.0 &  & 520.25 & 1.0 \\
Claude 2.0 & 0.9 & 1.6507928562164307 & 100.0 & 5.0 &  & 520.25 & 1.0 \\
Claude 2.1 & 0.92 & 2.37366126537323 & 100.0 & 5.0 &  & 520.25 & 1.0 \\
Claude 3 Haiku (20240307) & 0.95 & 0.6937756729125977 & 100.0 & 5.0 &  & 455.25 & 1.0 \\
Claude 3 Sonnet (20240229) & 0.94 & 1.2445136380195618 & 100.0 & 5.0 &  & 455.25 & 1.0 \\
Claude 3 Opus (20240229) & 0.96 & 4.002964313030243 & 100.0 & 5.0 &  & 455.25 & 1.0 \\
Claude 3.5 Sonnet (20240620) & 0.96 & 0.9727654385566712 & 100.0 & 5.0 &  & 464.25 & 1.0 \\
Command & 0.88 & 0.9560029816627502 & 100.0 & 5.0 &  & 444.07 & 1.0 \\
Command Light & 0.57 & 0.7429711484909057 & 100.0 & 5.0 &  & 444.07 & 1.0 \\
Command R & 0.82 & 0.18277841329574585 & 100.0 & 5.0 &  & 452.23 &  \\
Command R Plus & 0.89 & 0.29705020904541013 & 100.0 & 5.0 &  & 452.23 &  \\
Gemini 1.0 Pro (002) & 0.81 & 0.4082685565948486 & 100.0 & 5.0 &  & 438.2 &  \\
Gemini 1.5 Pro (001) & 0.93 & 0.6769772005081177 & 100.0 & 5.0 &  & 438.2 &  \\
Gemini 1.5 Flash (001) & 0.93 & 0.5247626876831055 & 100.0 & 5.0 &  & 438.2 &  \\
PaLM-2 (Bison) & 0.87 & 1.1012366461753844 & 100.0 & 5.0 &  & 477.35 & 1.0 \\
PaLM-2 (Unicorn) & 0.96 & 1.2385127997398377 & 100.0 & 5.0 &  & 477.35 & 1.0 \\
Yi Large (Preview) & 0.85 & 0.6958462524414063 & 100.0 & 5.0 &  & 471.53 & 1.0 \\
Mistral Small (2402) & 0.89 & 1.3467011404037477 & 100.0 & 5.0 &  & 473.19 & 1.0 \\
Mistral Medium (2312) & 0.91 & 2.8468315839767455 & 100.0 & 5.0 &  & 473.19 & 0.95 \\
Mistral Large (2402) & 0.92 & 1.6332264852523803 & 100.0 & 5.0 &  & 473.19 & 1.0 \\
Mistral NeMo (2402) & 0.9 & 0.8731695281134711 & 90.0 & 5.0 &  & 430.73333333333335 & 1.0 \\
GPT-3.5 (text-davinci-003) & 0.83 & 0.19882412433624266 & 100.0 & 5.0 &  & 433.12 & 1.0 \\
GPT-3.5 (text-davinci-002) & 0.84 & 0.18106817960739136 & 100.0 & 5.0 &  & 433.12 & 1.0 \\
GPT-3.5 Turbo (0613) & 0.88 & 0.17667593240737914 & 100.0 & 5.0 &  & 415.79 & 1.0 \\
GPT-4 Turbo (1106 preview) & 0.96 & 0.39724321842193605 & 100.0 & 5.0 &  & 415.79 & 1.0 \\
GPT-4 (0613) & 0.95 & 0.43441893100738527 & 100.0 & 5.0 &  & 415.79 & 1.0 \\
GPT-4 Turbo (2024-04-09) & 0.96 & 0.557673556804657 & 100.0 & 5.0 &  & 422.79 & 1.0 \\
GPT-4o (2024-05-13) & 0.95 & 0.35311189413070676 & 100.0 & 5.0 &  & 421.71 & 1.0 \\
GPT-4o mini (2024-07-18) & 0.9222222222222223 & 0.2981134255727132 & 90.0 & 5.0 &  & 421.5888888888889 & 1.0 \\
Palmyra X V2 (33B) & 0.91 & 0.5420086312294007 & 100.0 & 5.0 &  & 433.12 & 1.0 \\
Palmyra X V3 (72B) & 0.96 & 0.6037013912200928 & 100.0 & 5.0 &  & 433.12 & 1.0 \\
\bottomrule
\end{tabular}}
\caption{Results for mmlu_subject:us_foreign_policy (mmlu)}
\label{fig:mmlu_subject:us_foreign_policy (mmlu)}
\end{table*}