\begin{table*}[htp]
\resizebox{\textwidth}{!}{
\begin{tabular}{lrrrrrrr}
\toprule
Model/adapter & EM & Observed inference time (s) & # eval & # train & truncated & # prompt tokens & # output tokens \\
\midrule
Phi-3 (14B) & 0.538 & 6.059593142509461 & 1000.0 & 4.0 &  & 283.825 & 1.04 \\
Phi-3 (7B) & 0.471 & 6.37646821808815 & 1000.0 & 4.0 &  & 242.86 & 1.538 \\
DBRX Instruct & 0.234 & 0.6379435393810272 & 1000.0 & 4.0 &  & 298.86 & 1.0 \\
DeepSeek LLM Chat (67B) & 0.544 & 0.5235648410320282 & 1000.0 & 4.0 &  & 320.062 & 1.0 \\
Falcon (40B) & 0.597 & 1.3327487347126008 & 1000.0 & 4.0 &  & 251.171 & 1.0 \\
Falcon (7B) & 0.485 & 0.5580307400226593 & 1000.0 & 4.0 &  & 251.171 & 1.0 \\
Gemma 2 Instruct (27B) & 0.537 & 0.3543481981754303 & 1000.0 & 4.0 &  & 251.362 & 1.0 \\
Gemma 2 Instruct (9B) & 0.406 & 0.32026377773284914 & 1000.0 & 4.0 &  & 251.362 & 1.0 \\
Gemma (7B) & 0.536 & 0.305184095621109 & 1000.0 & 4.0 &  & 245.362 & 1.0 \\
Llama 2 (13B) & 0.618 & 0.3311143760681152 & 1000.0 & 4.0 &  & 276.825 & 1.0 \\
Llama 2 (70B) & 0.62 & 0.5295624468326569 & 1000.0 & 4.0 &  & 276.825 & 1.0 \\
Llama 2 (7B) & 0.566 & 0.3441107220649719 & 1000.0 & 4.0 &  & 276.825 & 1.0 \\
Llama 3 (70B) & 0.705 & 0.42025949263572693 & 1000.0 & 4.0 &  & 237.86 & 1.0 \\
Llama 3 (8B) & 0.601 & 0.32164009165763857 & 1000.0 & 4.0 &  & 237.86 & 1.0 \\
Llama 3.1 Instruct Turbo (405B) & 0.433 & 0.4916829204559326 & 1000.0 & 4.0 &  & 242.86 & 2.389 \\
Llama 3.1 Instruct Turbo (70B) & 0.439 & 3.4859169404506685 & 1000.0 & 4.0 &  & 242.86 & 2.011 \\
Llama 3.1 Instruct Turbo (8B) & 0.289 & 0.4709072856903076 & 1000.0 & 4.0 &  & 242.86 & 10.886 \\
Llama 3.2 Vision Instruct Turbo (11B) & 0.436 & 0.3004174392223358 & 1000.0 & 4.0 &  & 242.86 &  \\
Llama 3.2 Vision Instruct Turbo (90B) & 0.438 & 0.2998896398544312 & 1000.0 & 4.0 &  & 242.86 &  \\
LLaMA (65B) & 0.561 & 6.263856258392334 & 1000.0 & 4.0 &  & 276.825 & 1.0 \\
Mistral Instruct v0.3 (7B) & 0.264 & 0.38989536023139953 & 1000.0 & 4.0 &  & 293.575 & 9.31 \\
Mistral v0.1 (7B) & 0.53 & 0.2982086777687073 & 1000.0 & 4.0 &  & 274.575 & 1.0 \\
Mixtral (8x22B) & 0.555 & 0.3721002333164215 & 1000.0 & 4.0 &  & 274.575 & 1.0 \\
Mixtral (8x7B 32K seqlen) & 0.612 & 0.37267246675491333 & 1000.0 & 4.0 &  & 274.575 & 1.0 \\
OLMo (7B) & 0.54 & 0.36936369013786313 & 1000.0 & 4.0 &  & 250.447 & 1.0 \\
Phi-2 & 0.466 & 0.2683418924808502 & 1000.0 & 4.0 &  & 248.054 & 1.0 \\
Qwen1.5 Chat (110B) & 0.423 & 0.2714889543056488 & 1000.0 & 4.0 &  & 254.879 & 2.101 \\
Qwen1.5 (14B) & 0.609 & 0.3323783149719238 & 1000.0 & 4.0 &  & 239.879 & 1.0 \\
Qwen1.5 (32B) & 0.576 & 0.3927197141647339 & 1000.0 & 4.0 &  & 239.879 & 1.0 \\
Qwen1.5 (72B) & 0.612 & 0.42560584807395935 & 1000.0 & 4.0 &  & 239.879 & 1.0 \\
Qwen1.5 (7B) & 0.633 & 0.2975546569824219 & 1000.0 & 4.0 &  & 239.879 & 1.0 \\
Qwen2 Instruct (72B) & 0.653 & 0.2334742341041565 & 1000.0 & 4.0 &  & 254.879 & 2.0 \\
Qwen2.5 Instruct Turbo (72B) & 0.631 & 0.3057863731384277 & 1000.0 & 4.0 &  & 249.879 & 2.0 \\
Qwen2.5 Instruct Turbo (7B) & 0.607 & 0.18287641167640686 & 1000.0 & 4.0 &  & 249.879 & 2.0 \\
Arctic Instruct & 0.512 & 0.2918559632301331 & 1000.0 & 4.0 &  & 293.825 & 2.138 \\
Yi (34B) & 0.583 & 0.6558779480457306 & 1000.0 & 4.0 &  & 260.265 & 1.0 \\
Yi (6B) & 0.506 & 0.4091528770923615 & 1000.0 & 4.0 &  & 260.265 & 1.0 \\
Jurassic-2 Grande (17B) & 0.375 & 0.5345145518779755 & 1000.0 & 4.0 &  & 183.837 & 2.057 \\
Jurassic-2 Jumbo (178B) & 0.425 & 1.125831830739975 & 1000.0 & 4.0 &  & 183.837 & 2.043 \\
Jamba Instruct & 0.304 & 0.4825632996559143 & 1000.0 & 4.0 &  & 188.837 &  \\
Jamba 1.5 Mini & 0.412 & 0.806744626045227 & 1000.0 & 4.0 &  & 260.1 &  \\
Jamba 1.5 Large & 0.51 & 0.9868728687763214 & 1000.0 & 4.0 &  & 260.1 &  \\
Luminous Base (13B) & 0.601 & 0.7165834536552429 & 1000.0 & 4.0 &  & 250.418 & 1.169 \\
Luminous Extended (30B) & 0.54 & 0.8127039783000946 & 1000.0 & 4.0 &  & 250.418 & 1.28 \\
Luminous Supreme (70B) & 0.59 & 0.8659334790706634 & 1000.0 & 4.0 &  & 250.418 & 1.031 \\
Claude v1.3 & 0.449 & 2.0580357022285463 & 1000.0 & 4.0 &  & 323.505 & 1.0 \\
Claude Instant 1.2 & 0.348 & 0.6397523488998413 & 1000.0 & 4.0 &  & 323.505 & 2.219 \\
Claude 2.0 & 0.392 & 2.1553329265117647 & 1000.0 & 4.0 &  & 323.505 & 11.058 \\
Claude 2.1 & 0.459 & 2.229681488752365 & 1000.0 & 4.0 &  & 323.505 & 1.522 \\
Claude 3 Haiku (20240307) & 0.034 & 0.8477493212223053 & 1000.0 & 4.0 &  & 257.505 & 28.352 \\
Claude 3 Sonnet (20240229) & 0.524 & 0.9930782251358032 & 1000.0 & 4.0 &  & 257.505 & 1.627 \\
Claude 3 Opus (20240229) & 0.608 & 2.097750175476074 & 1000.0 & 4.0 &  & 257.505 & 1.0 \\
Claude 3.5 Sonnet (20240620) & 0.575 & 0.7966864895820618 & 1000.0 & 4.0 &  & 266.505 & 1.0 \\
Command & 0.409 & 1.0322861764431 & 1000.0 & 4.0 &  & 246.658 & 1.113 \\
Command Light & 0.567 & 0.47469998264312746 & 1000.0 & 4.0 &  & 246.658 & 3.328 \\
Command R & 0.416 & 0.1533742961883545 & 1000.0 & 4.0 &  & 268.376 &  \\
Command R Plus & 0.603 & 0.19508087706565858 & 1000.0 & 4.0 &  & 268.376 &  \\
Gemini 1.0 Pro (002) & 0.454 & 0.44708213448524475 & 1000.0 & 4.0 &  & 261.362 &  \\
Gemini 1.5 Pro (001) & 0.675 & 0.5774267916679382 & 1000.0 & 4.0 &  & 261.362 &  \\
Gemini 1.5 Flash (001) & 0.543 & 0.40872780966758726 & 1000.0 & 4.0 &  & 261.362 &  \\
PaLM-2 (Bison) & 0.484 & 0.6460402452945709 & 1000.0 & 4.0 &  & 358.853 & 1.019 \\
PaLM-2 (Unicorn) & 0.472 & 1.0323096747398377 & 1000.0 & 4.0 &  & 358.853 & 1.0 \\
Yi Large (Preview) & 0.591 & 0.854616739988327 & 1000.0 & 4.0 &  & 277.265 & 1.159 \\
Mistral Small (2402) & 0.099 & 0.7719580962657928 & 1000.0 & 4.0 &  & 344.575 & 25.624 \\
Mistral Medium (2312) & 0.066 & 2.6945335958004 & 1000.0 & 4.0 &  & 344.575 & 25.305 \\
Mistral Large (2402) & 0.461 & 1.556276372909546 & 1000.0 & 4.0 &  & 344.575 & 1.005 \\
Mistral Large 2 (2407) & 0.423 & 0.4852830271720886 & 1000.0 & 4.0 &  & 283.575 & 4.004 \\
Mistral NeMo (2402) & 0.338 & 0.8677918517589569 & 1000.0 & 4.0 &  & 245.094 & 7.619 \\
GPT-3.5 (text-davinci-003) & 0.598 & 0.2245520987510681 & 1000.0 & 4.0 &  & 248.054 & 1.443 \\
GPT-3.5 (text-davinci-002) & 0.57 & 0.18166398239135742 & 1000.0 & 4.0 &  & 248.054 & 1.238 \\
GPT-3.5 Turbo (0613) & 0.59 & 0.18215485739707946 & 1000.0 & 4.0 &  & 298.86 & 1.483 \\
GPT-4 Turbo (1106 preview) & 0.54 & 0.4680407977104187 & 1000.0 & 4.0 &  & 298.86 & 1.035 \\
GPT-4 (0613) & 0.63 & 0.48205151438713073 & 1000.0 & 4.0 &  & 298.86 & 1.0 \\
GPT-4 Turbo (2024-04-09) & 0.656 & 0.513949007987976 & 1000.0 & 4.0 &  & 252.86 & 1.0 \\
GPT-4o (2024-05-13) & 0.601 & 0.41031726384162903 & 1000.0 & 4.0 &  & 252.946 & 1.0 \\
GPT-4o mini (2024-07-18) & 0.457 & 0.33676894307136535 & 1000.0 & 4.0 &  & 242.946 & 1.0 \\
Palmyra X V2 (33B) & 0.511 & 0.42508620262145996 & 1000.0 & 4.0 &  & 248.054 & 1.043 \\
Palmyra X V3 (72B) & 0.607 & 0.6682215228080749 & 1000.0 & 4.0 &  & 248.054 & 1.0 \\
Palmyra-X-004 & 0.579 & 0.5104092361927033 & 1000.0 & 4.0 &  & 247.86 & 1.442 \\
\bottomrule
\end{tabular}}
\caption{Results for legalbench_subset:international_citizenship_questions (legalbench)}
\label{fig:legalbench_subset:international_citizenship_questions (legalbench)}
\end{table*}