{
  "title": "General information",
  "header": [
    {
      "value": "Model",
      "markdown": false,
      "metadata": {}
    },
    {
      "value": "CzechBankQA - # eval",
      "description": "The CzechBankQA\n\n# eval: Number of evaluation instances.",
      "markdown": false,
      "metadata": {
        "metric": "# eval",
        "run_group": "CzechBankQA"
      }
    },
    {
      "value": "CzechBankQA - # train",
      "description": "The CzechBankQA\n\n# train: Number of training instances (e.g., in-context examples).",
      "markdown": false,
      "metadata": {
        "metric": "# train",
        "run_group": "CzechBankQA"
      }
    },
    {
      "value": "CzechBankQA - truncated",
      "description": "The CzechBankQA\n\ntruncated: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).",
      "markdown": false,
      "metadata": {
        "metric": "truncated",
        "run_group": "CzechBankQA"
      }
    },
    {
      "value": "CzechBankQA - # prompt tokens",
      "description": "The CzechBankQA\n\n# prompt tokens: Number of tokens in the prompt.",
      "markdown": false,
      "metadata": {
        "metric": "# prompt tokens",
        "run_group": "CzechBankQA"
      }
    },
    {
      "value": "CzechBankQA - # output tokens",
      "description": "The CzechBankQA\n\n# output tokens: Actual number of output tokens.",
      "markdown": false,
      "metadata": {
        "metric": "# output tokens",
        "run_group": "CzechBankQA"
      }
    }
  ],
  "rows": [
    [
      {
        "value": "Llama 3.1 Instruct Turbo (405B)",
        "description": "",
        "markdown": false
      },
      {
        "value": 102.0,
        "description": "min=102, mean=102, max=102, sum=102 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=meta_llama-3.1-405b-instruct-turbo"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=meta_llama-3.1-405b-instruct-turbo"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=meta_llama-3.1-405b-instruct-turbo"
        ]
      },
      {
        "value": 973.8137254901961,
        "description": "min=973.814, mean=973.814, max=973.814, sum=973.814 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=meta_llama-3.1-405b-instruct-turbo"
        ]
      },
      {
        "value": 59.81372549019608,
        "description": "min=59.814, mean=59.814, max=59.814, sum=59.814 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=meta_llama-3.1-405b-instruct-turbo"
        ]
      }
    ],
    [
      {
        "value": "Llama 3.1 Instruct Turbo (70B)",
        "description": "",
        "markdown": false
      },
      {
        "value": 102.0,
        "description": "min=102, mean=102, max=102, sum=102 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=meta_llama-3.1-70b-instruct-turbo"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=meta_llama-3.1-70b-instruct-turbo"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=meta_llama-3.1-70b-instruct-turbo"
        ]
      },
      {
        "value": 973.8137254901961,
        "description": "min=973.814, mean=973.814, max=973.814, sum=973.814 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=meta_llama-3.1-70b-instruct-turbo"
        ]
      },
      {
        "value": 51.990196078431374,
        "description": "min=51.99, mean=51.99, max=51.99, sum=51.99 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=meta_llama-3.1-70b-instruct-turbo"
        ]
      }
    ],
    [
      {
        "value": "Llama 3.1 Instruct Turbo (8B)",
        "description": "",
        "markdown": false
      },
      {
        "value": 102.0,
        "description": "min=102, mean=102, max=102, sum=102 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=meta_llama-3.1-8b-instruct-turbo"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=meta_llama-3.1-8b-instruct-turbo"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=meta_llama-3.1-8b-instruct-turbo"
        ]
      },
      {
        "value": 973.8137254901961,
        "description": "min=973.814, mean=973.814, max=973.814, sum=973.814 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=meta_llama-3.1-8b-instruct-turbo"
        ]
      },
      {
        "value": 54.72549019607843,
        "description": "min=54.725, mean=54.725, max=54.725, sum=54.725 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=meta_llama-3.1-8b-instruct-turbo"
        ]
      }
    ],
    [
      {
        "value": "Mistral Instruct v0.3 (7B)",
        "description": "",
        "markdown": false
      },
      {
        "value": 102.0,
        "description": "min=102, mean=102, max=102, sum=102 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=mistralai_mistral-7b-instruct-v0.3"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=mistralai_mistral-7b-instruct-v0.3"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=mistralai_mistral-7b-instruct-v0.3"
        ]
      },
      {
        "value": 1205.7254901960785,
        "description": "min=1205.725, mean=1205.725, max=1205.725, sum=1205.725 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=mistralai_mistral-7b-instruct-v0.3"
        ]
      },
      {
        "value": 65.88235294117646,
        "description": "min=65.882, mean=65.882, max=65.882, sum=65.882 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=mistralai_mistral-7b-instruct-v0.3"
        ]
      }
    ],
    [
      {
        "value": "Mixtral Instruct (8x22B)",
        "description": "",
        "markdown": false
      },
      {
        "value": 102.0,
        "description": "min=102, mean=102, max=102, sum=102 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=mistralai_mixtral-8x22b-instruct-v0.1"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=mistralai_mixtral-8x22b-instruct-v0.1"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=mistralai_mixtral-8x22b-instruct-v0.1"
        ]
      },
      {
        "value": 1205.7254901960785,
        "description": "min=1205.725, mean=1205.725, max=1205.725, sum=1205.725 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=mistralai_mixtral-8x22b-instruct-v0.1"
        ]
      },
      {
        "value": 45.627450980392155,
        "description": "min=45.627, mean=45.627, max=45.627, sum=45.627 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=mistralai_mixtral-8x22b-instruct-v0.1"
        ]
      }
    ],
    [
      {
        "value": "Mixtral Instruct (8x7B)",
        "description": "",
        "markdown": false
      },
      {
        "value": 102.0,
        "description": "min=102, mean=102, max=102, sum=102 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=mistralai_mixtral-8x7b-instruct-v0.1"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=mistralai_mixtral-8x7b-instruct-v0.1"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=mistralai_mixtral-8x7b-instruct-v0.1"
        ]
      },
      {
        "value": 1205.7254901960785,
        "description": "min=1205.725, mean=1205.725, max=1205.725, sum=1205.725 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=mistralai_mixtral-8x7b-instruct-v0.1"
        ]
      },
      {
        "value": 68.6470588235294,
        "description": "min=68.647, mean=68.647, max=68.647, sum=68.647 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=mistralai_mixtral-8x7b-instruct-v0.1"
        ]
      }
    ],
    [
      {
        "value": "Qwen2 Instruct (72B)",
        "description": "",
        "markdown": false
      },
      {
        "value": 102.0,
        "description": "min=102, mean=102, max=102, sum=102 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=qwen_qwen2-72b-instruct"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=qwen_qwen2-72b-instruct"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=qwen_qwen2-72b-instruct"
        ]
      },
      {
        "value": 986.3529411764706,
        "description": "min=986.353, mean=986.353, max=986.353, sum=986.353 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=qwen_qwen2-72b-instruct"
        ]
      },
      {
        "value": 52.90196078431372,
        "description": "min=52.902, mean=52.902, max=52.902, sum=52.902 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=qwen_qwen2-72b-instruct"
        ]
      }
    ],
    [
      {
        "value": "Claude 3.5 Haiku (20241022)",
        "description": "",
        "markdown": false
      },
      {
        "value": 102.0,
        "description": "min=102, mean=102, max=102, sum=102 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=anthropic_claude-3-5-haiku-20241022"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=anthropic_claude-3-5-haiku-20241022"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=anthropic_claude-3-5-haiku-20241022"
        ]
      },
      {
        "value": 1095.235294117647,
        "description": "min=1095.235, mean=1095.235, max=1095.235, sum=1095.235 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=anthropic_claude-3-5-haiku-20241022"
        ]
      },
      {
        "value": 74.49019607843137,
        "description": "min=74.49, mean=74.49, max=74.49, sum=74.49 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=anthropic_claude-3-5-haiku-20241022"
        ]
      }
    ],
    [
      {
        "value": "Claude 3.5 Sonnet (20240620)",
        "description": "",
        "markdown": false
      },
      {
        "value": 102.0,
        "description": "min=102, mean=102, max=102, sum=102 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=anthropic_claude-3-5-sonnet-20240620"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=anthropic_claude-3-5-sonnet-20240620"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=anthropic_claude-3-5-sonnet-20240620"
        ]
      },
      {
        "value": 1095.235294117647,
        "description": "min=1095.235, mean=1095.235, max=1095.235, sum=1095.235 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=anthropic_claude-3-5-sonnet-20240620"
        ]
      },
      {
        "value": 68.90196078431373,
        "description": "min=68.902, mean=68.902, max=68.902, sum=68.902 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=anthropic_claude-3-5-sonnet-20240620"
        ]
      }
    ],
    [
      {
        "value": "Gemini 1.5 Pro (002)",
        "description": "",
        "markdown": false
      },
      {
        "value": 102.0,
        "description": "min=102, mean=102, max=102, sum=102 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=google_gemini-1.5-pro-002"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=google_gemini-1.5-pro-002"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=google_gemini-1.5-pro-002"
        ]
      },
      {
        "value": 1090.2941176470588,
        "description": "min=1090.294, mean=1090.294, max=1090.294, sum=1090.294 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=google_gemini-1.5-pro-002"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=google_gemini-1.5-pro-002"
        ]
      }
    ],
    [
      {
        "value": "Gemini 1.5 Flash (002)",
        "description": "",
        "markdown": false
      },
      {
        "value": 102.0,
        "description": "min=102, mean=102, max=102, sum=102 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=google_gemini-1.5-flash-002"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=google_gemini-1.5-flash-002"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=google_gemini-1.5-flash-002"
        ]
      },
      {
        "value": 1090.2941176470588,
        "description": "min=1090.294, mean=1090.294, max=1090.294, sum=1090.294 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=google_gemini-1.5-flash-002"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=google_gemini-1.5-flash-002"
        ]
      }
    ],
    [
      {
        "value": "GPT-4o (2024-08-06)",
        "description": "",
        "markdown": false
      },
      {
        "value": 102.0,
        "description": "min=102, mean=102, max=102, sum=102 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=openai_gpt-4o-2024-08-06"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=openai_gpt-4o-2024-08-06"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=openai_gpt-4o-2024-08-06"
        ]
      },
      {
        "value": 981.8137254901961,
        "description": "min=981.814, mean=981.814, max=981.814, sum=981.814 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=openai_gpt-4o-2024-08-06"
        ]
      },
      {
        "value": 54.470588235294116,
        "description": "min=54.471, mean=54.471, max=54.471, sum=54.471 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=openai_gpt-4o-2024-08-06"
        ]
      }
    ],
    [
      {
        "value": "GPT-4o mini (2024-07-18)",
        "description": "",
        "markdown": false
      },
      {
        "value": 102.0,
        "description": "min=102, mean=102, max=102, sum=102 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=openai_gpt-4o-mini-2024-07-18"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=openai_gpt-4o-mini-2024-07-18"
        ]
      },
      {
        "value": 0.0,
        "description": "min=0, mean=0, max=0, sum=0 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=openai_gpt-4o-mini-2024-07-18"
        ]
      },
      {
        "value": 981.8137254901961,
        "description": "min=981.814, mean=981.814, max=981.814, sum=981.814 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=openai_gpt-4o-mini-2024-07-18"
        ]
      },
      {
        "value": 49.549019607843135,
        "description": "min=49.549, mean=49.549, max=49.549, sum=49.549 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "czech_bank_qa:model=openai_gpt-4o-mini-2024-07-18"
        ]
      }
    ]
  ],
  "links": [
    {
      "text": "LaTeX",
      "href": "benchmark_output/releases/v0.4.0/groups/latex/financial_scenarios_general_information.tex"
    },
    {
      "text": "JSON",
      "href": "benchmark_output/releases/v0.4.0/groups/json/financial_scenarios_general_information.json"
    }
  ],
  "name": "general_information"
}