[
  {
    "title": "",
    "header": [
      {
        "value": "Model/adapter",
        "markdown": false,
        "metadata": {}
      },
      {
        "value": "Recall",
        "description": "summarization\n\nRecall: How many key items were recalled",
        "markdown": false,
        "lower_is_better": false,
        "metadata": {
          "metric": "Recall",
          "run_group": "Summarization (Key Points Recall)"
        }
      },
      {
        "value": "# eval",
        "description": "summarization\n\n# eval: Number of evaluation instances.",
        "markdown": false,
        "metadata": {
          "metric": "# eval",
          "run_group": "Summarization (Key Points Recall)"
        }
      },
      {
        "value": "# train",
        "description": "summarization\n\n# train: Number of training instances (e.g., in-context examples).",
        "markdown": false,
        "metadata": {
          "metric": "# train",
          "run_group": "Summarization (Key Points Recall)"
        }
      },
      {
        "value": "truncated",
        "description": "summarization\n\ntruncated: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).",
        "markdown": false,
        "metadata": {
          "metric": "truncated",
          "run_group": "Summarization (Key Points Recall)"
        }
      },
      {
        "value": "# prompt tokens",
        "description": "summarization\n\n# prompt tokens: Number of tokens in the prompt.",
        "markdown": false,
        "metadata": {
          "metric": "# prompt tokens",
          "run_group": "Summarization (Key Points Recall)"
        }
      },
      {
        "value": "# output tokens",
        "description": "summarization\n\n# output tokens: Actual number of output tokens.",
        "markdown": false,
        "metadata": {
          "metric": "# output tokens",
          "run_group": "Summarization (Key Points Recall)"
        }
      }
    ],
    "rows": [
      [
        {
          "value": "Llama 3 Instruct (70B)",
          "description": "",
          "href": "?group=call_center_summarization_key_points_recall&subgroup=&runSpecs=%5B%22call_center_summarization_key_points_recall%3Amodel%3Dmeta_llama-3-70b-chat%22%5D",
          "markdown": false,
          "run_spec_names": [
            "call_center_summarization_key_points_recall:model=meta_llama-3-70b-chat"
          ]
        },
        {
          "value": 0.8087500000000011,
          "description": "min=0.809, mean=0.809, max=0.809, sum=0.809 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 240.0,
          "description": "min=240, mean=240, max=240, sum=240 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 731.5833333333334,
          "description": "min=731.583, mean=731.583, max=731.583, sum=731.583 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 169.7125,
          "description": "min=169.713, mean=169.713, max=169.713, sum=169.713 (1)",
          "style": {},
          "markdown": false
        }
      ],
      [
        {
          "value": "Llama 3 Instruct (8B)",
          "description": "",
          "href": "?group=call_center_summarization_key_points_recall&subgroup=&runSpecs=%5B%22call_center_summarization_key_points_recall%3Amodel%3Dmeta_llama-3-8b-chat%22%5D",
          "markdown": false,
          "run_spec_names": [
            "call_center_summarization_key_points_recall:model=meta_llama-3-8b-chat"
          ]
        },
        {
          "value": 0.784583333333334,
          "description": "min=0.785, mean=0.785, max=0.785, sum=0.785 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 240.0,
          "description": "min=240, mean=240, max=240, sum=240 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 731.5833333333334,
          "description": "min=731.583, mean=731.583, max=731.583, sum=731.583 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 174.11666666666667,
          "description": "min=174.117, mean=174.117, max=174.117, sum=174.117 (1)",
          "style": {},
          "markdown": false
        }
      ],
      [
        {
          "value": "Claude 3.5 Sonnet (20240620)",
          "description": "",
          "href": "?group=call_center_summarization_key_points_recall&subgroup=&runSpecs=%5B%22call_center_summarization_key_points_recall%3Amodel%3Danthropic_claude-3-5-sonnet-20240620%22%5D",
          "markdown": false,
          "run_spec_names": [
            "call_center_summarization_key_points_recall:model=anthropic_claude-3-5-sonnet-20240620"
          ]
        },
        {
          "value": 0.8287500000000013,
          "description": "min=0.829, mean=0.829, max=0.829, sum=0.829 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 240.0,
          "description": "min=240, mean=240, max=240, sum=240 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 801.1083333333333,
          "description": "min=801.108, mean=801.108, max=801.108, sum=801.108 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 151.10833333333332,
          "description": "min=151.108, mean=151.108, max=151.108, sum=151.108 (1)",
          "style": {},
          "markdown": false
        }
      ],
      [
        {
          "value": "GPT-4o mini (2024-07-18)",
          "description": "",
          "href": "?group=call_center_summarization_key_points_recall&subgroup=&runSpecs=%5B%22call_center_summarization_key_points_recall%3Amodel%3Dopenai_gpt-4o-mini-2024-07-18%22%5D",
          "markdown": false,
          "run_spec_names": [
            "call_center_summarization_key_points_recall:model=openai_gpt-4o-mini-2024-07-18"
          ]
        },
        {
          "value": 0.9044907407407423,
          "description": "min=0.904, mean=0.904, max=0.904, sum=0.904 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 240.0,
          "description": "min=240, mean=240, max=240, sum=240 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 728.7625,
          "description": "min=728.763, mean=728.763, max=728.763, sum=728.763 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 179.03333333333333,
          "description": "min=179.033, mean=179.033, max=179.033, sum=179.033 (1)",
          "style": {},
          "markdown": false
        }
      ]
    ],
    "links": [
      {
        "text": "compare all",
        "href": "?group=call_center_summarization_key_points_recall&subgroup=&runSpecs=%5B%22call_center_summarization_key_points_recall%3Amodel%3Danthropic_claude-3-5-sonnet-20240620%22%2C%20%22call_center_summarization_key_points_recall%3Amodel%3Dmeta_llama-3-70b-chat%22%2C%20%22call_center_summarization_key_points_recall%3Amodel%3Dmeta_llama-3-8b-chat%22%2C%20%22call_center_summarization_key_points_recall%3Amodel%3Dopenai_gpt-4o-mini-2024-07-18%22%5D"
      },
      {
        "text": "LaTeX",
        "href": "benchmark_output/releases/v0.1.0/groups/latex/call_center_summarization_key_points_recall_call_center_summarization_key_points_recall_.tex"
      },
      {
        "text": "JSON",
        "href": "benchmark_output/releases/v0.1.0/groups/json/call_center_summarization_key_points_recall_call_center_summarization_key_points_recall_.json"
      }
    ],
    "name": "call_center_summarization_key_points_recall_"
  }
]