[
  {
    "title": "revision: main",
    "header": [
      {
        "value": "Model/adapter",
        "markdown": false,
        "metadata": {}
      },
      {
        "value": "Faithfulness",
        "description": "summarization\n\nFaithfulness: Whether all the information expressed by the summary can be inferred from the source transcript.",
        "markdown": false,
        "lower_is_better": false,
        "metadata": {
          "metric": "Faithfulness",
          "run_group": "Summarization"
        }
      },
      {
        "value": "Relevance",
        "description": "summarization\n\nRelevance: Whether the summary includes only important information from the source.",
        "markdown": false,
        "lower_is_better": false,
        "metadata": {
          "metric": "Relevance",
          "run_group": "Summarization"
        }
      },
      {
        "value": "Coherence",
        "description": "summarization\n\nCoherence: Whether the summary organizes the relevant information into a well-structured summary.",
        "markdown": false,
        "lower_is_better": false,
        "metadata": {
          "metric": "Coherence",
          "run_group": "Summarization"
        }
      },
      {
        "value": "# eval",
        "description": "summarization\n\n# eval: Number of evaluation instances.",
        "markdown": false,
        "metadata": {
          "metric": "# eval",
          "run_group": "Summarization"
        }
      },
      {
        "value": "# train",
        "description": "summarization\n\n# train: Number of training instances (e.g., in-context examples).",
        "markdown": false,
        "metadata": {
          "metric": "# train",
          "run_group": "Summarization"
        }
      },
      {
        "value": "truncated",
        "description": "summarization\n\ntruncated: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).",
        "markdown": false,
        "metadata": {
          "metric": "truncated",
          "run_group": "Summarization"
        }
      },
      {
        "value": "# prompt tokens",
        "description": "summarization\n\n# prompt tokens: Number of tokens in the prompt.",
        "markdown": false,
        "metadata": {
          "metric": "# prompt tokens",
          "run_group": "Summarization"
        }
      },
      {
        "value": "# output tokens",
        "description": "summarization\n\n# output tokens: Actual number of output tokens.",
        "markdown": false,
        "metadata": {
          "metric": "# output tokens",
          "run_group": "Summarization"
        }
      }
    ],
    "rows": [
      [
        {
          "value": "Llama 3 Instruct (70B)",
          "description": "",
          "href": "?group=call_center_summarization&subgroup=revision%3A%20main&runSpecs=%5B%22call_center_summarization%3Amodel%3Dmeta_llama-3-70b-chat%22%5D",
          "markdown": false,
          "run_spec_names": [
            "call_center_summarization:model=meta_llama-3-70b-chat"
          ]
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 240.0,
          "description": "min=240, mean=240, max=240, sum=240 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 731.5833333333334,
          "description": "min=731.583, mean=731.583, max=731.583, sum=731.583 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 169.04166666666666,
          "description": "min=169.042, mean=169.042, max=169.042, sum=169.042 (1)",
          "style": {},
          "markdown": false
        }
      ],
      [
        {
          "value": "Llama 3 Instruct (8B)",
          "description": "",
          "href": "?group=call_center_summarization&subgroup=revision%3A%20main&runSpecs=%5B%22call_center_summarization%3Amodel%3Dmeta_llama-3-8b-chat%22%5D",
          "markdown": false,
          "run_spec_names": [
            "call_center_summarization:model=meta_llama-3-8b-chat"
          ]
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 240.0,
          "description": "min=240, mean=240, max=240, sum=240 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 731.5833333333334,
          "description": "min=731.583, mean=731.583, max=731.583, sum=731.583 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 173.4125,
          "description": "min=173.412, mean=173.412, max=173.412, sum=173.412 (1)",
          "style": {},
          "markdown": false
        }
      ],
      [
        {
          "value": "Claude 3.5 Sonnet (20240620)",
          "description": "",
          "href": "?group=call_center_summarization&subgroup=revision%3A%20main&runSpecs=%5B%22call_center_summarization%3Amodel%3Danthropic_claude-3-5-sonnet-20240620%22%5D",
          "markdown": false,
          "run_spec_names": [
            "call_center_summarization:model=anthropic_claude-3-5-sonnet-20240620"
          ]
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 240.0,
          "description": "min=240, mean=240, max=240, sum=240 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 801.1083333333333,
          "description": "min=801.108, mean=801.108, max=801.108, sum=801.108 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 151.3,
          "description": "min=151.3, mean=151.3, max=151.3, sum=151.3 (1)",
          "style": {},
          "markdown": false
        }
      ],
      [
        {
          "value": "GPT-4o (2024-05-13)",
          "description": "",
          "href": "?group=call_center_summarization&subgroup=revision%3A%20main&runSpecs=%5B%22call_center_summarization%3Amodel%3Dopenai_gpt-4o-2024-05-13%22%5D",
          "markdown": false,
          "run_spec_names": [
            "call_center_summarization:model=openai_gpt-4o-2024-05-13"
          ]
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 240.0,
          "description": "min=240, mean=240, max=240, sum=240 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 728.7625,
          "description": "min=728.763, mean=728.763, max=728.763, sum=728.763 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 155.52916666666667,
          "description": "min=155.529, mean=155.529, max=155.529, sum=155.529 (1)",
          "style": {},
          "markdown": false
        }
      ],
      [
        {
          "value": "GPT-4o mini (2024-07-18)",
          "description": "",
          "href": "?group=call_center_summarization&subgroup=revision%3A%20main&runSpecs=%5B%22call_center_summarization%3Amodel%3Dopenai_gpt-4o-mini-2024-07-18%22%5D",
          "markdown": false,
          "run_spec_names": [
            "call_center_summarization:model=openai_gpt-4o-mini-2024-07-18"
          ]
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 240.0,
          "description": "min=240, mean=240, max=240, sum=240 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 728.7625,
          "description": "min=728.763, mean=728.763, max=728.763, sum=728.763 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 178.33333333333334,
          "description": "min=178.333, mean=178.333, max=178.333, sum=178.333 (1)",
          "style": {},
          "markdown": false
        }
      ]
    ],
    "links": [
      {
        "text": "compare all",
        "href": "?group=call_center_summarization&subgroup=revision%3A%20main&runSpecs=%5B%22call_center_summarization%3Amodel%3Danthropic_claude-3-5-sonnet-20240620%22%2C%20%22call_center_summarization%3Amodel%3Dmeta_llama-3-70b-chat%22%2C%20%22call_center_summarization%3Amodel%3Dmeta_llama-3-8b-chat%22%2C%20%22call_center_summarization%3Amodel%3Dopenai_gpt-4o-2024-05-13%22%2C%20%22call_center_summarization%3Amodel%3Dopenai_gpt-4o-mini-2024-07-18%22%5D"
      },
      {
        "text": "LaTeX",
        "href": "benchmark_output/releases/v0.1.0/groups/latex/call_center_summarization_call_center_summarization_revision:main.tex"
      },
      {
        "text": "JSON",
        "href": "benchmark_output/releases/v0.1.0/groups/json/call_center_summarization_call_center_summarization_revision:main.json"
      }
    ],
    "name": "call_center_summarization_revision:main"
  }
]