[
  {
    "title": "subset: real_call_transcripts",
    "header": [
      {
        "value": "Model/adapter",
        "markdown": false,
        "metadata": {}
      },
      {
        "value": "Faithfulness",
        "description": "Summarization with real call transcripts\n\nFaithfulness: Whether all the information expressed by the summary can be inferred from the source transcript.",
        "markdown": false,
        "lower_is_better": false,
        "metadata": {
          "metric": "Faithfulness",
          "run_group": "Summarization (Real)"
        }
      },
      {
        "value": "Relevance",
        "description": "Summarization with real call transcripts\n\nRelevance: Whether the summary includes only important information from the source.",
        "markdown": false,
        "lower_is_better": false,
        "metadata": {
          "metric": "Relevance",
          "run_group": "Summarization (Real)"
        }
      },
      {
        "value": "Coherence",
        "description": "Summarization with real call transcripts\n\nCoherence: Whether the summary organizes the relevant information into a well-structured summary.",
        "markdown": false,
        "lower_is_better": false,
        "metadata": {
          "metric": "Coherence",
          "run_group": "Summarization (Real)"
        }
      },
      {
        "value": "# eval",
        "description": "Summarization with real call transcripts\n\n# eval: Number of evaluation instances.",
        "markdown": false,
        "metadata": {
          "metric": "# eval",
          "run_group": "Summarization (Real)"
        }
      },
      {
        "value": "# train",
        "description": "Summarization with real call transcripts\n\n# train: Number of training instances (e.g., in-context examples).",
        "markdown": false,
        "metadata": {
          "metric": "# train",
          "run_group": "Summarization (Real)"
        }
      },
      {
        "value": "truncated",
        "description": "Summarization with real call transcripts\n\ntruncated: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).",
        "markdown": false,
        "metadata": {
          "metric": "truncated",
          "run_group": "Summarization (Real)"
        }
      },
      {
        "value": "# prompt tokens",
        "description": "Summarization with real call transcripts\n\n# prompt tokens: Number of tokens in the prompt.",
        "markdown": false,
        "metadata": {
          "metric": "# prompt tokens",
          "run_group": "Summarization (Real)"
        }
      },
      {
        "value": "# output tokens",
        "description": "Summarization with real call transcripts\n\n# output tokens: Actual number of output tokens.",
        "markdown": false,
        "metadata": {
          "metric": "# output tokens",
          "run_group": "Summarization (Real)"
        }
      }
    ],
    "rows": [
      [
        {
          "value": "Llama 3 Instruct (70B)",
          "description": "",
          "href": "?group=call_center_summarization_real_call_transcripts&subgroup=subset%3A%20real_call_transcripts&runSpecs=%5B%22call_center_summarization%3Asubset%3Dreal_call_transcripts%2Cmodel%3Dmeta_llama-3-70b-chat%22%5D",
          "markdown": false,
          "run_spec_names": [
            "call_center_summarization:subset=real_call_transcripts,model=meta_llama-3-70b-chat"
          ]
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 5.0,
          "description": "min=5, mean=5, max=5, sum=5 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 1688.0,
          "description": "min=1688, mean=1688, max=1688, sum=1688 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 171.0,
          "description": "min=171, mean=171, max=171, sum=171 (1)",
          "style": {},
          "markdown": false
        }
      ],
      [
        {
          "value": "Llama 3 Instruct (8B)",
          "description": "",
          "href": "?group=call_center_summarization_real_call_transcripts&subgroup=subset%3A%20real_call_transcripts&runSpecs=%5B%22call_center_summarization%3Asubset%3Dreal_call_transcripts%2Cmodel%3Dmeta_llama-3-8b-chat%22%5D",
          "markdown": false,
          "run_spec_names": [
            "call_center_summarization:subset=real_call_transcripts,model=meta_llama-3-8b-chat"
          ]
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 5.0,
          "description": "min=5, mean=5, max=5, sum=5 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 1688.0,
          "description": "min=1688, mean=1688, max=1688, sum=1688 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 185.2,
          "description": "min=185.2, mean=185.2, max=185.2, sum=185.2 (1)",
          "style": {},
          "markdown": false
        }
      ],
      [
        {
          "value": "Claude 3.5 Sonnet (20240620)",
          "description": "",
          "href": "?group=call_center_summarization_real_call_transcripts&subgroup=subset%3A%20real_call_transcripts&runSpecs=%5B%22call_center_summarization%3Asubset%3Dreal_call_transcripts%2Cmodel%3Danthropic_claude-3-5-sonnet-20240620%22%5D",
          "markdown": false,
          "run_spec_names": [
            "call_center_summarization:subset=real_call_transcripts,model=anthropic_claude-3-5-sonnet-20240620"
          ]
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 5.0,
          "description": "min=5, mean=5, max=5, sum=5 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 1726.4,
          "description": "min=1726.4, mean=1726.4, max=1726.4, sum=1726.4 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 173.0,
          "description": "min=173, mean=173, max=173, sum=173 (1)",
          "style": {},
          "markdown": false
        }
      ],
      [
        {
          "value": "GPT-4o (2024-05-13)",
          "description": "",
          "href": "?group=call_center_summarization_real_call_transcripts&subgroup=subset%3A%20real_call_transcripts&runSpecs=%5B%22call_center_summarization%3Asubset%3Dreal_call_transcripts%2Cmodel%3Dopenai_gpt-4o-2024-05-13%22%5D",
          "markdown": false,
          "run_spec_names": [
            "call_center_summarization:subset=real_call_transcripts,model=openai_gpt-4o-2024-05-13"
          ]
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 0.95,
          "description": "min=0.95, mean=0.95, max=0.95, sum=0.95 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 5.0,
          "description": "min=5, mean=5, max=5, sum=5 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 1642.6,
          "description": "min=1642.6, mean=1642.6, max=1642.6, sum=1642.6 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 142.8,
          "description": "min=142.8, mean=142.8, max=142.8, sum=142.8 (1)",
          "style": {},
          "markdown": false
        }
      ],
      [
        {
          "value": "GPT-4o mini (2024-07-18)",
          "description": "",
          "href": "?group=call_center_summarization_real_call_transcripts&subgroup=subset%3A%20real_call_transcripts&runSpecs=%5B%22call_center_summarization%3Asubset%3Dreal_call_transcripts%2Cmodel%3Dopenai_gpt-4o-mini-2024-07-18%22%5D",
          "markdown": false,
          "run_spec_names": [
            "call_center_summarization:subset=real_call_transcripts,model=openai_gpt-4o-mini-2024-07-18"
          ]
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 1.0,
          "description": "min=1, mean=1, max=1, sum=1 (1)",
          "style": {
            "font-weight": "bold"
          },
          "markdown": false
        },
        {
          "value": 5.0,
          "description": "min=5, mean=5, max=5, sum=5 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 0.0,
          "description": "min=0, mean=0, max=0, sum=0 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 1642.6,
          "description": "min=1642.6, mean=1642.6, max=1642.6, sum=1642.6 (1)",
          "style": {},
          "markdown": false
        },
        {
          "value": 164.0,
          "description": "min=164, mean=164, max=164, sum=164 (1)",
          "style": {},
          "markdown": false
        }
      ]
    ],
    "links": [
      {
        "text": "compare all",
        "href": "?group=call_center_summarization_real_call_transcripts&subgroup=subset%3A%20real_call_transcripts&runSpecs=%5B%22call_center_summarization%3Asubset%3Dreal_call_transcripts%2Cmodel%3Danthropic_claude-3-5-sonnet-20240620%22%2C%20%22call_center_summarization%3Asubset%3Dreal_call_transcripts%2Cmodel%3Dmeta_llama-3-70b-chat%22%2C%20%22call_center_summarization%3Asubset%3Dreal_call_transcripts%2Cmodel%3Dmeta_llama-3-8b-chat%22%2C%20%22call_center_summarization%3Asubset%3Dreal_call_transcripts%2Cmodel%3Dopenai_gpt-4o-2024-05-13%22%2C%20%22call_center_summarization%3Asubset%3Dreal_call_transcripts%2Cmodel%3Dopenai_gpt-4o-mini-2024-07-18%22%5D"
      },
      {
        "text": "LaTeX",
        "href": "benchmark_output/releases/v0.1.0/groups/latex/call_center_summarization_real_call_transcripts_call_center_summarization_real_call_transcripts_subset:real_call_transcripts.tex"
      },
      {
        "text": "JSON",
        "href": "benchmark_output/releases/v0.1.0/groups/json/call_center_summarization_real_call_transcripts_call_center_summarization_real_call_transcripts_subset:real_call_transcripts.json"
      }
    ],
    "name": "call_center_summarization_real_call_transcripts_subset:real_call_transcripts"
  }
]