{
  "name": "helpdesk_call_summarization:model=google_gemini-1.5-pro-002",
  "scenario_spec": {
    "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario",
    "args": {}
  },
  "adapter_spec": {
    "method": "generation",
    "global_prefix": "",
    "global_suffix": "",
    "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.",
    "input_prefix": "### Call Transcript\n",
    "input_suffix": "",
    "reference_prefix": "A. ",
    "reference_suffix": "\n",
    "chain_of_thought_prefix": "",
    "chain_of_thought_suffix": "\n",
    "output_prefix": "",
    "output_suffix": "",
    "instance_prefix": "\n",
    "substitutions": [],
    "max_train_instances": 0,
    "max_eval_instances": 10000,
    "num_outputs": 1,
    "num_train_trials": 1,
    "num_trials": 1,
    "sample_train": true,
    "model_deployment": "google/gemini-1.5-pro-002",
    "model": "google/gemini-1.5-pro-002",
    "temperature": 0.0,
    "max_tokens": 512,
    "stop_sequences": [],
    "multi_label": false
  },
  "metric_specs": [
    {
      "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric",
      "args": {
        "names": []
      }
    },
    {
      "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric",
      "args": {}
    },
    {
      "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric",
      "args": {}
    },
    {
      "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric",
      "args": {}
    }
  ],
  "data_augmenter_spec": {
    "perturbation_specs": [],
    "should_augment_train_instances": false,
    "should_include_original_train": false,
    "should_skip_unchanged_train": false,
    "should_augment_eval_instances": false,
    "should_include_original_eval": false,
    "should_skip_unchanged_eval": false,
    "seeds_per_instance": 1
  },
  "groups": [
    "helpdesk_call_summarization"
  ],
  "annotators": [
    {
      "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator",
      "args": {}
    }
  ]
}