{
  "name": "narrative_qa:model=openai_gpt-3.5-turbo-0613,additional_instructions=narrative_qa",
  "scenario_spec": {
    "class_name": "helm.benchmark.scenarios.narrativeqa_scenario.NarrativeQAScenario",
    "args": {}
  },
  "adapter_spec": {
    "method": "generation",
    "global_prefix": "",
    "global_suffix": "",
    "instructions": " Answer only with a single letter corresponding to the correct option.\n",
    "input_prefix": "Passage: ",
    "input_suffix": "\n",
    "reference_prefix": "A. ",
    "reference_suffix": "\n",
    "output_prefix": "Answer: ",
    "output_suffix": "\n",
    "instance_prefix": "\n",
    "substitutions": [],
    "max_train_instances": 5,
    "max_eval_instances": 50,
    "num_outputs": 1,
    "num_train_trials": 1,
    "num_trials": 1,
    "sample_train": true,
    "model_deployment": "openai/gpt-3.5-turbo-0613",
    "model": "openai/gpt-3.5-turbo-0613",
    "temperature": 0.0,
    "max_tokens": 100,
    "stop_sequences": [
      "\n"
    ],
    "multi_label": false
  },
  "metric_specs": [
    {
      "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric",
      "args": {
        "names": [
          "exact_match",
          "quasi_exact_match",
          "f1_score",
          "rouge_l",
          "bleu_1",
          "bleu_4"
        ]
      }
    },
    {
      "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric",
      "args": {}
    },
    {
      "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric",
      "args": {}
    }
  ],
  "data_augmenter_spec": {
    "perturbation_specs": [],
    "should_augment_train_instances": false,
    "should_include_original_train": false,
    "should_skip_unchanged_train": false,
    "should_augment_eval_instances": false,
    "should_include_original_eval": false,
    "should_skip_unchanged_eval": false,
    "seeds_per_instance": 1
  },
  "groups": [
    "narrative_qa"
  ]
}