[ { "name": "bird_sql:model=anthropic_claude-3-5-haiku-20241022", "scenario_spec": { "class_name": "helm.benchmark.scenarios.bird_sql_scenario.BIRDSQLScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "anthropic/claude-3-5-haiku-20241022", "model": "anthropic/claude-3-5-haiku-20241022", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.bird_sql_metrics.BirdSQLMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "bird_sql" ], "annotators": [ { "class_name": "helm.benchmark.annotation.bird_sql_annotator.BirdSQLAnnotator", "args": {} } ] }, { "name": "bird_sql:model=anthropic_claude-3-5-sonnet-20240620", "scenario_spec": { "class_name": "helm.benchmark.scenarios.bird_sql_scenario.BIRDSQLScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "anthropic/claude-3-5-sonnet-20240620", "model": "anthropic/claude-3-5-sonnet-20240620", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.bird_sql_metrics.BirdSQLMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "bird_sql" ], "annotators": [ { "class_name": "helm.benchmark.annotation.bird_sql_annotator.BirdSQLAnnotator", "args": {} } ] }, { "name": "bird_sql:model=google_gemini-1.5-flash-002", "scenario_spec": { "class_name": "helm.benchmark.scenarios.bird_sql_scenario.BIRDSQLScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "google/gemini-1.5-flash-002", "model": "google/gemini-1.5-flash-002", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.bird_sql_metrics.BirdSQLMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "bird_sql" ], "annotators": [ { "class_name": "helm.benchmark.annotation.bird_sql_annotator.BirdSQLAnnotator", "args": {} } ] }, { "name": "bird_sql:model=google_gemini-1.5-pro-002", "scenario_spec": { "class_name": "helm.benchmark.scenarios.bird_sql_scenario.BIRDSQLScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "google/gemini-1.5-pro-002", "model": "google/gemini-1.5-pro-002", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.bird_sql_metrics.BirdSQLMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "bird_sql" ], "annotators": [ { "class_name": "helm.benchmark.annotation.bird_sql_annotator.BirdSQLAnnotator", "args": {} } ] }, { "name": "bird_sql:model=meta_llama-3.1-405b-instruct-turbo", "scenario_spec": { "class_name": "helm.benchmark.scenarios.bird_sql_scenario.BIRDSQLScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "together/llama-3.1-405b-instruct-turbo", "model": "meta/llama-3.1-405b-instruct-turbo", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.bird_sql_metrics.BirdSQLMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "bird_sql" ], "annotators": [ { "class_name": "helm.benchmark.annotation.bird_sql_annotator.BirdSQLAnnotator", "args": {} } ] }, { "name": "bird_sql:model=meta_llama-3.1-70b-instruct-turbo", "scenario_spec": { "class_name": "helm.benchmark.scenarios.bird_sql_scenario.BIRDSQLScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "together/llama-3.1-70b-instruct-turbo", "model": "meta/llama-3.1-70b-instruct-turbo", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.bird_sql_metrics.BirdSQLMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "bird_sql" ], "annotators": [ { "class_name": "helm.benchmark.annotation.bird_sql_annotator.BirdSQLAnnotator", "args": {} } ] }, { "name": "bird_sql:model=meta_llama-3.1-8b-instruct-turbo", "scenario_spec": { "class_name": "helm.benchmark.scenarios.bird_sql_scenario.BIRDSQLScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "together/llama-3.1-8b-instruct-turbo", "model": "meta/llama-3.1-8b-instruct-turbo", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.bird_sql_metrics.BirdSQLMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "bird_sql" ], "annotators": [ { "class_name": "helm.benchmark.annotation.bird_sql_annotator.BirdSQLAnnotator", "args": {} } ] }, { "name": "bird_sql:model=openai_gpt-4o-2024-08-06", "scenario_spec": { "class_name": "helm.benchmark.scenarios.bird_sql_scenario.BIRDSQLScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "openai/gpt-4o-2024-08-06", "model": "openai/gpt-4o-2024-08-06", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.bird_sql_metrics.BirdSQLMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "bird_sql" ], "annotators": [ { "class_name": "helm.benchmark.annotation.bird_sql_annotator.BirdSQLAnnotator", "args": {} } ] }, { "name": "bird_sql:model=openai_gpt-4o-mini-2024-07-18", "scenario_spec": { "class_name": "helm.benchmark.scenarios.bird_sql_scenario.BIRDSQLScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "openai/gpt-4o-mini-2024-07-18", "model": "openai/gpt-4o-mini-2024-07-18", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.bird_sql_metrics.BirdSQLMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "bird_sql" ], "annotators": [ { "class_name": "helm.benchmark.annotation.bird_sql_annotator.BirdSQLAnnotator", "args": {} } ] }, { "name": "spider:model=anthropic_claude-3-5-haiku-20241022", "scenario_spec": { "class_name": "helm.benchmark.scenarios.spider_scenario.SpiderScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "anthropic/claude-3-5-haiku-20241022", "model": "anthropic/claude-3-5-haiku-20241022", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.spider_metrics.SpiderMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "spider" ], "annotators": [ { "class_name": "helm.benchmark.annotation.spider_annotator.SpiderAnnotator", "args": {} } ] }, { "name": "spider:model=anthropic_claude-3-5-sonnet-20240620", "scenario_spec": { "class_name": "helm.benchmark.scenarios.spider_scenario.SpiderScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "anthropic/claude-3-5-sonnet-20240620", "model": "anthropic/claude-3-5-sonnet-20240620", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.spider_metrics.SpiderMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "spider" ], "annotators": [ { "class_name": "helm.benchmark.annotation.spider_annotator.SpiderAnnotator", "args": {} } ] }, { "name": "spider:model=google_gemini-1.5-flash-002", "scenario_spec": { "class_name": "helm.benchmark.scenarios.spider_scenario.SpiderScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "google/gemini-1.5-flash-002", "model": "google/gemini-1.5-flash-002", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.spider_metrics.SpiderMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "spider" ], "annotators": [ { "class_name": "helm.benchmark.annotation.spider_annotator.SpiderAnnotator", "args": {} } ] }, { "name": "spider:model=google_gemini-1.5-pro-002", "scenario_spec": { "class_name": "helm.benchmark.scenarios.spider_scenario.SpiderScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "google/gemini-1.5-pro-002", "model": "google/gemini-1.5-pro-002", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.spider_metrics.SpiderMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "spider" ], "annotators": [ { "class_name": "helm.benchmark.annotation.spider_annotator.SpiderAnnotator", "args": {} } ] }, { "name": "spider:model=meta_llama-3.1-405b-instruct-turbo", "scenario_spec": { "class_name": "helm.benchmark.scenarios.spider_scenario.SpiderScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "together/llama-3.1-405b-instruct-turbo", "model": "meta/llama-3.1-405b-instruct-turbo", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.spider_metrics.SpiderMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "spider" ], "annotators": [ { "class_name": "helm.benchmark.annotation.spider_annotator.SpiderAnnotator", "args": {} } ] }, { "name": "spider:model=meta_llama-3.1-70b-instruct-turbo", "scenario_spec": { "class_name": "helm.benchmark.scenarios.spider_scenario.SpiderScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "together/llama-3.1-70b-instruct-turbo", "model": "meta/llama-3.1-70b-instruct-turbo", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.spider_metrics.SpiderMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "spider" ], "annotators": [ { "class_name": "helm.benchmark.annotation.spider_annotator.SpiderAnnotator", "args": {} } ] }, { "name": "spider:model=meta_llama-3.1-8b-instruct-turbo", "scenario_spec": { "class_name": "helm.benchmark.scenarios.spider_scenario.SpiderScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "together/llama-3.1-8b-instruct-turbo", "model": "meta/llama-3.1-8b-instruct-turbo", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.spider_metrics.SpiderMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "spider" ], "annotators": [ { "class_name": "helm.benchmark.annotation.spider_annotator.SpiderAnnotator", "args": {} } ] }, { "name": "spider:model=openai_gpt-4o-2024-08-06", "scenario_spec": { "class_name": "helm.benchmark.scenarios.spider_scenario.SpiderScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "openai/gpt-4o-2024-08-06", "model": "openai/gpt-4o-2024-08-06", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.spider_metrics.SpiderMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "spider" ], "annotators": [ { "class_name": "helm.benchmark.annotation.spider_annotator.SpiderAnnotator", "args": {} } ] }, { "name": "spider:model=openai_gpt-4o-mini-2024-07-18", "scenario_spec": { "class_name": "helm.benchmark.scenarios.spider_scenario.SpiderScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "", "input_prefix": "", "input_suffix": "\n", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "\n", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 5, "max_eval_instances": 100, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "openai/gpt-4o-mini-2024-07-18", "model": "openai/gpt-4o-mini-2024-07-18", "temperature": 0.0, "max_tokens": 1024, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [ "exact_match", "quasi_exact_match", "prefix_exact_match", "quasi_prefix_exact_match" ] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.spider_metrics.SpiderMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "spider" ], "annotators": [ { "class_name": "helm.benchmark.annotation.spider_annotator.SpiderAnnotator", "args": {} } ] } ]