[ { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=anthropic_claude-3-5-haiku-20241022", "run_spec": { "name": "helpdesk_call_summarization:model=anthropic_claude-3-5-haiku-20241022", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "anthropic/claude-3-5-haiku-20241022", "model": "anthropic/claude-3-5-haiku-20241022", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1372.1666666666667, "sum_squared": 1882841.3611111112, "min": 1372.1666666666667, "max": 1372.1666666666667, "mean": 1372.1666666666667, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 192.54320987654322, "sum_squared": 37072.887669562566, "min": 192.54320987654322, "max": 192.54320987654322, "mean": 192.54320987654322, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 192.54320987654322, "sum_squared": 37072.887669562566, "min": 192.54320987654322, "max": 192.54320987654322, "mean": 192.54320987654322, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 5.2558664892926625, "sum_squared": 27.62413255326958, "min": 5.2558664892926625, "max": 5.2558664892926625, "mean": 5.2558664892926625, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 192.54320987654322, "sum_squared": 37072.887669562566, "min": 192.54320987654322, "max": 192.54320987654322, "mean": 192.54320987654322, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 1032.888888888889, "sum_squared": 1066859.4567901236, "min": 1032.888888888889, "max": 1032.888888888889, "mean": 1032.888888888889, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1372.1666666666667, "sum_squared": 1882841.3611111112, "min": 1372.1666666666667, "max": 1372.1666666666667, "mean": 1372.1666666666667, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1372.1666666666667, "sum_squared": 1882841.3611111112, "min": 1372.1666666666667, "max": 1372.1666666666667, "mean": 1372.1666666666667, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 192.54320987654322, "sum_squared": 37072.887669562566, "min": 192.54320987654322, "max": 192.54320987654322, "mean": 192.54320987654322, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 192.54320987654322, "sum_squared": 37072.887669562566, "min": 192.54320987654322, "max": 192.54320987654322, "mean": 192.54320987654322, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 192.54320987654322, "sum_squared": 37072.887669562566, "min": 192.54320987654322, "max": 192.54320987654322, "mean": 192.54320987654322, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 192.54320987654322, "sum_squared": 37072.887669562566, "min": 192.54320987654322, "max": 192.54320987654322, "mean": 192.54320987654322, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 5.2558664892926625, "sum_squared": 27.62413255326958, "min": 5.2558664892926625, "max": 5.2558664892926625, "mean": 5.2558664892926625, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 5.2558664892926625, "sum_squared": 27.62413255326958, "min": 5.2558664892926625, "max": 5.2558664892926625, "mean": 5.2558664892926625, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 192.54320987654322, "sum_squared": 37072.887669562566, "min": 192.54320987654322, "max": 192.54320987654322, "mean": 192.54320987654322, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 192.54320987654322, "sum_squared": 37072.887669562566, "min": 192.54320987654322, "max": 192.54320987654322, "mean": 192.54320987654322, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1032.888888888889, "sum_squared": 1066859.4567901236, "min": 1032.888888888889, "max": 1032.888888888889, "mean": 1032.888888888889, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1032.888888888889, "sum_squared": 1066859.4567901236, "min": 1032.888888888889, "max": 1032.888888888889, "mean": 1032.888888888889, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.8566529492455397, "sum_squared": 0.7338542754510812, "min": 0.8566529492455397, "max": 0.8566529492455397, "mean": 0.8566529492455397, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.8566529492455397, "sum_squared": 0.7338542754510812, "min": 0.8566529492455397, "max": 0.8566529492455397, "mean": 0.8566529492455397, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.8566529492455397, "sum_squared": 0.7338542754510812, "min": 0.8566529492455397, "max": 0.8566529492455397, "mean": 0.8566529492455397, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=anthropic_claude-3-5-sonnet-20240620", "run_spec": { "name": "helpdesk_call_summarization:model=anthropic_claude-3-5-sonnet-20240620", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "anthropic/claude-3-5-sonnet-20240620", "model": "anthropic/claude-3-5-sonnet-20240620", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1372.1666666666667, "sum_squared": 1882841.3611111112, "min": 1372.1666666666667, "max": 1372.1666666666667, "mean": 1372.1666666666667, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 214.20987654320987, "sum_squared": 45885.87120865721, "min": 214.20987654320987, "max": 214.20987654320987, "mean": 214.20987654320987, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 214.20987654320987, "sum_squared": 45885.87120865721, "min": 214.20987654320987, "max": 214.20987654320987, "mean": 214.20987654320987, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 5.177797008443762, "sum_squared": 26.80958186064917, "min": 5.177797008443762, "max": 5.177797008443762, "mean": 5.177797008443762, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 214.20987654320987, "sum_squared": 45885.87120865721, "min": 214.20987654320987, "max": 214.20987654320987, "mean": 214.20987654320987, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 1120.2962962962963, "sum_squared": 1255063.7914951988, "min": 1120.2962962962963, "max": 1120.2962962962963, "mean": 1120.2962962962963, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1372.1666666666667, "sum_squared": 1882841.3611111112, "min": 1372.1666666666667, "max": 1372.1666666666667, "mean": 1372.1666666666667, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1372.1666666666667, "sum_squared": 1882841.3611111112, "min": 1372.1666666666667, "max": 1372.1666666666667, "mean": 1372.1666666666667, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 214.20987654320987, "sum_squared": 45885.87120865721, "min": 214.20987654320987, "max": 214.20987654320987, "mean": 214.20987654320987, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 214.20987654320987, "sum_squared": 45885.87120865721, "min": 214.20987654320987, "max": 214.20987654320987, "mean": 214.20987654320987, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 214.20987654320987, "sum_squared": 45885.87120865721, "min": 214.20987654320987, "max": 214.20987654320987, "mean": 214.20987654320987, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 214.20987654320987, "sum_squared": 45885.87120865721, "min": 214.20987654320987, "max": 214.20987654320987, "mean": 214.20987654320987, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 5.177797008443762, "sum_squared": 26.80958186064917, "min": 5.177797008443762, "max": 5.177797008443762, "mean": 5.177797008443762, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 5.177797008443762, "sum_squared": 26.80958186064917, "min": 5.177797008443762, "max": 5.177797008443762, "mean": 5.177797008443762, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 214.20987654320987, "sum_squared": 45885.87120865721, "min": 214.20987654320987, "max": 214.20987654320987, "mean": 214.20987654320987, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 214.20987654320987, "sum_squared": 45885.87120865721, "min": 214.20987654320987, "max": 214.20987654320987, "mean": 214.20987654320987, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1120.2962962962963, "sum_squared": 1255063.7914951988, "min": 1120.2962962962963, "max": 1120.2962962962963, "mean": 1120.2962962962963, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1120.2962962962963, "sum_squared": 1255063.7914951988, "min": 1120.2962962962963, "max": 1120.2962962962963, "mean": 1120.2962962962963, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.8431641518061251, "sum_squared": 0.7109257868909424, "min": 0.8431641518061251, "max": 0.8431641518061251, "mean": 0.8431641518061251, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.8431641518061251, "sum_squared": 0.7109257868909424, "min": 0.8431641518061251, "max": 0.8431641518061251, "mean": 0.8431641518061251, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.8431641518061251, "sum_squared": 0.7109257868909424, "min": 0.8431641518061251, "max": 0.8431641518061251, "mean": 0.8431641518061251, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=anthropic_claude-3-7-sonnet-20250219", "run_spec": { "name": "helpdesk_call_summarization:model=anthropic_claude-3-7-sonnet-20250219", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "anthropic/claude-3-7-sonnet-20250219", "model": "anthropic/claude-3-7-sonnet-20250219", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1372.1666666666667, "sum_squared": 1882841.3611111112, "min": 1372.1666666666667, "max": 1372.1666666666667, "mean": 1372.1666666666667, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 195.42592592592592, "sum_squared": 38191.29252400548, "min": 195.42592592592592, "max": 195.42592592592592, "mean": 195.42592592592592, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 195.42592592592592, "sum_squared": 38191.29252400548, "min": 195.42592592592592, "max": 195.42592592592592, "mean": 195.42592592592592, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 5.205429774743539, "sum_squared": 27.096499139786573, "min": 5.205429774743539, "max": 5.205429774743539, "mean": 5.205429774743539, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 195.42592592592592, "sum_squared": 38191.29252400548, "min": 195.42592592592592, "max": 195.42592592592592, "mean": 195.42592592592592, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 1022.4012345679013, "sum_squared": 1045304.2844459687, "min": 1022.4012345679013, "max": 1022.4012345679013, "mean": 1022.4012345679013, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1372.1666666666667, "sum_squared": 1882841.3611111112, "min": 1372.1666666666667, "max": 1372.1666666666667, "mean": 1372.1666666666667, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1372.1666666666667, "sum_squared": 1882841.3611111112, "min": 1372.1666666666667, "max": 1372.1666666666667, "mean": 1372.1666666666667, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 195.42592592592592, "sum_squared": 38191.29252400548, "min": 195.42592592592592, "max": 195.42592592592592, "mean": 195.42592592592592, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 195.42592592592592, "sum_squared": 38191.29252400548, "min": 195.42592592592592, "max": 195.42592592592592, "mean": 195.42592592592592, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 195.42592592592592, "sum_squared": 38191.29252400548, "min": 195.42592592592592, "max": 195.42592592592592, "mean": 195.42592592592592, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 195.42592592592592, "sum_squared": 38191.29252400548, "min": 195.42592592592592, "max": 195.42592592592592, "mean": 195.42592592592592, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 5.205429774743539, "sum_squared": 27.096499139786573, "min": 5.205429774743539, "max": 5.205429774743539, "mean": 5.205429774743539, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 5.205429774743539, "sum_squared": 27.096499139786573, "min": 5.205429774743539, "max": 5.205429774743539, "mean": 5.205429774743539, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 195.42592592592592, "sum_squared": 38191.29252400548, "min": 195.42592592592592, "max": 195.42592592592592, "mean": 195.42592592592592, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 195.42592592592592, "sum_squared": 38191.29252400548, "min": 195.42592592592592, "max": 195.42592592592592, "mean": 195.42592592592592, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1022.4012345679013, "sum_squared": 1045304.2844459687, "min": 1022.4012345679013, "max": 1022.4012345679013, "mean": 1022.4012345679013, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1022.4012345679013, "sum_squared": 1045304.2844459687, "min": 1022.4012345679013, "max": 1022.4012345679013, "mean": 1022.4012345679013, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.8687700045724716, "sum_squared": 0.7547613208448523, "min": 0.8687700045724716, "max": 0.8687700045724716, "mean": 0.8687700045724716, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.8687700045724716, "sum_squared": 0.7547613208448523, "min": 0.8687700045724716, "max": 0.8687700045724716, "mean": 0.8687700045724716, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.8687700045724716, "sum_squared": 0.7547613208448523, "min": 0.8687700045724716, "max": 0.8687700045724716, "mean": 0.8687700045724716, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=deepseek-ai_deepseek-v3", "run_spec": { "name": "helpdesk_call_summarization:model=deepseek-ai_deepseek-v3", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "together/deepseek-v3", "model": "deepseek-ai/deepseek-v3", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1336.8456790123457, "sum_squared": 1787156.3694939797, "min": 1336.8456790123457, "max": 1336.8456790123457, "mean": 1336.8456790123457, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 12.101275133497921, "sum_squared": 146.44085985661513, "min": 12.101275133497921, "max": 12.101275133497921, "mean": 12.101275133497921, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1336.8456790123457, "sum_squared": 1787156.3694939797, "min": 1336.8456790123457, "max": 1336.8456790123457, "mean": 1336.8456790123457, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1336.8456790123457, "sum_squared": 1787156.3694939797, "min": 1336.8456790123457, "max": 1336.8456790123457, "mean": 1336.8456790123457, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 12.101275133497921, "sum_squared": 146.44085985661513, "min": 12.101275133497921, "max": 12.101275133497921, "mean": 12.101275133497921, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 12.101275133497921, "sum_squared": 146.44085985661513, "min": 12.101275133497921, "max": 12.101275133497921, "mean": 12.101275133497921, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.8404206675811593, "sum_squared": 0.7063068984975615, "min": 0.8404206675811593, "max": 0.8404206675811593, "mean": 0.8404206675811593, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.8404206675811593, "sum_squared": 0.7063068984975615, "min": 0.8404206675811593, "max": 0.8404206675811593, "mean": 0.8404206675811593, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.8404206675811593, "sum_squared": 0.7063068984975615, "min": 0.8404206675811593, "max": 0.8404206675811593, "mean": 0.8404206675811593, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=google_gemini-1.5-flash-002", "run_spec": { "name": "helpdesk_call_summarization:model=google_gemini-1.5-flash-002", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "google/gemini-1.5-flash-002", "model": "google/gemini-1.5-flash-002", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1394.8086419753085, "sum_squared": 1945491.1477290045, "min": 1394.8086419753085, "max": 1394.8086419753085, "mean": 1394.8086419753085, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 1.0893347969761602, "sum_squared": 1.1866502999030921, "min": 1.0893347969761602, "max": 1.0893347969761602, "mean": 1.0893347969761602, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1394.8086419753085, "sum_squared": 1945491.1477290045, "min": 1394.8086419753085, "max": 1394.8086419753085, "mean": 1394.8086419753085, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1394.8086419753085, "sum_squared": 1945491.1477290045, "min": 1394.8086419753085, "max": 1394.8086419753085, "mean": 1394.8086419753085, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0893347969761602, "sum_squared": 1.1866502999030921, "min": 1.0893347969761602, "max": 1.0893347969761602, "mean": 1.0893347969761602, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0893347969761602, "sum_squared": 1.1866502999030921, "min": 1.0893347969761602, "max": 1.0893347969761602, "mean": 1.0893347969761602, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.8449931412894355, "sum_squared": 0.7140134088261879, "min": 0.8449931412894355, "max": 0.8449931412894355, "mean": 0.8449931412894355, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.8449931412894355, "sum_squared": 0.7140134088261879, "min": 0.8449931412894355, "max": 0.8449931412894355, "mean": 0.8449931412894355, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.8449931412894355, "sum_squared": 0.7140134088261879, "min": 0.8449931412894355, "max": 0.8449931412894355, "mean": 0.8449931412894355, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=google_gemini-1.5-pro-002", "run_spec": { "name": "helpdesk_call_summarization:model=google_gemini-1.5-pro-002", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "google/gemini-1.5-pro-002", "model": "google/gemini-1.5-pro-002", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1394.8086419753085, "sum_squared": 1945491.1477290045, "min": 1394.8086419753085, "max": 1394.8086419753085, "mean": 1394.8086419753085, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 3.760884882491312, "sum_squared": 14.144255099351689, "min": 3.760884882491312, "max": 3.760884882491312, "mean": 3.760884882491312, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1394.8086419753085, "sum_squared": 1945491.1477290045, "min": 1394.8086419753085, "max": 1394.8086419753085, "mean": 1394.8086419753085, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1394.8086419753085, "sum_squared": 1945491.1477290045, "min": 1394.8086419753085, "max": 1394.8086419753085, "mean": 1394.8086419753085, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 3.760884882491312, "sum_squared": 14.144255099351689, "min": 3.760884882491312, "max": 3.760884882491312, "mean": 3.760884882491312, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 3.760884882491312, "sum_squared": 14.144255099351689, "min": 3.760884882491312, "max": 3.760884882491312, "mean": 3.760884882491312, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.8516232281664361, "sum_squared": 0.7252621227526217, "min": 0.8516232281664361, "max": 0.8516232281664361, "mean": 0.8516232281664361, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.8516232281664361, "sum_squared": 0.7252621227526217, "min": 0.8516232281664361, "max": 0.8516232281664361, "mean": 0.8516232281664361, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.8516232281664361, "sum_squared": 0.7252621227526217, "min": 0.8516232281664361, "max": 0.8516232281664361, "mean": 0.8516232281664361, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=google_gemini-2.0-flash-001", "run_spec": { "name": "helpdesk_call_summarization:model=google_gemini-2.0-flash-001", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "google/gemini-2.0-flash-001", "model": "google/gemini-2.0-flash-001", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1394.8086419753085, "sum_squared": 1945491.1477290045, "min": 1394.8086419753085, "max": 1394.8086419753085, "mean": 1394.8086419753085, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 1.6605455904831121, "sum_squared": 2.7574116580729076, "min": 1.6605455904831121, "max": 1.6605455904831121, "mean": 1.6605455904831121, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1394.8086419753085, "sum_squared": 1945491.1477290045, "min": 1394.8086419753085, "max": 1394.8086419753085, "mean": 1394.8086419753085, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1394.8086419753085, "sum_squared": 1945491.1477290045, "min": 1394.8086419753085, "max": 1394.8086419753085, "mean": 1394.8086419753085, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.6605455904831121, "sum_squared": 2.7574116580729076, "min": 1.6605455904831121, "max": 1.6605455904831121, "mean": 1.6605455904831121, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.6605455904831121, "sum_squared": 2.7574116580729076, "min": 1.6605455904831121, "max": 1.6605455904831121, "mean": 1.6605455904831121, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.8513946044810221, "sum_squared": 0.7248727725393961, "min": 0.8513946044810221, "max": 0.8513946044810221, "mean": 0.8513946044810221, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.8513946044810221, "sum_squared": 0.7248727725393961, "min": 0.8513946044810221, "max": 0.8513946044810221, "mean": 0.8513946044810221, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.8513946044810221, "sum_squared": 0.7248727725393961, "min": 0.8513946044810221, "max": 0.8513946044810221, "mean": 0.8513946044810221, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=meta_llama-3.1-405b-instruct-turbo", "run_spec": { "name": "helpdesk_call_summarization:model=meta_llama-3.1-405b-instruct-turbo", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "together/llama-3.1-405b-instruct-turbo", "model": "meta/llama-3.1-405b-instruct-turbo", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1325.1604938271605, "sum_squared": 1756050.3344002438, "min": 1325.1604938271605, "max": 1325.1604938271605, "mean": 1325.1604938271605, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 176.6358024691358, "sum_squared": 31200.20671391556, "min": 176.6358024691358, "max": 176.6358024691358, "mean": 176.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 176.6358024691358, "sum_squared": 31200.20671391556, "min": 176.6358024691358, "max": 176.6358024691358, "mean": 176.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 3.870884820267006, "sum_squared": 14.983749291773533, "min": 3.870884820267006, "max": 3.870884820267006, "mean": 3.870884820267006, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": -25.32609970287818, "sum_squared": 641.4113261601264, "min": -25.32609970287818, "max": -25.32609970287818, "mean": -25.32609970287818, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 176.6358024691358, "sum_squared": 31200.20671391556, "min": 176.6358024691358, "max": 176.6358024691358, "mean": 176.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 941.4074074074074, "sum_squared": 886247.9067215363, "min": 941.4074074074074, "max": 941.4074074074074, "mean": 941.4074074074074, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "test" }, "count": 1, "sum": 1.154168683535722, "sum_squared": 1.3321053500545819, "min": 1.154168683535722, "max": 1.154168683535722, "mean": 1.154168683535722, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "test" }, "count": 1, "sum": 0.03881193005165035, "sum_squared": 0.0015063659143341997, "min": 0.03881193005165035, "max": 0.03881193005165035, "mean": 0.03881193005165035, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "test" }, "count": 1, "sum": -0.02690237988739125, "sum_squared": 0.0007237380436055133, "min": -0.02690237988739125, "max": -0.02690237988739125, "mean": -0.02690237988739125, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1325.1604938271605, "sum_squared": 1756050.3344002438, "min": 1325.1604938271605, "max": 1325.1604938271605, "mean": 1325.1604938271605, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1325.1604938271605, "sum_squared": 1756050.3344002438, "min": 1325.1604938271605, "max": 1325.1604938271605, "mean": 1325.1604938271605, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 176.6358024691358, "sum_squared": 31200.20671391556, "min": 176.6358024691358, "max": 176.6358024691358, "mean": 176.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 176.6358024691358, "sum_squared": 31200.20671391556, "min": 176.6358024691358, "max": 176.6358024691358, "mean": 176.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 176.6358024691358, "sum_squared": 31200.20671391556, "min": 176.6358024691358, "max": 176.6358024691358, "mean": 176.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 176.6358024691358, "sum_squared": 31200.20671391556, "min": 176.6358024691358, "max": 176.6358024691358, "mean": 176.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 3.870884820267006, "sum_squared": 14.983749291773533, "min": 3.870884820267006, "max": 3.870884820267006, "mean": 3.870884820267006, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 3.870884820267006, "sum_squared": 14.983749291773533, "min": 3.870884820267006, "max": 3.870884820267006, "mean": 3.870884820267006, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": -25.32609970287818, "sum_squared": 641.4113261601264, "min": -25.32609970287818, "max": -25.32609970287818, "mean": -25.32609970287818, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": -25.32609970287818, "sum_squared": 641.4113261601264, "min": -25.32609970287818, "max": -25.32609970287818, "mean": -25.32609970287818, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 176.6358024691358, "sum_squared": 31200.20671391556, "min": 176.6358024691358, "max": 176.6358024691358, "mean": 176.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 176.6358024691358, "sum_squared": 31200.20671391556, "min": 176.6358024691358, "max": 176.6358024691358, "mean": 176.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 941.4074074074074, "sum_squared": 886247.9067215363, "min": 941.4074074074074, "max": 941.4074074074074, "mean": 941.4074074074074, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 941.4074074074074, "sum_squared": 886247.9067215363, "min": 941.4074074074074, "max": 941.4074074074074, "mean": 941.4074074074074, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.8401920438957455, "sum_squared": 0.7059226706257103, "min": 0.8401920438957455, "max": 0.8401920438957455, "mean": 0.8401920438957455, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.8401920438957455, "sum_squared": 0.7059226706257103, "min": 0.8401920438957455, "max": 0.8401920438957455, "mean": 0.8401920438957455, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.8401920438957455, "sum_squared": 0.7059226706257103, "min": 0.8401920438957455, "max": 0.8401920438957455, "mean": 0.8401920438957455, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=meta_llama-3.1-70b-instruct-turbo", "run_spec": { "name": "helpdesk_call_summarization:model=meta_llama-3.1-70b-instruct-turbo", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "together/llama-3.1-70b-instruct-turbo", "model": "meta/llama-3.1-70b-instruct-turbo", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1325.1604938271605, "sum_squared": 1756050.3344002438, "min": 1325.1604938271605, "max": 1325.1604938271605, "mean": 1325.1604938271605, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 154.84567901234567, "sum_squared": 23977.18430879439, "min": 154.84567901234567, "max": 154.84567901234567, "mean": 154.84567901234567, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 154.84567901234567, "sum_squared": 23977.18430879439, "min": 154.84567901234567, "max": 154.84567901234567, "mean": 154.84567901234567, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 1.5525983895784543, "sum_squared": 2.41056175932161, "min": 1.5525983895784543, "max": 1.5525983895784543, "mean": 1.5525983895784543, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": -31.501275116797427, "sum_squared": 992.3303339841607, "min": -31.501275116797427, "max": -31.501275116797427, "mean": -31.501275116797427, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 154.84567901234567, "sum_squared": 23977.18430879439, "min": 154.84567901234567, "max": 154.84567901234567, "mean": 154.84567901234567, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 812.1604938271605, "sum_squared": 659604.6677335772, "min": 812.1604938271605, "max": 812.1604938271605, "mean": 812.1604938271605, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "test" }, "count": 1, "sum": 1.2256074251805338, "sum_squared": 1.5021135606576577, "min": 1.2256074251805338, "max": 1.2256074251805338, "mean": 1.2256074251805338, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "test" }, "count": 1, "sum": 0.055957823285054144, "sum_squared": 0.0031312779868013477, "min": 0.055957823285054144, "max": 0.055957823285054144, "mean": 0.055957823285054144, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "test" }, "count": 1, "sum": -0.038787007440306935, "sum_squared": 0.0015044319461744256, "min": -0.038787007440306935, "max": -0.038787007440306935, "mean": -0.038787007440306935, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1325.1604938271605, "sum_squared": 1756050.3344002438, "min": 1325.1604938271605, "max": 1325.1604938271605, "mean": 1325.1604938271605, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1325.1604938271605, "sum_squared": 1756050.3344002438, "min": 1325.1604938271605, "max": 1325.1604938271605, "mean": 1325.1604938271605, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 154.84567901234567, "sum_squared": 23977.18430879439, "min": 154.84567901234567, "max": 154.84567901234567, "mean": 154.84567901234567, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 154.84567901234567, "sum_squared": 23977.18430879439, "min": 154.84567901234567, "max": 154.84567901234567, "mean": 154.84567901234567, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 154.84567901234567, "sum_squared": 23977.18430879439, "min": 154.84567901234567, "max": 154.84567901234567, "mean": 154.84567901234567, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 154.84567901234567, "sum_squared": 23977.18430879439, "min": 154.84567901234567, "max": 154.84567901234567, "mean": 154.84567901234567, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.5525983895784543, "sum_squared": 2.41056175932161, "min": 1.5525983895784543, "max": 1.5525983895784543, "mean": 1.5525983895784543, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.5525983895784543, "sum_squared": 2.41056175932161, "min": 1.5525983895784543, "max": 1.5525983895784543, "mean": 1.5525983895784543, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": -31.501275116797427, "sum_squared": 992.3303339841607, "min": -31.501275116797427, "max": -31.501275116797427, "mean": -31.501275116797427, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": -31.501275116797427, "sum_squared": 992.3303339841607, "min": -31.501275116797427, "max": -31.501275116797427, "mean": -31.501275116797427, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 154.84567901234567, "sum_squared": 23977.18430879439, "min": 154.84567901234567, "max": 154.84567901234567, "mean": 154.84567901234567, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 154.84567901234567, "sum_squared": 23977.18430879439, "min": 154.84567901234567, "max": 154.84567901234567, "mean": 154.84567901234567, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 812.1604938271605, "sum_squared": 659604.6677335772, "min": 812.1604938271605, "max": 812.1604938271605, "mean": 812.1604938271605, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 812.1604938271605, "sum_squared": 659604.6677335772, "min": 812.1604938271605, "max": 812.1604938271605, "mean": 812.1604938271605, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.824874256973021, "sum_squared": 0.6804175398167934, "min": 0.824874256973021, "max": 0.824874256973021, "mean": 0.824874256973021, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.824874256973021, "sum_squared": 0.6804175398167934, "min": 0.824874256973021, "max": 0.824874256973021, "mean": 0.824874256973021, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.824874256973021, "sum_squared": 0.6804175398167934, "min": 0.824874256973021, "max": 0.824874256973021, "mean": 0.824874256973021, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=meta_llama-3.1-8b-instruct-turbo", "run_spec": { "name": "helpdesk_call_summarization:model=meta_llama-3.1-8b-instruct-turbo", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "together/llama-3.1-8b-instruct-turbo", "model": "meta/llama-3.1-8b-instruct-turbo", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1325.1604938271605, "sum_squared": 1756050.3344002438, "min": 1325.1604938271605, "max": 1325.1604938271605, "mean": 1325.1604938271605, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 174.98765432098764, "sum_squared": 30620.679164761466, "min": 174.98765432098764, "max": 174.98765432098764, "mean": 174.98765432098764, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 174.98765432098764, "sum_squared": 30620.679164761466, "min": 174.98765432098764, "max": 174.98765432098764, "mean": 174.98765432098764, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 1.1497825825655903, "sum_squared": 1.3219999871711985, "min": 1.1497825825655903, "max": 1.1497825825655903, "mean": 1.1497825825655903, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": -50.67078544615318, "sum_squared": 2567.5284977300894, "min": -50.67078544615318, "max": -50.67078544615318, "mean": -50.67078544615318, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 174.98765432098764, "sum_squared": 30620.679164761466, "min": 174.98765432098764, "max": 174.98765432098764, "mean": 174.98765432098764, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 910.9567901234568, "sum_squared": 829842.2734720317, "min": 910.9567901234568, "max": 910.9567901234568, "mean": 910.9567901234568, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "test" }, "count": 1, "sum": 1.3358499734248912, "sum_squared": 1.7844951514992826, "min": 1.3358499734248912, "max": 1.3358499734248912, "mean": 1.3358499734248912, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "test" }, "count": 1, "sum": 0.08024803335755003, "sum_squared": 0.006439746857754462, "min": 0.08024803335755003, "max": 0.08024803335755003, "mean": 0.08024803335755003, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "test" }, "count": 1, "sum": -0.05562369806726624, "sum_squared": 0.003093995786678398, "min": -0.05562369806726624, "max": -0.05562369806726624, "mean": -0.05562369806726624, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1325.1604938271605, "sum_squared": 1756050.3344002438, "min": 1325.1604938271605, "max": 1325.1604938271605, "mean": 1325.1604938271605, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1325.1604938271605, "sum_squared": 1756050.3344002438, "min": 1325.1604938271605, "max": 1325.1604938271605, "mean": 1325.1604938271605, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 174.98765432098764, "sum_squared": 30620.679164761466, "min": 174.98765432098764, "max": 174.98765432098764, "mean": 174.98765432098764, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 174.98765432098764, "sum_squared": 30620.679164761466, "min": 174.98765432098764, "max": 174.98765432098764, "mean": 174.98765432098764, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 174.98765432098764, "sum_squared": 30620.679164761466, "min": 174.98765432098764, "max": 174.98765432098764, "mean": 174.98765432098764, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 174.98765432098764, "sum_squared": 30620.679164761466, "min": 174.98765432098764, "max": 174.98765432098764, "mean": 174.98765432098764, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.1497825825655903, "sum_squared": 1.3219999871711985, "min": 1.1497825825655903, "max": 1.1497825825655903, "mean": 1.1497825825655903, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.1497825825655903, "sum_squared": 1.3219999871711985, "min": 1.1497825825655903, "max": 1.1497825825655903, "mean": 1.1497825825655903, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": -50.67078544615318, "sum_squared": 2567.5284977300894, "min": -50.67078544615318, "max": -50.67078544615318, "mean": -50.67078544615318, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": -50.67078544615318, "sum_squared": 2567.5284977300894, "min": -50.67078544615318, "max": -50.67078544615318, "mean": -50.67078544615318, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 174.98765432098764, "sum_squared": 30620.679164761466, "min": 174.98765432098764, "max": 174.98765432098764, "mean": 174.98765432098764, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 174.98765432098764, "sum_squared": 30620.679164761466, "min": 174.98765432098764, "max": 174.98765432098764, "mean": 174.98765432098764, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 910.9567901234568, "sum_squared": 829842.2734720317, "min": 910.9567901234568, "max": 910.9567901234568, "mean": 910.9567901234568, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 910.9567901234568, "sum_squared": 829842.2734720317, "min": 910.9567901234568, "max": 910.9567901234568, "mean": 910.9567901234568, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.7750342935528121, "sum_squared": 0.6006781561829064, "min": 0.7750342935528121, "max": 0.7750342935528121, "mean": 0.7750342935528121, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.7750342935528121, "sum_squared": 0.6006781561829064, "min": 0.7750342935528121, "max": 0.7750342935528121, "mean": 0.7750342935528121, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.7750342935528121, "sum_squared": 0.6006781561829064, "min": 0.7750342935528121, "max": 0.7750342935528121, "mean": 0.7750342935528121, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=mistralai_mistral-7b-instruct-v0.3", "run_spec": { "name": "helpdesk_call_summarization:model=mistralai_mistral-7b-instruct-v0.3", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "together/mistral-7b-instruct-v0.3", "model": "mistralai/mistral-7b-instruct-v0.3", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1491.9197530864199, "sum_squared": 2225824.549649444, "min": 1491.9197530864199, "max": 1491.9197530864199, "mean": 1491.9197530864199, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 167.8641975308642, "sum_squared": 28178.388812680994, "min": 167.8641975308642, "max": 167.8641975308642, "mean": 167.8641975308642, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 167.8641975308642, "sum_squared": 28178.388812680994, "min": 167.8641975308642, "max": 167.8641975308642, "mean": 167.8641975308642, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 1.234996128965307, "sum_squared": 1.5252154385592933, "min": 1.234996128965307, "max": 1.234996128965307, "mean": 1.234996128965307, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": -36.88174999418843, "sum_squared": 1360.2634826338185, "min": -36.88174999418843, "max": -36.88174999418843, "mean": -36.88174999418843, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 167.8641975308642, "sum_squared": 28178.388812680994, "min": 167.8641975308642, "max": 167.8641975308642, "mean": 167.8641975308642, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 801.9506172839506, "sum_squared": 643124.7925621094, "min": 801.9506172839506, "max": 801.9506172839506, "mean": 801.9506172839506, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "test" }, "count": 1, "sum": 1.2457177001161917, "sum_squared": 1.551812588382774, "min": 1.2457177001161917, "max": 1.2457177001161917, "mean": 1.2457177001161917, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "test" }, "count": 1, "sum": 0.06634961887819357, "sum_squared": 0.004402271925281541, "min": 0.06634961887819357, "max": 0.06634961887819357, "mean": 0.06634961887819357, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "test" }, "count": 1, "sum": -0.0459900512566468, "sum_squared": 0.0021150848145889997, "min": -0.0459900512566468, "max": -0.0459900512566468, "mean": -0.0459900512566468, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1491.9197530864199, "sum_squared": 2225824.549649444, "min": 1491.9197530864199, "max": 1491.9197530864199, "mean": 1491.9197530864199, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1491.9197530864199, "sum_squared": 2225824.549649444, "min": 1491.9197530864199, "max": 1491.9197530864199, "mean": 1491.9197530864199, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 167.8641975308642, "sum_squared": 28178.388812680994, "min": 167.8641975308642, "max": 167.8641975308642, "mean": 167.8641975308642, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 167.8641975308642, "sum_squared": 28178.388812680994, "min": 167.8641975308642, "max": 167.8641975308642, "mean": 167.8641975308642, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 167.8641975308642, "sum_squared": 28178.388812680994, "min": 167.8641975308642, "max": 167.8641975308642, "mean": 167.8641975308642, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 167.8641975308642, "sum_squared": 28178.388812680994, "min": 167.8641975308642, "max": 167.8641975308642, "mean": 167.8641975308642, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.234996128965307, "sum_squared": 1.5252154385592933, "min": 1.234996128965307, "max": 1.234996128965307, "mean": 1.234996128965307, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.234996128965307, "sum_squared": 1.5252154385592933, "min": 1.234996128965307, "max": 1.234996128965307, "mean": 1.234996128965307, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": -36.88174999418843, "sum_squared": 1360.2634826338185, "min": -36.88174999418843, "max": -36.88174999418843, "mean": -36.88174999418843, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": -36.88174999418843, "sum_squared": 1360.2634826338185, "min": -36.88174999418843, "max": -36.88174999418843, "mean": -36.88174999418843, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 167.8641975308642, "sum_squared": 28178.388812680994, "min": 167.8641975308642, "max": 167.8641975308642, "mean": 167.8641975308642, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 167.8641975308642, "sum_squared": 28178.388812680994, "min": 167.8641975308642, "max": 167.8641975308642, "mean": 167.8641975308642, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 801.9506172839506, "sum_squared": 643124.7925621094, "min": 801.9506172839506, "max": 801.9506172839506, "mean": 801.9506172839506, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 801.9506172839506, "sum_squared": 643124.7925621094, "min": 801.9506172839506, "max": 801.9506172839506, "mean": 801.9506172839506, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.7597165066300872, "sum_squared": 0.5771691704462233, "min": 0.7597165066300872, "max": 0.7597165066300872, "mean": 0.7597165066300872, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.7597165066300872, "sum_squared": 0.5771691704462233, "min": 0.7597165066300872, "max": 0.7597165066300872, "mean": 0.7597165066300872, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.7597165066300872, "sum_squared": 0.5771691704462233, "min": 0.7597165066300872, "max": 0.7597165066300872, "mean": 0.7597165066300872, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=mistralai_mixtral-8x22b-instruct-v0.1", "run_spec": { "name": "helpdesk_call_summarization:model=mistralai_mixtral-8x22b-instruct-v0.1", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "together/mixtral-8x22b-instruct-v0.1", "model": "mistralai/mixtral-8x22b-instruct-v0.1", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1491.9197530864199, "sum_squared": 2225824.549649444, "min": 1491.9197530864199, "max": 1491.9197530864199, "mean": 1491.9197530864199, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 150.08641975308643, "sum_squared": 22525.933394299653, "min": 150.08641975308643, "max": 150.08641975308643, "mean": 150.08641975308643, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 150.08641975308643, "sum_squared": 22525.933394299653, "min": 150.08641975308643, "max": 150.08641975308643, "mean": 150.08641975308643, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 2.455468370590681, "sum_squared": 6.029324918971254, "min": 2.455468370590681, "max": 2.455468370590681, "mean": 2.455468370590681, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": -33.760157285950115, "sum_squared": 1139.7482199720905, "min": -33.760157285950115, "max": -33.760157285950115, "mean": -33.760157285950115, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 150.08641975308643, "sum_squared": 22525.933394299653, "min": 150.08641975308643, "max": 150.08641975308643, "mean": 150.08641975308643, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 718.358024691358, "sum_squared": 516038.2516384697, "min": 718.358024691358, "max": 718.358024691358, "mean": 718.358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "test" }, "count": 1, "sum": 1.2522452264380484, "sum_squared": 1.568118107136879, "min": 1.2522452264380484, "max": 1.2522452264380484, "mean": 1.2522452264380484, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "test" }, "count": 1, "sum": 0.06780130495096504, "sum_squared": 0.0045970169530537555, "min": 0.06780130495096504, "max": 0.06780130495096504, "mean": 0.06780130495096504, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "test" }, "count": 1, "sum": -0.04699628336504647, "sum_squared": 0.0022086506501277437, "min": -0.04699628336504647, "max": -0.04699628336504647, "mean": -0.04699628336504647, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1491.9197530864199, "sum_squared": 2225824.549649444, "min": 1491.9197530864199, "max": 1491.9197530864199, "mean": 1491.9197530864199, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1491.9197530864199, "sum_squared": 2225824.549649444, "min": 1491.9197530864199, "max": 1491.9197530864199, "mean": 1491.9197530864199, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 150.08641975308643, "sum_squared": 22525.933394299653, "min": 150.08641975308643, "max": 150.08641975308643, "mean": 150.08641975308643, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 150.08641975308643, "sum_squared": 22525.933394299653, "min": 150.08641975308643, "max": 150.08641975308643, "mean": 150.08641975308643, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 150.08641975308643, "sum_squared": 22525.933394299653, "min": 150.08641975308643, "max": 150.08641975308643, "mean": 150.08641975308643, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 150.08641975308643, "sum_squared": 22525.933394299653, "min": 150.08641975308643, "max": 150.08641975308643, "mean": 150.08641975308643, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 2.455468370590681, "sum_squared": 6.029324918971254, "min": 2.455468370590681, "max": 2.455468370590681, "mean": 2.455468370590681, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 2.455468370590681, "sum_squared": 6.029324918971254, "min": 2.455468370590681, "max": 2.455468370590681, "mean": 2.455468370590681, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": -33.760157285950115, "sum_squared": 1139.7482199720905, "min": -33.760157285950115, "max": -33.760157285950115, "mean": -33.760157285950115, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": -33.760157285950115, "sum_squared": 1139.7482199720905, "min": -33.760157285950115, "max": -33.760157285950115, "mean": -33.760157285950115, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 150.08641975308643, "sum_squared": 22525.933394299653, "min": 150.08641975308643, "max": 150.08641975308643, "mean": 150.08641975308643, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 150.08641975308643, "sum_squared": 22525.933394299653, "min": 150.08641975308643, "max": 150.08641975308643, "mean": 150.08641975308643, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 718.358024691358, "sum_squared": 516038.2516384697, "min": 718.358024691358, "max": 718.358024691358, "mean": 718.358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 718.358024691358, "sum_squared": 516038.2516384697, "min": 718.358024691358, "max": 718.358024691358, "mean": 718.358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.8182441700960201, "sum_squared": 0.6695235218961247, "min": 0.8182441700960201, "max": 0.8182441700960201, "mean": 0.8182441700960201, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.8182441700960201, "sum_squared": 0.6695235218961247, "min": 0.8182441700960201, "max": 0.8182441700960201, "mean": 0.8182441700960201, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.8182441700960201, "sum_squared": 0.6695235218961247, "min": 0.8182441700960201, "max": 0.8182441700960201, "mean": 0.8182441700960201, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=mistralai_mixtral-8x7b-instruct-v0.1", "run_spec": { "name": "helpdesk_call_summarization:model=mistralai_mixtral-8x7b-instruct-v0.1", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "together/mixtral-8x7b-instruct-v0.1", "model": "mistralai/mixtral-8x7b-instruct-v0.1", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1491.9197530864199, "sum_squared": 2225824.549649444, "min": 1491.9197530864199, "max": 1491.9197530864199, "mean": 1491.9197530864199, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 204.59259259259258, "sum_squared": 41858.128943758566, "min": 204.59259259259258, "max": 204.59259259259258, "mean": 204.59259259259258, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 204.59259259259258, "sum_squared": 41858.128943758566, "min": 204.59259259259258, "max": 204.59259259259258, "mean": 204.59259259259258, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 2.228428923053506, "sum_squared": 4.96589546510141, "min": 2.228428923053506, "max": 2.228428923053506, "mean": 2.228428923053506, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": -20.934984658828295, "sum_squared": 438.27358266537607, "min": -20.934984658828295, "max": -20.934984658828295, "mean": -20.934984658828295, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 204.59259259259258, "sum_squared": 41858.128943758566, "min": 204.59259259259258, "max": 204.59259259259258, "mean": 204.59259259259258, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 974.7222222222222, "sum_squared": 950083.4104938271, "min": 974.7222222222222, "max": 974.7222222222222, "mean": 974.7222222222222, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "test" }, "count": 1, "sum": 1.107743687689098, "sum_squared": 1.227096077615042, "min": 1.107743687689098, "max": 1.107743687689098, "mean": 1.107743687689098, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "test" }, "count": 1, "sum": 0.030986057216916843, "sum_squared": 0.0009601357418500444, "min": 0.030986057216916843, "max": 0.030986057216916843, "mean": 0.030986057216916843, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "test" }, "count": 1, "sum": -0.021477898196575054, "sum_squared": 0.00046130011094244196, "min": -0.021477898196575054, "max": -0.021477898196575054, "mean": -0.021477898196575054, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1491.9197530864199, "sum_squared": 2225824.549649444, "min": 1491.9197530864199, "max": 1491.9197530864199, "mean": 1491.9197530864199, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1491.9197530864199, "sum_squared": 2225824.549649444, "min": 1491.9197530864199, "max": 1491.9197530864199, "mean": 1491.9197530864199, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 204.59259259259258, "sum_squared": 41858.128943758566, "min": 204.59259259259258, "max": 204.59259259259258, "mean": 204.59259259259258, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 204.59259259259258, "sum_squared": 41858.128943758566, "min": 204.59259259259258, "max": 204.59259259259258, "mean": 204.59259259259258, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 204.59259259259258, "sum_squared": 41858.128943758566, "min": 204.59259259259258, "max": 204.59259259259258, "mean": 204.59259259259258, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 204.59259259259258, "sum_squared": 41858.128943758566, "min": 204.59259259259258, "max": 204.59259259259258, "mean": 204.59259259259258, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 2.228428923053506, "sum_squared": 4.96589546510141, "min": 2.228428923053506, "max": 2.228428923053506, "mean": 2.228428923053506, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 2.228428923053506, "sum_squared": 4.96589546510141, "min": 2.228428923053506, "max": 2.228428923053506, "mean": 2.228428923053506, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": -20.934984658828295, "sum_squared": 438.27358266537607, "min": -20.934984658828295, "max": -20.934984658828295, "mean": -20.934984658828295, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": -20.934984658828295, "sum_squared": 438.27358266537607, "min": -20.934984658828295, "max": -20.934984658828295, "mean": -20.934984658828295, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 204.59259259259258, "sum_squared": 41858.128943758566, "min": 204.59259259259258, "max": 204.59259259259258, "mean": 204.59259259259258, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 204.59259259259258, "sum_squared": 41858.128943758566, "min": 204.59259259259258, "max": 204.59259259259258, "mean": 204.59259259259258, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 974.7222222222222, "sum_squared": 950083.4104938271, "min": 974.7222222222222, "max": 974.7222222222222, "mean": 974.7222222222222, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 974.7222222222222, "sum_squared": 950083.4104938271, "min": 974.7222222222222, "max": 974.7222222222222, "mean": 974.7222222222222, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.7624599908550526, "sum_squared": 0.5813452376546868, "min": 0.7624599908550526, "max": 0.7624599908550526, "mean": 0.7624599908550526, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.7624599908550526, "sum_squared": 0.5813452376546868, "min": 0.7624599908550526, "max": 0.7624599908550526, "mean": 0.7624599908550526, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.7624599908550526, "sum_squared": 0.5813452376546868, "min": 0.7624599908550526, "max": 0.7624599908550526, "mean": 0.7624599908550526, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=openai_gpt-4o-2024-11-20", "run_spec": { "name": "helpdesk_call_summarization:model=openai_gpt-4o-2024-11-20", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "openai/gpt-4o-2024-11-20", "model": "openai/gpt-4o-2024-11-20", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1298.9444444444443, "sum_squared": 1687256.669753086, "min": 1298.9444444444443, "max": 1298.9444444444443, "mean": 1298.9444444444443, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 177.4814814814815, "sum_squared": 31499.676268861458, "min": 177.4814814814815, "max": 177.4814814814815, "mean": 177.4814814814815, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 177.4814814814815, "sum_squared": 31499.676268861458, "min": 177.4814814814815, "max": 177.4814814814815, "mean": 177.4814814814815, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 4.624861865867803, "sum_squared": 21.38934727835822, "min": 4.624861865867803, "max": 4.624861865867803, "mean": 4.624861865867803, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 177.4814814814815, "sum_squared": 31499.676268861458, "min": 177.4814814814815, "max": 177.4814814814815, "mean": 177.4814814814815, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 992.3518518518518, "sum_squared": 984762.1978737997, "min": 992.3518518518518, "max": 992.3518518518518, "mean": 992.3518518518518, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1298.9444444444443, "sum_squared": 1687256.669753086, "min": 1298.9444444444443, "max": 1298.9444444444443, "mean": 1298.9444444444443, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1298.9444444444443, "sum_squared": 1687256.669753086, "min": 1298.9444444444443, "max": 1298.9444444444443, "mean": 1298.9444444444443, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 177.4814814814815, "sum_squared": 31499.676268861458, "min": 177.4814814814815, "max": 177.4814814814815, "mean": 177.4814814814815, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 177.4814814814815, "sum_squared": 31499.676268861458, "min": 177.4814814814815, "max": 177.4814814814815, "mean": 177.4814814814815, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 177.4814814814815, "sum_squared": 31499.676268861458, "min": 177.4814814814815, "max": 177.4814814814815, "mean": 177.4814814814815, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 177.4814814814815, "sum_squared": 31499.676268861458, "min": 177.4814814814815, "max": 177.4814814814815, "mean": 177.4814814814815, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 4.624861865867803, "sum_squared": 21.38934727835822, "min": 4.624861865867803, "max": 4.624861865867803, "mean": 4.624861865867803, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 4.624861865867803, "sum_squared": 21.38934727835822, "min": 4.624861865867803, "max": 4.624861865867803, "mean": 4.624861865867803, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 177.4814814814815, "sum_squared": 31499.676268861458, "min": 177.4814814814815, "max": 177.4814814814815, "mean": 177.4814814814815, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 177.4814814814815, "sum_squared": 31499.676268861458, "min": 177.4814814814815, "max": 177.4814814814815, "mean": 177.4814814814815, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 992.3518518518518, "sum_squared": 984762.1978737997, "min": 992.3518518518518, "max": 992.3518518518518, "mean": 992.3518518518518, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 992.3518518518518, "sum_squared": 984762.1978737997, "min": 992.3518518518518, "max": 992.3518518518518, "mean": 992.3518518518518, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.866941015089161, "sum_squared": 0.7515867236438248, "min": 0.866941015089161, "max": 0.866941015089161, "mean": 0.866941015089161, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.866941015089161, "sum_squared": 0.7515867236438248, "min": 0.866941015089161, "max": 0.866941015089161, "mean": 0.866941015089161, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.866941015089161, "sum_squared": 0.7515867236438248, "min": 0.866941015089161, "max": 0.866941015089161, "mean": 0.866941015089161, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=openai_gpt-4o-mini-2024-07-18", "run_spec": { "name": "helpdesk_call_summarization:model=openai_gpt-4o-mini-2024-07-18", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "openai/gpt-4o-mini-2024-07-18", "model": "openai/gpt-4o-mini-2024-07-18", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1298.9444444444443, "sum_squared": 1687256.669753086, "min": 1298.9444444444443, "max": 1298.9444444444443, "mean": 1298.9444444444443, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 161.12962962962962, "sum_squared": 25962.757544581615, "min": 161.12962962962962, "max": 161.12962962962962, "mean": 161.12962962962962, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 161.12962962962962, "sum_squared": 25962.757544581615, "min": 161.12962962962962, "max": 161.12962962962962, "mean": 161.12962962962962, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 3.0872432420283187, "sum_squared": 9.531070835449524, "min": 3.0872432420283187, "max": 3.0872432420283187, "mean": 3.0872432420283187, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 161.12962962962962, "sum_squared": 25962.757544581615, "min": 161.12962962962962, "max": 161.12962962962962, "mean": 161.12962962962962, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 901.8641975308642, "sum_squared": 813359.0307879896, "min": 901.8641975308642, "max": 901.8641975308642, "mean": 901.8641975308642, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1298.9444444444443, "sum_squared": 1687256.669753086, "min": 1298.9444444444443, "max": 1298.9444444444443, "mean": 1298.9444444444443, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1298.9444444444443, "sum_squared": 1687256.669753086, "min": 1298.9444444444443, "max": 1298.9444444444443, "mean": 1298.9444444444443, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 161.12962962962962, "sum_squared": 25962.757544581615, "min": 161.12962962962962, "max": 161.12962962962962, "mean": 161.12962962962962, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 161.12962962962962, "sum_squared": 25962.757544581615, "min": 161.12962962962962, "max": 161.12962962962962, "mean": 161.12962962962962, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 161.12962962962962, "sum_squared": 25962.757544581615, "min": 161.12962962962962, "max": 161.12962962962962, "mean": 161.12962962962962, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 161.12962962962962, "sum_squared": 25962.757544581615, "min": 161.12962962962962, "max": 161.12962962962962, "mean": 161.12962962962962, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 3.0872432420283187, "sum_squared": 9.531070835449524, "min": 3.0872432420283187, "max": 3.0872432420283187, "mean": 3.0872432420283187, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 3.0872432420283187, "sum_squared": 9.531070835449524, "min": 3.0872432420283187, "max": 3.0872432420283187, "mean": 3.0872432420283187, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 161.12962962962962, "sum_squared": 25962.757544581615, "min": 161.12962962962962, "max": 161.12962962962962, "mean": 161.12962962962962, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 161.12962962962962, "sum_squared": 25962.757544581615, "min": 161.12962962962962, "max": 161.12962962962962, "mean": 161.12962962962962, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 901.8641975308642, "sum_squared": 813359.0307879896, "min": 901.8641975308642, "max": 901.8641975308642, "mean": 901.8641975308642, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 901.8641975308642, "sum_squared": 813359.0307879896, "min": 901.8641975308642, "max": 901.8641975308642, "mean": 901.8641975308642, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.8463648834019184, "sum_squared": 0.716333515855943, "min": 0.8463648834019184, "max": 0.8463648834019184, "mean": 0.8463648834019184, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.8463648834019184, "sum_squared": 0.716333515855943, "min": 0.8463648834019184, "max": 0.8463648834019184, "mean": 0.8463648834019184, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.8463648834019184, "sum_squared": 0.716333515855943, "min": 0.8463648834019184, "max": 0.8463648834019184, "mean": 0.8463648834019184, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=qwen_qwen2.5-72b-instruct-turbo", "run_spec": { "name": "helpdesk_call_summarization:model=qwen_qwen2.5-72b-instruct-turbo", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "together/qwen2.5-72b-instruct-turbo", "model": "qwen/qwen2.5-72b-instruct-turbo", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1332.7777777777778, "sum_squared": 1776296.6049382717, "min": 1332.7777777777778, "max": 1332.7777777777778, "mean": 1332.7777777777778, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 130.6358024691358, "sum_squared": 17065.712886755067, "min": 130.6358024691358, "max": 130.6358024691358, "mean": 130.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 130.6358024691358, "sum_squared": 17065.712886755067, "min": 130.6358024691358, "max": 130.6358024691358, "mean": 130.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 2.4985873272389543, "sum_squared": 6.242938631839102, "min": 2.4985873272389543, "max": 2.4985873272389543, "mean": 2.4985873272389543, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": -21.667342511604403, "sum_squared": 469.4737315151794, "min": -21.667342511604403, "max": -21.667342511604403, "mean": -21.667342511604403, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 130.6358024691358, "sum_squared": 17065.712886755067, "min": 130.6358024691358, "max": 130.6358024691358, "mean": 130.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 699.0555555555555, "sum_squared": 488678.6697530864, "min": 699.0555555555555, "max": 699.0555555555555, "mean": 699.0555555555555, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "test" }, "count": 1, "sum": 1.180408631052669, "sum_squared": 1.3933645362636358, "min": 1.180408631052669, "max": 1.180408631052669, "mean": 1.180408631052669, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "test" }, "count": 1, "sum": 0.04471657129724368, "sum_squared": 0.001999571748581477, "min": 0.04471657129724368, "max": 0.04471657129724368, "mean": 0.04471657129724368, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "test" }, "count": 1, "sum": -0.03099516531899223, "sum_squared": 0.0009607002731516587, "min": -0.03099516531899223, "max": -0.03099516531899223, "mean": -0.03099516531899223, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1332.7777777777778, "sum_squared": 1776296.6049382717, "min": 1332.7777777777778, "max": 1332.7777777777778, "mean": 1332.7777777777778, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1332.7777777777778, "sum_squared": 1776296.6049382717, "min": 1332.7777777777778, "max": 1332.7777777777778, "mean": 1332.7777777777778, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 130.6358024691358, "sum_squared": 17065.712886755067, "min": 130.6358024691358, "max": 130.6358024691358, "mean": 130.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 130.6358024691358, "sum_squared": 17065.712886755067, "min": 130.6358024691358, "max": 130.6358024691358, "mean": 130.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 130.6358024691358, "sum_squared": 17065.712886755067, "min": 130.6358024691358, "max": 130.6358024691358, "mean": 130.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 130.6358024691358, "sum_squared": 17065.712886755067, "min": 130.6358024691358, "max": 130.6358024691358, "mean": 130.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 2.4985873272389543, "sum_squared": 6.242938631839102, "min": 2.4985873272389543, "max": 2.4985873272389543, "mean": 2.4985873272389543, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 2.4985873272389543, "sum_squared": 6.242938631839102, "min": 2.4985873272389543, "max": 2.4985873272389543, "mean": 2.4985873272389543, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": -21.667342511604403, "sum_squared": 469.4737315151794, "min": -21.667342511604403, "max": -21.667342511604403, "mean": -21.667342511604403, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": -21.667342511604403, "sum_squared": 469.4737315151794, "min": -21.667342511604403, "max": -21.667342511604403, "mean": -21.667342511604403, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 130.6358024691358, "sum_squared": 17065.712886755067, "min": 130.6358024691358, "max": 130.6358024691358, "mean": 130.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 130.6358024691358, "sum_squared": 17065.712886755067, "min": 130.6358024691358, "max": 130.6358024691358, "mean": 130.6358024691358, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 699.0555555555555, "sum_squared": 488678.6697530864, "min": 699.0555555555555, "max": 699.0555555555555, "mean": 699.0555555555555, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 699.0555555555555, "sum_squared": 488678.6697530864, "min": 699.0555555555555, "max": 699.0555555555555, "mean": 699.0555555555555, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.8481938728852292, "sum_squared": 0.7194328460000443, "min": 0.8481938728852292, "max": 0.8481938728852292, "mean": 0.8481938728852292, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.8481938728852292, "sum_squared": 0.7194328460000443, "min": 0.8481938728852292, "max": 0.8481938728852292, "mean": 0.8481938728852292, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.8481938728852292, "sum_squared": 0.7194328460000443, "min": 0.8481938728852292, "max": 0.8481938728852292, "mean": 0.8481938728852292, "variance": 0.0, "stddev": 0.0 } ] }, { "run_path": "benchmark_output/runs/v1.0.0/helpdesk_call_summarization:model=qwen_qwen2.5-7b-instruct-turbo", "run_spec": { "name": "helpdesk_call_summarization:model=qwen_qwen2.5-7b-instruct-turbo", "scenario_spec": { "class_name": "helm.benchmark.scenarios.helpdesk_call_summarization_scenario.HelpdeskCallSummarizationScenario", "args": {} }, "adapter_spec": { "method": "generation", "global_prefix": "", "global_suffix": "", "instructions": "The following is a call transcript of a call between a compnay's employee and the company's IT helpdesk. Summarize the call transcript in under 200 words.", "input_prefix": "### Call Transcript\n", "input_suffix": "", "reference_prefix": "A. ", "reference_suffix": "\n", "chain_of_thought_prefix": "", "chain_of_thought_suffix": "\n", "output_prefix": "", "output_suffix": "", "instance_prefix": "\n", "substitutions": [], "max_train_instances": 0, "max_eval_instances": 10000, "num_outputs": 1, "num_train_trials": 1, "num_trials": 1, "sample_train": true, "model_deployment": "together/qwen2.5-7b-instruct-turbo", "model": "qwen/qwen2.5-7b-instruct-turbo", "temperature": 0.0, "max_tokens": 512, "stop_sequences": [], "multi_label": false }, "metric_specs": [ { "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric", "args": { "names": [] } }, { "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric", "args": {} }, { "class_name": "helm.benchmark.metrics.helpdesk_call_summarization_metrics.HelpdeskCallSummarizationMetric", "args": {} } ], "data_augmenter_spec": { "perturbation_specs": [], "should_augment_train_instances": false, "should_include_original_train": false, "should_skip_unchanged_train": false, "should_augment_eval_instances": false, "should_include_original_eval": false, "should_skip_unchanged_eval": false, "seeds_per_instance": 1 }, "groups": [ "helpdesk_call_summarization" ], "annotators": [ { "class_name": "helm.benchmark.annotation.helpdesk_call_summarization_annotator.HelpdeskCallSummarizationAnnotator", "args": {} } ] }, "stats": [ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1332.7777777777778, "sum_squared": 1776296.6049382717, "min": 1332.7777777777778, "max": 1332.7777777777778, "mean": 1332.7777777777778, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 117.24074074074075, "sum_squared": 13745.391289437588, "min": 117.24074074074075, "max": 117.24074074074075, "mean": 117.24074074074075, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 117.24074074074075, "sum_squared": 13745.391289437588, "min": 117.24074074074075, "max": 117.24074074074075, "mean": 117.24074074074075, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0.0, "sum_squared": 0.0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 0.9551631962811505, "sum_squared": 0.9123367315300237, "min": 0.9551631962811505, "max": 0.9551631962811505, "mean": 0.9551631962811505, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": -35.09634890129903, "sum_squared": 1231.7537062017136, "min": -35.09634890129903, "max": -35.09634890129903, "mean": -35.09634890129903, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 117.24074074074075, "sum_squared": 13745.391289437588, "min": 117.24074074074075, "max": 117.24074074074075, "mean": 117.24074074074075, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 607.3888888888889, "sum_squared": 368921.26234567905, "min": 607.3888888888889, "max": 607.3888888888889, "mean": 607.3888888888889, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "test" }, "count": 1, "sum": 1.3489855178254333, "sum_squared": 1.8197619273027523, "min": 1.3489855178254333, "max": 1.3489855178254333, "mean": 1.3489855178254333, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "test" }, "count": 1, "sum": 0.08336228969521932, "sum_squared": 0.006949271343229669, "min": 0.08336228969521932, "max": 0.08336228969521932, "mean": 0.08336228969521932, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "test" }, "count": 1, "sum": -0.05778233606726265, "sum_squared": 0.003338798361390082, "min": -0.05778233606726265, "max": -0.05778233606726265, "mean": -0.05778233606726265, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1332.7777777777778, "sum_squared": 1776296.6049382717, "min": 1332.7777777777778, "max": 1332.7777777777778, "mean": 1332.7777777777778, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1332.7777777777778, "sum_squared": 1776296.6049382717, "min": 1332.7777777777778, "max": 1332.7777777777778, "mean": 1332.7777777777778, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 117.24074074074075, "sum_squared": 13745.391289437588, "min": 117.24074074074075, "max": 117.24074074074075, "mean": 117.24074074074075, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 117.24074074074075, "sum_squared": 13745.391289437588, "min": 117.24074074074075, "max": 117.24074074074075, "mean": 117.24074074074075, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 117.24074074074075, "sum_squared": 13745.391289437588, "min": 117.24074074074075, "max": 117.24074074074075, "mean": 117.24074074074075, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 117.24074074074075, "sum_squared": 13745.391289437588, "min": 117.24074074074075, "max": 117.24074074074075, "mean": 117.24074074074075, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.9551631962811505, "sum_squared": 0.9123367315300237, "min": 0.9551631962811505, "max": 0.9551631962811505, "mean": 0.9551631962811505, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.9551631962811505, "sum_squared": 0.9123367315300237, "min": 0.9551631962811505, "max": 0.9551631962811505, "mean": 0.9551631962811505, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": -35.09634890129903, "sum_squared": 1231.7537062017136, "min": -35.09634890129903, "max": -35.09634890129903, "mean": -35.09634890129903, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": -35.09634890129903, "sum_squared": 1231.7537062017136, "min": -35.09634890129903, "max": -35.09634890129903, "mean": -35.09634890129903, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 117.24074074074075, "sum_squared": 13745.391289437588, "min": 117.24074074074075, "max": 117.24074074074075, "mean": 117.24074074074075, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 117.24074074074075, "sum_squared": 13745.391289437588, "min": 117.24074074074075, "max": 117.24074074074075, "mean": 117.24074074074075, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 607.3888888888889, "sum_squared": 368921.26234567905, "min": 607.3888888888889, "max": 607.3888888888889, "mean": 607.3888888888889, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 607.3888888888889, "sum_squared": 368921.26234567905, "min": 607.3888888888889, "max": 607.3888888888889, "mean": 607.3888888888889, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 162.0, "sum_squared": 26244.0, "min": 162.0, "max": 162.0, "mean": 162.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test" }, "count": 1, "sum": 0.7583447645176039, "sum_squared": 0.5750867818712602, "min": 0.7583447645176039, "max": 0.7583447645176039, "mean": 0.7583447645176039, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.7583447645176039, "sum_squared": 0.5750867818712602, "min": 0.7583447645176039, "max": 0.7583447645176039, "mean": 0.7583447645176039, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "call_summarization_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.7583447645176039, "sum_squared": 0.5750867818712602, "min": 0.7583447645176039, "max": 0.7583447645176039, "mean": 0.7583447645176039, "variance": 0.0, "stddev": 0.0 } ] } ]