[ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 2.0, "sum_squared": 4.0, "min": 2.0, "max": 2.0, "mean": 2.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 3671.4615384615386, "sum_squared": 13479629.828402368, "min": 3671.4615384615386, "max": 3671.4615384615386, "mean": 3671.4615384615386, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0, "sum_squared": 0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0, "sum_squared": 0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 1.4449573846963735, "sum_squared": 2.0879018435885834, "min": 1.4449573846963735, "max": 1.4449573846963735, "mean": 1.4449573846963735, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 4.358974358974359, "sum_squared": 19.00065746219592, "min": 4.358974358974359, "max": 4.358974358974359, "mean": 4.358974358974359, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "max_prob", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "exact_match", "split": "test" }, "count": 1, "sum": 0.3333333333333333, "sum_squared": 0.1111111111111111, "min": 0.3333333333333333, "max": 0.3333333333333333, "mean": 0.3333333333333333, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "quasi_exact_match", "split": "test" }, "count": 1, "sum": 0.48717948717948717, "sum_squared": 0.2373438527284681, "min": 0.48717948717948717, "max": 0.48717948717948717, "mean": 0.48717948717948717, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "f1_score", "split": "test" }, "count": 1, "sum": 0.7104635653040035, "sum_squared": 0.504758477624476, "min": 0.7104635653040035, "max": 0.7104635653040035, "mean": 0.7104635653040035, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "rouge_l", "split": "test" }, "count": 1, "sum": 0.7098909039623417, "sum_squared": 0.5039450955284706, "min": 0.7098909039623417, "max": 0.7098909039623417, "mean": 0.7098909039623417, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_1", "split": "test" }, "count": 1, "sum": 0.5624194727683055, "sum_squared": 0.31631566334897876, "min": 0.5624194727683055, "max": 0.5624194727683055, "mean": 0.5624194727683055, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_4", "split": "test" }, "count": 1, "sum": 0.05738985976942292, "sum_squared": 0.0032935960043540278, "min": 0.05738985976942292, "max": 0.05738985976942292, "mean": 0.05738985976942292, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 29.743589743589745, "sum_squared": 884.681130834977, "min": 29.743589743589745, "max": 29.743589743589745, "mean": 29.743589743589745, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "valid" }, "count": 1, "sum": 2.0, "sum_squared": 4.0, "min": 2.0, "max": 2.0, "mean": 2.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "valid" }, "count": 1, "sum": 3593.3636363636365, "sum_squared": 12912262.223140497, "min": 3593.3636363636365, "max": 3593.3636363636365, "mean": 3593.3636363636365, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "valid" }, "count": 0, "sum": 0, "sum_squared": 0 }, { "name": { "name": "training_energy_cost", "split": "valid" }, "count": 0, "sum": 0, "sum_squared": 0 }, { "name": { "name": "inference_runtime", "split": "valid" }, "count": 1, "sum": 1.4895370223305442, "sum_squared": 2.218720540893344, "min": 1.4895370223305442, "max": 1.4895370223305442, "mean": 1.4895370223305442, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "valid" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "valid" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "valid" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "valid" }, "count": 1, "sum": 4.2727272727272725, "sum_squared": 18.256198347107436, "min": 4.2727272727272725, "max": 4.2727272727272725, "mean": 4.2727272727272725, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "valid" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "max_prob", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "exact_match", "split": "valid" }, "count": 1, "sum": 0.2727272727272727, "sum_squared": 0.07438016528925619, "min": 0.2727272727272727, "max": 0.2727272727272727, "mean": 0.2727272727272727, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "quasi_exact_match", "split": "valid" }, "count": 1, "sum": 0.5454545454545454, "sum_squared": 0.29752066115702475, "min": 0.5454545454545454, "max": 0.5454545454545454, "mean": 0.5454545454545454, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "f1_score", "split": "valid" }, "count": 1, "sum": 0.7696969696969697, "sum_squared": 0.5924334251606979, "min": 0.7696969696969697, "max": 0.7696969696969697, "mean": 0.7696969696969697, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "rouge_l", "split": "valid" }, "count": 1, "sum": 0.8405483405483406, "sum_squared": 0.7065215127985691, "min": 0.8405483405483406, "max": 0.8405483405483406, "mean": 0.8405483405483406, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_1", "split": "valid" }, "count": 1, "sum": 0.5726611281552104, "sum_squared": 0.32794076769999836, "min": 0.5726611281552104, "max": 0.5726611281552104, "mean": 0.5726611281552104, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_4", "split": "valid" }, "count": 1, "sum": 0.1496071529837806, "sum_squared": 0.022382300223912334, "min": 0.1496071529837806, "max": 0.1496071529837806, "mean": 0.1496071529837806, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "valid" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "valid" }, "count": 1, "sum": 17.818181818181817, "sum_squared": 317.48760330578506, "min": 17.818181818181817, "max": 17.818181818181817, "mean": 17.818181818181817, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "valid" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "valid" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 2.0, "sum_squared": 4.0, "min": 2.0, "max": 2.0, "mean": 2.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 2.0, "sum_squared": 4.0, "min": 2.0, "max": 2.0, "mean": 2.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 3671.4615384615386, "sum_squared": 13479629.828402368, "min": 3671.4615384615386, "max": 3671.4615384615386, "mean": 3671.4615384615386, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 3671.4615384615386, "sum_squared": 13479629.828402368, "min": 3671.4615384615386, "max": 3671.4615384615386, "mean": 3671.4615384615386, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.4449573846963735, "sum_squared": 2.0879018435885834, "min": 1.4449573846963735, "max": 1.4449573846963735, "mean": 1.4449573846963735, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.4449573846963735, "sum_squared": 2.0879018435885834, "min": 1.4449573846963735, "max": 1.4449573846963735, "mean": 1.4449573846963735, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 4.358974358974359, "sum_squared": 19.00065746219592, "min": 4.358974358974359, "max": 4.358974358974359, "mean": 4.358974358974359, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 4.358974358974359, "sum_squared": 19.00065746219592, "min": 4.358974358974359, "max": 4.358974358974359, "mean": 4.358974358974359, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "max_prob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "max_prob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "exact_match", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.3333333333333333, "sum_squared": 0.1111111111111111, "min": 0.3333333333333333, "max": 0.3333333333333333, "mean": 0.3333333333333333, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "exact_match", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.3333333333333333, "sum_squared": 0.1111111111111111, "min": 0.3333333333333333, "max": 0.3333333333333333, "mean": 0.3333333333333333, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "quasi_exact_match", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.48717948717948717, "sum_squared": 0.2373438527284681, "min": 0.48717948717948717, "max": 0.48717948717948717, "mean": 0.48717948717948717, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "quasi_exact_match", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.48717948717948717, "sum_squared": 0.2373438527284681, "min": 0.48717948717948717, "max": 0.48717948717948717, "mean": 0.48717948717948717, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "f1_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.7104635653040035, "sum_squared": 0.504758477624476, "min": 0.7104635653040035, "max": 0.7104635653040035, "mean": 0.7104635653040035, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "f1_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.7104635653040035, "sum_squared": 0.504758477624476, "min": 0.7104635653040035, "max": 0.7104635653040035, "mean": 0.7104635653040035, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "rouge_l", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.7098909039623417, "sum_squared": 0.5039450955284706, "min": 0.7098909039623417, "max": 0.7098909039623417, "mean": 0.7098909039623417, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "rouge_l", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.7098909039623417, "sum_squared": 0.5039450955284706, "min": 0.7098909039623417, "max": 0.7098909039623417, "mean": 0.7098909039623417, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_1", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.5624194727683055, "sum_squared": 0.31631566334897876, "min": 0.5624194727683055, "max": 0.5624194727683055, "mean": 0.5624194727683055, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_1", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.5624194727683055, "sum_squared": 0.31631566334897876, "min": 0.5624194727683055, "max": 0.5624194727683055, "mean": 0.5624194727683055, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_4", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.05738985976942292, "sum_squared": 0.0032935960043540278, "min": 0.05738985976942292, "max": 0.05738985976942292, "mean": 0.05738985976942292, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_4", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.05738985976942292, "sum_squared": 0.0032935960043540278, "min": 0.05738985976942292, "max": 0.05738985976942292, "mean": 0.05738985976942292, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 29.743589743589745, "sum_squared": 884.681130834977, "min": 29.743589743589745, "max": 29.743589743589745, "mean": 29.743589743589745, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 29.743589743589745, "sum_squared": 884.681130834977, "min": 29.743589743589745, "max": 29.743589743589745, "mean": 29.743589743589745, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 2.0, "sum_squared": 4.0, "min": 2.0, "max": 2.0, "mean": 2.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 2.0, "sum_squared": 4.0, "min": 2.0, "max": 2.0, "mean": 2.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 3593.3636363636365, "sum_squared": 12912262.223140497, "min": 3593.3636363636365, "max": 3593.3636363636365, "mean": 3593.3636363636365, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 3593.3636363636365, "sum_squared": 12912262.223140497, "min": 3593.3636363636365, "max": 3593.3636363636365, "mean": 3593.3636363636365, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.4895370223305442, "sum_squared": 2.218720540893344, "min": 1.4895370223305442, "max": 1.4895370223305442, "mean": 1.4895370223305442, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.4895370223305442, "sum_squared": 2.218720540893344, "min": 1.4895370223305442, "max": 1.4895370223305442, "mean": 1.4895370223305442, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 4.2727272727272725, "sum_squared": 18.256198347107436, "min": 4.2727272727272725, "max": 4.2727272727272725, "mean": 4.2727272727272725, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 4.2727272727272725, "sum_squared": 18.256198347107436, "min": 4.2727272727272725, "max": 4.2727272727272725, "mean": 4.2727272727272725, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "max_prob", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "max_prob", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "exact_match", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.2727272727272727, "sum_squared": 0.07438016528925619, "min": 0.2727272727272727, "max": 0.2727272727272727, "mean": 0.2727272727272727, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "exact_match", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.2727272727272727, "sum_squared": 0.07438016528925619, "min": 0.2727272727272727, "max": 0.2727272727272727, "mean": 0.2727272727272727, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "quasi_exact_match", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.5454545454545454, "sum_squared": 0.29752066115702475, "min": 0.5454545454545454, "max": 0.5454545454545454, "mean": 0.5454545454545454, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "quasi_exact_match", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.5454545454545454, "sum_squared": 0.29752066115702475, "min": 0.5454545454545454, "max": 0.5454545454545454, "mean": 0.5454545454545454, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "f1_score", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.7696969696969697, "sum_squared": 0.5924334251606979, "min": 0.7696969696969697, "max": 0.7696969696969697, "mean": 0.7696969696969697, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "f1_score", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.7696969696969697, "sum_squared": 0.5924334251606979, "min": 0.7696969696969697, "max": 0.7696969696969697, "mean": 0.7696969696969697, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "rouge_l", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.8405483405483406, "sum_squared": 0.7065215127985691, "min": 0.8405483405483406, "max": 0.8405483405483406, "mean": 0.8405483405483406, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "rouge_l", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.8405483405483406, "sum_squared": 0.7065215127985691, "min": 0.8405483405483406, "max": 0.8405483405483406, "mean": 0.8405483405483406, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_1", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.5726611281552104, "sum_squared": 0.32794076769999836, "min": 0.5726611281552104, "max": 0.5726611281552104, "mean": 0.5726611281552104, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_1", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.5726611281552104, "sum_squared": 0.32794076769999836, "min": 0.5726611281552104, "max": 0.5726611281552104, "mean": 0.5726611281552104, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_4", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.1496071529837806, "sum_squared": 0.022382300223912334, "min": 0.1496071529837806, "max": 0.1496071529837806, "mean": 0.1496071529837806, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_4", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.1496071529837806, "sum_squared": 0.022382300223912334, "min": 0.1496071529837806, "max": 0.1496071529837806, "mean": 0.1496071529837806, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 17.818181818181817, "sum_squared": 317.48760330578506, "min": 17.818181818181817, "max": 17.818181818181817, "mean": 17.818181818181817, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 17.818181818181817, "sum_squared": 317.48760330578506, "min": 17.818181818181817, "max": 17.818181818181817, "mean": 17.818181818181817, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 39.0, "sum_squared": 1521.0, "min": 39.0, "max": 39.0, "mean": 39.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "valid" }, "count": 1, "sum": 11.0, "sum_squared": 121.0, "min": 11.0, "max": 11.0, "mean": 11.0, "variance": 0.0, "stddev": 0.0 } ]