[ { "name": { "name": "num_references", "split": "test" }, "count": 1, "sum": 2.0, "sum_squared": 4.0, "min": 2.0, "max": 2.0, "mean": 2.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test" }, "count": 1, "sum": 1717.6666666666667, "sum_squared": 2950378.777777778, "min": 1717.6666666666667, "max": 1717.6666666666667, "mean": 1717.6666666666667, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "test" }, "count": 0, "sum": 0, "sum_squared": 0 }, { "name": { "name": "training_energy_cost", "split": "test" }, "count": 0, "sum": 0, "sum_squared": 0 }, { "name": { "name": "inference_runtime", "split": "test" }, "count": 1, "sum": 0.5483537698403383, "sum_squared": 0.30069185689811073, "min": 0.5483537698403383, "max": 0.5483537698403383, "mean": 0.5483537698403383, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test" }, "count": 1, "sum": 2.051282051282051, "sum_squared": 4.207758053911899, "min": 2.051282051282051, "max": 2.051282051282051, "mean": 2.051282051282051, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "max_prob", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "exact_match", "split": "test" }, "count": 1, "sum": 0.1794871794871795, "sum_squared": 0.032215647600262985, "min": 0.1794871794871795, "max": 0.1794871794871795, "mean": 0.1794871794871795, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "quasi_exact_match", "split": "test" }, "count": 1, "sum": 0.38461538461538464, "sum_squared": 0.14792899408284024, "min": 0.38461538461538464, "max": 0.38461538461538464, "mean": 0.38461538461538464, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "f1_score", "split": "test" }, "count": 1, "sum": 0.6404732271248108, "sum_squared": 0.4102059546636695, "min": 0.6404732271248108, "max": 0.6404732271248108, "mean": 0.6404732271248108, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "rouge_l", "split": "test" }, "count": 1, "sum": 0.6533316635551568, "sum_squared": 0.42684226260374863, "min": 0.6533316635551568, "max": 0.6533316635551568, "mean": 0.6533316635551568, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_1", "split": "test" }, "count": 1, "sum": 0.4281173133480012, "sum_squared": 0.18328443398831062, "min": 0.4281173133480012, "max": 0.4281173133480012, "mean": 0.4281173133480012, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_4", "split": "test" }, "count": 1, "sum": 0.03874318458569219, "sum_squared": 0.0015010343518410169, "min": 0.03874318458569219, "max": 0.03874318458569219, "mean": 0.03874318458569219, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test" }, "count": 1, "sum": 25.82051282051282, "sum_squared": 666.6988823142669, "min": 25.82051282051282, "max": 25.82051282051282, "mean": 25.82051282051282, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "valid" }, "count": 1, "sum": 2.0, "sum_squared": 4.0, "min": 2.0, "max": 2.0, "mean": 2.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "valid" }, "count": 1, "sum": 1721.909090909091, "sum_squared": 2964970.917355372, "min": 1721.909090909091, "max": 1721.909090909091, "mean": 1721.909090909091, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "training_co2_cost", "split": "valid" }, "count": 0, "sum": 0, "sum_squared": 0 }, { "name": { "name": "training_energy_cost", "split": "valid" }, "count": 0, "sum": 0, "sum_squared": 0 }, { "name": { "name": "inference_runtime", "split": "valid" }, "count": 1, "sum": 0.5653736157850786, "sum_squared": 0.3196473254258937, "min": 0.5653736157850786, "max": 0.5653736157850786, "mean": 0.5653736157850786, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "valid" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "valid" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "valid" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "valid" }, "count": 1, "sum": 2.090909090909091, "sum_squared": 4.37190082644628, "min": 2.090909090909091, "max": 2.090909090909091, "mean": 2.090909090909091, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "valid" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "max_prob", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "exact_match", "split": "valid" }, "count": 1, "sum": 0.18181818181818182, "sum_squared": 0.03305785123966942, "min": 0.18181818181818182, "max": 0.18181818181818182, "mean": 0.18181818181818182, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "quasi_exact_match", "split": "valid" }, "count": 1, "sum": 0.45454545454545453, "sum_squared": 0.20661157024793386, "min": 0.45454545454545453, "max": 0.45454545454545453, "mean": 0.45454545454545453, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "f1_score", "split": "valid" }, "count": 1, "sum": 0.7575757575757577, "sum_squared": 0.5739210284664832, "min": 0.7575757575757577, "max": 0.7575757575757577, "mean": 0.7575757575757577, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "rouge_l", "split": "valid" }, "count": 1, "sum": 0.7534271284271284, "sum_squared": 0.5676524378499487, "min": 0.7534271284271284, "max": 0.7534271284271284, "mean": 0.7534271284271284, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_1", "split": "valid" }, "count": 1, "sum": 0.5103042628099558, "sum_squared": 0.2604104406420124, "min": 0.5103042628099558, "max": 0.5103042628099558, "mean": 0.5103042628099558, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_4", "split": "valid" }, "count": 1, "sum": 0.116254616642855, "sum_squared": 0.013515135890777179, "min": 0.116254616642855, "max": 0.116254616642855, "mean": 0.116254616642855, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "valid" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "valid" }, "count": 1, "sum": 26.09090909090909, "sum_squared": 680.7355371900826, "min": 26.09090909090909, "max": 26.09090909090909, "mean": 26.09090909090909, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "test" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "test" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "perplexity", "split": "valid" }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bits_per_byte", "split": "valid" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob_per_byte", "split": "valid" }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 2.0, "sum_squared": 4.0, "min": 2.0, "max": 2.0, "mean": 2.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 2.0, "sum_squared": 4.0, "min": 2.0, "max": 2.0, "mean": 2.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1717.6666666666667, "sum_squared": 2950378.777777778, "min": 1717.6666666666667, "max": 1717.6666666666667, "mean": 1717.6666666666667, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1717.6666666666667, "sum_squared": 2950378.777777778, "min": 1717.6666666666667, "max": 1717.6666666666667, "mean": 1717.6666666666667, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.5483537698403383, "sum_squared": 0.30069185689811073, "min": 0.5483537698403383, "max": 0.5483537698403383, "mean": 0.5483537698403383, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.5483537698403383, "sum_squared": 0.30069185689811073, "min": 0.5483537698403383, "max": 0.5483537698403383, "mean": 0.5483537698403383, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 2.051282051282051, "sum_squared": 4.207758053911899, "min": 2.051282051282051, "max": 2.051282051282051, "mean": 2.051282051282051, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 2.051282051282051, "sum_squared": 4.207758053911899, "min": 2.051282051282051, "max": 2.051282051282051, "mean": 2.051282051282051, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "max_prob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "max_prob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "exact_match", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.1794871794871795, "sum_squared": 0.032215647600262985, "min": 0.1794871794871795, "max": 0.1794871794871795, "mean": 0.1794871794871795, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "exact_match", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.1794871794871795, "sum_squared": 0.032215647600262985, "min": 0.1794871794871795, "max": 0.1794871794871795, "mean": 0.1794871794871795, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "quasi_exact_match", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.38461538461538464, "sum_squared": 0.14792899408284024, "min": 0.38461538461538464, "max": 0.38461538461538464, "mean": 0.38461538461538464, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "quasi_exact_match", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.38461538461538464, "sum_squared": 0.14792899408284024, "min": 0.38461538461538464, "max": 0.38461538461538464, "mean": 0.38461538461538464, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "f1_score", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.6404732271248108, "sum_squared": 0.4102059546636695, "min": 0.6404732271248108, "max": 0.6404732271248108, "mean": 0.6404732271248108, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "f1_score", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.6404732271248108, "sum_squared": 0.4102059546636695, "min": 0.6404732271248108, "max": 0.6404732271248108, "mean": 0.6404732271248108, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "rouge_l", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.6533316635551568, "sum_squared": 0.42684226260374863, "min": 0.6533316635551568, "max": 0.6533316635551568, "mean": 0.6533316635551568, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "rouge_l", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.6533316635551568, "sum_squared": 0.42684226260374863, "min": 0.6533316635551568, "max": 0.6533316635551568, "mean": 0.6533316635551568, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_1", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.4281173133480012, "sum_squared": 0.18328443398831062, "min": 0.4281173133480012, "max": 0.4281173133480012, "mean": 0.4281173133480012, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_1", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.4281173133480012, "sum_squared": 0.18328443398831062, "min": 0.4281173133480012, "max": 0.4281173133480012, "mean": 0.4281173133480012, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_4", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.03874318458569219, "sum_squared": 0.0015010343518410169, "min": 0.03874318458569219, "max": 0.03874318458569219, "mean": 0.03874318458569219, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_4", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.03874318458569219, "sum_squared": 0.0015010343518410169, "min": 0.03874318458569219, "max": 0.03874318458569219, "mean": 0.03874318458569219, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 25.82051282051282, "sum_squared": 666.6988823142669, "min": 25.82051282051282, "max": 25.82051282051282, "mean": 25.82051282051282, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "test", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 25.82051282051282, "sum_squared": 666.6988823142669, "min": 25.82051282051282, "max": 25.82051282051282, "mean": 25.82051282051282, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 2.0, "sum_squared": 4.0, "min": 2.0, "max": 2.0, "mean": 2.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_references", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 2.0, "sum_squared": 4.0, "min": 2.0, "max": 2.0, "mean": 2.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_trials", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1721.909090909091, "sum_squared": 2964970.917355372, "min": 1721.909090909091, "max": 1721.909090909091, "mean": 1721.909090909091, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_prompt_tokens", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1721.909090909091, "sum_squared": 2964970.917355372, "min": 1721.909090909091, "max": 1721.909090909091, "mean": 1721.909090909091, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_completion_tokens", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_output_tokens", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.5653736157850786, "sum_squared": 0.3196473254258937, "min": 0.5653736157850786, "max": 0.5653736157850786, "mean": 0.5653736157850786, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "inference_runtime", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.5653736157850786, "sum_squared": 0.3196473254258937, "min": 0.5653736157850786, "max": 0.5653736157850786, "mean": 0.5653736157850786, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "batch_size", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_length", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_stop", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_endoftext", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "finish_reason_unknown", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 2.090909090909091, "sum_squared": 4.37190082644628, "min": 2.090909090909091, "max": 2.090909090909091, "mean": 2.090909090909091, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_train_instances", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 2.090909090909091, "sum_squared": 4.37190082644628, "min": 2.090909090909091, "max": 2.090909090909091, "mean": 2.090909090909091, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "prompt_truncated", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "max_prob", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "max_prob", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "exact_match", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.18181818181818182, "sum_squared": 0.03305785123966942, "min": 0.18181818181818182, "max": 0.18181818181818182, "mean": 0.18181818181818182, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "exact_match", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.18181818181818182, "sum_squared": 0.03305785123966942, "min": 0.18181818181818182, "max": 0.18181818181818182, "mean": 0.18181818181818182, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "quasi_exact_match", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.45454545454545453, "sum_squared": 0.20661157024793386, "min": 0.45454545454545453, "max": 0.45454545454545453, "mean": 0.45454545454545453, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "quasi_exact_match", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.45454545454545453, "sum_squared": 0.20661157024793386, "min": 0.45454545454545453, "max": 0.45454545454545453, "mean": 0.45454545454545453, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "f1_score", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.7575757575757577, "sum_squared": 0.5739210284664832, "min": 0.7575757575757577, "max": 0.7575757575757577, "mean": 0.7575757575757577, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "f1_score", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.7575757575757577, "sum_squared": 0.5739210284664832, "min": 0.7575757575757577, "max": 0.7575757575757577, "mean": 0.7575757575757577, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "rouge_l", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.7534271284271284, "sum_squared": 0.5676524378499487, "min": 0.7534271284271284, "max": 0.7534271284271284, "mean": 0.7534271284271284, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "rouge_l", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.7534271284271284, "sum_squared": 0.5676524378499487, "min": 0.7534271284271284, "max": 0.7534271284271284, "mean": 0.7534271284271284, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_1", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.5103042628099558, "sum_squared": 0.2604104406420124, "min": 0.5103042628099558, "max": 0.5103042628099558, "mean": 0.5103042628099558, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_1", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.5103042628099558, "sum_squared": 0.2604104406420124, "min": 0.5103042628099558, "max": 0.5103042628099558, "mean": 0.5103042628099558, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_4", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.116254616642855, "sum_squared": 0.013515135890777179, "min": 0.116254616642855, "max": 0.116254616642855, "mean": 0.116254616642855, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "bleu_4", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.116254616642855, "sum_squared": 0.013515135890777179, "min": 0.116254616642855, "max": 0.116254616642855, "mean": 0.116254616642855, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "logprob", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 0.0, "sum_squared": 0.0, "min": 0.0, "max": 0.0, "mean": 0.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_perplexity_tokens", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 1.0, "sum_squared": 1.0, "min": 1.0, "max": 1.0, "mean": 1.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "valid", "perturbation": { "name": "robustness", "robustness": true, "fairness": false, "computed_on": "worst" } }, "count": 1, "sum": 26.09090909090909, "sum_squared": 680.7355371900826, "min": 26.09090909090909, "max": 26.09090909090909, "mean": 26.09090909090909, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_bytes", "split": "valid", "perturbation": { "name": "fairness", "robustness": false, "fairness": true, "computed_on": "worst" } }, "count": 1, "sum": 26.09090909090909, "sum_squared": 680.7355371900826, "min": 26.09090909090909, "max": 26.09090909090909, "mean": 26.09090909090909, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "test" }, "count": 1, "sum": 39.0, "sum_squared": 1521.0, "min": 39.0, "max": 39.0, "mean": 39.0, "variance": 0.0, "stddev": 0.0 }, { "name": { "name": "num_instances", "split": "valid" }, "count": 1, "sum": 11.0, "sum_squared": 121.0, "min": 11.0, "max": 11.0, "mean": 11.0, "variance": 0.0, "stddev": 0.0 } ]