{
  "title": "Accuracy",
  "header": [
    {
      "value": "Model",
      "markdown": false,
      "metadata": {}
    },
    {
      "value": "Spider 1.0 (Test) - Execution Accuracy",
      "description": "Spider 1.0 (Test)\n\nExecution Accuracy: Execution Accuracy",
      "markdown": false,
      "lower_is_better": false,
      "metadata": {
        "metric": "Execution Accuracy",
        "run_group": "Spider 1.0 (Test)"
      }
    },
    {
      "value": "BIRD-SQL (Dev) - Execution Accuracy",
      "description": "BIRD-SQL (Dev)\n\nExecution Accuracy: Execution Accuracy",
      "markdown": false,
      "lower_is_better": false,
      "metadata": {
        "metric": "Execution Accuracy",
        "run_group": "BIRD-SQL (Dev)"
      }
    }
  ],
  "rows": [
    [
      {
        "value": "Llama 3.1 Instruct Turbo (405B)",
        "description": "",
        "markdown": false
      },
      {
        "value": 0.77,
        "description": "min=0.77, mean=0.77, max=0.77, sum=0.77 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "spider:model=meta_llama-3.1-405b-instruct-turbo"
        ]
      },
      {
        "value": 0.61,
        "description": "min=0.61, mean=0.61, max=0.61, sum=0.61 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "bird_sql:model=meta_llama-3.1-405b-instruct-turbo"
        ]
      }
    ],
    [
      {
        "value": "Llama 3.1 Instruct Turbo (70B)",
        "description": "",
        "markdown": false
      },
      {
        "value": 0.71,
        "description": "min=0.71, mean=0.71, max=0.71, sum=0.71 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "spider:model=meta_llama-3.1-70b-instruct-turbo"
        ]
      },
      {
        "value": 0.55,
        "description": "min=0.55, mean=0.55, max=0.55, sum=0.55 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "bird_sql:model=meta_llama-3.1-70b-instruct-turbo"
        ]
      }
    ],
    [
      {
        "value": "Llama 3.1 Instruct Turbo (8B)",
        "description": "",
        "markdown": false
      },
      {
        "value": 0.61,
        "description": "min=0.61, mean=0.61, max=0.61, sum=0.61 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "spider:model=meta_llama-3.1-8b-instruct-turbo"
        ]
      },
      {
        "value": 0.37,
        "description": "min=0.37, mean=0.37, max=0.37, sum=0.37 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "bird_sql:model=meta_llama-3.1-8b-instruct-turbo"
        ]
      }
    ],
    [
      {
        "value": "Claude 3.5 Haiku (20241022)",
        "description": "",
        "markdown": false
      },
      {
        "value": 0.63,
        "description": "min=0.63, mean=0.63, max=0.63, sum=0.63 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "spider:model=anthropic_claude-3-5-haiku-20241022"
        ]
      },
      {
        "value": 0.48,
        "description": "min=0.48, mean=0.48, max=0.48, sum=0.48 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "bird_sql:model=anthropic_claude-3-5-haiku-20241022"
        ]
      }
    ],
    [
      {
        "value": "Claude 3.5 Sonnet (20240620)",
        "description": "",
        "markdown": false
      },
      {
        "value": 0.66,
        "description": "min=0.66, mean=0.66, max=0.66, sum=0.66 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "spider:model=anthropic_claude-3-5-sonnet-20240620"
        ]
      },
      {
        "value": 0.47,
        "description": "min=0.47, mean=0.47, max=0.47, sum=0.47 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "bird_sql:model=anthropic_claude-3-5-sonnet-20240620"
        ]
      }
    ],
    [
      {
        "value": "Gemini 1.5 Pro (002)",
        "description": "",
        "markdown": false
      },
      {
        "value": 0.8,
        "description": "min=0.8, mean=0.8, max=0.8, sum=0.8 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "spider:model=google_gemini-1.5-pro-002"
        ]
      },
      {
        "value": 0.61,
        "description": "min=0.61, mean=0.61, max=0.61, sum=0.61 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "bird_sql:model=google_gemini-1.5-pro-002"
        ]
      }
    ],
    [
      {
        "value": "Gemini 1.5 Flash (002)",
        "description": "",
        "markdown": false
      },
      {
        "value": 0.79,
        "description": "min=0.79, mean=0.79, max=0.79, sum=0.79 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "spider:model=google_gemini-1.5-flash-002"
        ]
      },
      {
        "value": 0.63,
        "description": "min=0.63, mean=0.63, max=0.63, sum=0.63 (1)",
        "style": {
          "font-weight": "bold"
        },
        "markdown": false,
        "run_spec_names": [
          "bird_sql:model=google_gemini-1.5-flash-002"
        ]
      }
    ],
    [
      {
        "value": "GPT-4o (2024-08-06)",
        "description": "",
        "markdown": false
      },
      {
        "value": 0.81,
        "description": "min=0.81, mean=0.81, max=0.81, sum=0.81 (1)",
        "style": {
          "font-weight": "bold"
        },
        "markdown": false,
        "run_spec_names": [
          "spider:model=openai_gpt-4o-2024-08-06"
        ]
      },
      {
        "value": 0.62,
        "description": "min=0.62, mean=0.62, max=0.62, sum=0.62 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "bird_sql:model=openai_gpt-4o-2024-08-06"
        ]
      }
    ],
    [
      {
        "value": "GPT-4o mini (2024-07-18)",
        "description": "",
        "markdown": false
      },
      {
        "value": 0.72,
        "description": "min=0.72, mean=0.72, max=0.72, sum=0.72 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "spider:model=openai_gpt-4o-mini-2024-07-18"
        ]
      },
      {
        "value": 0.54,
        "description": "min=0.54, mean=0.54, max=0.54, sum=0.54 (1)",
        "style": {},
        "markdown": false,
        "run_spec_names": [
          "bird_sql:model=openai_gpt-4o-mini-2024-07-18"
        ]
      }
    ]
  ],
  "links": [
    {
      "text": "LaTeX",
      "href": "benchmark_output/releases/v0.5.0/groups/latex/text_to_sql_scenarios_accuracy.tex"
    },
    {
      "text": "JSON",
      "href": "benchmark_output/releases/v0.5.0/groups/json/text_to_sql_scenarios_accuracy.json"
    }
  ],
  "name": "accuracy"
}