{
  "name": "czech_bank_qa:model=mistralai_mistral-7b-instruct-v0.3",
  "scenario_spec": {
    "class_name": "helm.benchmark.scenarios.czech_bank_qa_scenario.CzechBankQAScenario",
    "args": {
      "config_name": "berka_queries_1024_2024_12_18"
    }
  },
  "adapter_spec": {
    "method": "generation",
    "global_prefix": "",
    "global_suffix": "",
    "instructions": "Given a SQLite database schema and the following instructions, generate a SQLite query that corresponds to the instructions. Answer with only the query.\n\nDatabase schema:\nCREATE TABLE \"account\" (\n  \"account_id\" integer NOT NULL DEFAULT '0'\n,  \"district_id\" integer NOT NULL DEFAULT '0'\n,  \"frequency\" varchar(18) NOT NULL\n,  \"date\" date NOT NULL\n,  PRIMARY KEY (\"account_id\")\n,  CONSTRAINT \"account_ibfk_1\" FOREIGN KEY (\"district_id\") REFERENCES \"district\" (\"district_id\")\n);\nCREATE TABLE \"card\" (\n  \"card_id\" integer NOT NULL DEFAULT '0'\n,  \"disp_id\" integer NOT NULL\n,  \"type\" varchar(7) NOT NULL\n,  \"issued\" date NOT NULL\n,  PRIMARY KEY (\"card_id\")\n,  CONSTRAINT \"card_ibfk_1\" FOREIGN KEY (\"disp_id\") REFERENCES \"disp\" (\"disp_id\")\n);\nCREATE TABLE \"client\" (\n  \"client_id\" integer NOT NULL\n,  \"gender\" varchar(1) NOT NULL\n,  \"birth_date\" date NOT NULL\n,  \"district_id\" integer NOT NULL\n,  PRIMARY KEY (\"client_id\")\n,  CONSTRAINT \"client_ibfk_1\" FOREIGN KEY (\"district_id\") REFERENCES \"district\" (\"district_id\")\n);\nCREATE TABLE \"disp\" (\n  \"disp_id\" integer NOT NULL\n,  \"client_id\" integer NOT NULL\n,  \"account_id\" integer NOT NULL\n,  \"type\" varchar(9) NOT NULL\n,  PRIMARY KEY (\"disp_id\")\n,  CONSTRAINT \"disp_ibfk_1\" FOREIGN KEY (\"account_id\") REFERENCES \"account\" (\"account_id\")\n,  CONSTRAINT \"disp_ibfk_2\" FOREIGN KEY (\"client_id\") REFERENCES \"client\" (\"client_id\")\n);\nCREATE TABLE \"district\" (\n  \"district_id\" integer NOT NULL DEFAULT '0'\n,  \"A2\" varchar(19) NOT NULL\n,  \"A3\" varchar(15) NOT NULL\n,  \"A4\" integer NOT NULL\n,  \"A5\" integer NOT NULL\n,  \"A6\" integer NOT NULL\n,  \"A7\" integer NOT NULL\n,  \"A8\" integer NOT NULL\n,  \"A9\" integer NOT NULL\n,  \"A10\" decimal(4,1) NOT NULL\n,  \"A11\" integer NOT NULL\n,  \"A12\" decimal(4,1) DEFAULT NULL\n,  \"A13\" decimal(3,2) NOT NULL\n,  \"A14\" integer NOT NULL\n,  \"A15\" integer DEFAULT NULL\n,  \"A16\" integer NOT NULL\n,  PRIMARY KEY (\"district_id\")\n);\nCREATE TABLE \"loan\" (\n  \"loan_id\" integer NOT NULL DEFAULT '0'\n,  \"account_id\" integer NOT NULL\n,  \"date\" date NOT NULL\n,  \"amount\" integer NOT NULL\n,  \"duration\" integer NOT NULL\n,  \"payments\" decimal(6,2) NOT NULL\n,  \"status\" varchar(1) NOT NULL\n,  PRIMARY KEY (\"loan_id\")\n,  CONSTRAINT \"loan_ibfk_1\" FOREIGN KEY (\"account_id\") REFERENCES \"account\" (\"account_id\")\n);\nCREATE TABLE \"order\" (\n  \"order_id\" integer NOT NULL DEFAULT '0'\n,  \"account_id\" integer NOT NULL\n,  \"bank_to\" varchar(2) NOT NULL\n,  \"account_to\" integer NOT NULL\n,  \"amount\" decimal(6,1) NOT NULL\n,  \"k_symbol\" varchar(8) NOT NULL\n,  PRIMARY KEY (\"order_id\")\n,  CONSTRAINT \"order_ibfk_1\" FOREIGN KEY (\"account_id\") REFERENCES \"account\" (\"account_id\")\n);\nCREATE TABLE \"trans\" (\n  \"trans_id\" integer NOT NULL DEFAULT '0'\n,  \"account_id\" integer NOT NULL DEFAULT '0'\n,  \"date\" date NOT NULL\n,  \"type\" varchar(6) NOT NULL\n,  \"operation\" varchar(14) DEFAULT NULL\n,  \"amount\" integer NOT NULL\n,  \"balance\" integer NOT NULL\n,  \"k_symbol\" varchar(11) DEFAULT NULL\n,  \"bank\" varchar(2) DEFAULT NULL\n,  \"account\" integer  DEFAULT NULL\n,  PRIMARY KEY (\"trans_id\")\n,  CONSTRAINT \"trans_ibfk_1\" FOREIGN KEY (\"account_id\") REFERENCES \"account\" (\"account_id\")\n);\n",
    "input_prefix": "Instruction: ",
    "input_suffix": "\n",
    "reference_prefix": "A. ",
    "reference_suffix": "\n",
    "chain_of_thought_prefix": "",
    "chain_of_thought_suffix": "\n",
    "output_prefix": "SQL Query: ",
    "output_suffix": "\n",
    "instance_prefix": "\n",
    "substitutions": [],
    "max_train_instances": 5,
    "max_eval_instances": 1000,
    "num_outputs": 1,
    "num_train_trials": 1,
    "num_trials": 1,
    "sample_train": true,
    "model_deployment": "together/mistral-7b-instruct-v0.3",
    "model": "mistralai/mistral-7b-instruct-v0.3",
    "temperature": 0.0,
    "max_tokens": 512,
    "stop_sequences": [
      "\n\n"
    ],
    "multi_label": false
  },
  "metric_specs": [
    {
      "class_name": "helm.benchmark.metrics.basic_metrics.BasicGenerationMetric",
      "args": {
        "names": []
      }
    },
    {
      "class_name": "helm.benchmark.metrics.basic_metrics.BasicReferenceMetric",
      "args": {}
    },
    {
      "class_name": "helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric",
      "args": {}
    },
    {
      "class_name": "helm.benchmark.metrics.czech_bank_qa_metrics.CzechBankQAMetrics",
      "args": {}
    }
  ],
  "data_augmenter_spec": {
    "perturbation_specs": [],
    "should_augment_train_instances": false,
    "should_include_original_train": false,
    "should_skip_unchanged_train": false,
    "should_augment_eval_instances": false,
    "should_include_original_eval": false,
    "should_skip_unchanged_eval": false,
    "seeds_per_instance": 1
  },
  "groups": [
    "czech_bank_qa"
  ],
  "annotators": [
    {
      "class_name": "helm.benchmark.annotation.czech_bank_qa_annotator.CzechBankQAAnnotator",
      "args": {}
    }
  ]
}