added root files

file_copies_from_project_root/finetune.py

@@ -0,0 +1,308 @@
+import os
+import sys
+from typing import List
+
+import fire
+import torch
+import transformers
+from datasets import load_dataset
+import json
+
+"""
+Unused imports:
+import torch.nn as nn
+import bitsandbytes as bnb
+"""
+
+from peft import (
+    LoraConfig,
+    get_peft_model,
+    get_peft_model_state_dict,
+    prepare_model_for_int8_training,
+    set_peft_model_state_dict,
+)
+from transformers import LlamaForCausalLM, LlamaTokenizer
+
+from utils.prompter import Prompter
+
+def train(
+    # model/data params
+    base_model: str = "",  # the only required argument
+    data_path: str = "yahma/alpaca-cleaned",
+    output_dir: str = "./lora-alpaca",
+    # training hyperparams
+    batch_size: int = 128,
+    micro_batch_size: int = 4,
+    num_epochs: int = 3,
+    learning_rate: float = 3e-4,
+    cutoff_len: int = 256,
+    val_set_size: int = 2000,
+    # lora hyperparams
+    lora_r: int = 8,
+    lora_alpha: int = 16,
+    lora_dropout: float = 0.05,
+    lora_target_modules: List[str] = [
+        "q_proj",
+        "v_proj",
+    ],
+    # llm hyperparams
+    train_on_inputs: bool = True,  # if False, masks out inputs in loss
+    add_eos_token: bool = False,
+    group_by_length: bool = False,  # faster, but produces an odd training loss curve
+    # wandb params
+    wandb_project: str = "",
+    wandb_run_name: str = "",
+    wandb_watch: str = "",  # options: false | gradients | all
+    wandb_log_model: str = "",  # options: false | true
+    resume_from_checkpoint: str = None,  # either training checkpoint or final adapter
+    prompt_template_name: str = "alpaca",  # The prompt template to use, will default to alpaca.
+):
+    if int(os.environ.get("LOCAL_RANK", 0)) == 0:
+        print(
+            f"Training Alpaca-LoRA model with params:\n"
+            f"base_model: {base_model}\n"
+            f"data_path: {data_path}\n"
+            f"output_dir: {output_dir}\n"
+            f"batch_size: {batch_size}\n"
+            f"micro_batch_size: {micro_batch_size}\n"
+            f"num_epochs: {num_epochs}\n"
+            f"learning_rate: {learning_rate}\n"
+            f"cutoff_len: {cutoff_len}\n"
+            f"val_set_size: {val_set_size}\n"
+            f"lora_r: {lora_r}\n"
+            f"lora_alpha: {lora_alpha}\n"
+            f"lora_dropout: {lora_dropout}\n"
+            f"lora_target_modules: {lora_target_modules}\n"
+            f"train_on_inputs: {train_on_inputs}\n"
+            f"add_eos_token: {add_eos_token}\n"
+            f"group_by_length: {group_by_length}\n"
+            f"wandb_project: {wandb_project}\n"
+            f"wandb_run_name: {wandb_run_name}\n"
+            f"wandb_watch: {wandb_watch}\n"
+            f"wandb_log_model: {wandb_log_model}\n"
+            f"resume_from_checkpoint: {resume_from_checkpoint or False}\n"
+            f"prompt template: {prompt_template_name}\n"
+        )
+    assert (
+        base_model
+    ), "Please specify a --base_model, e.g. --base_model='huggyllama/llama-7b'"
+    gradient_accumulation_steps = batch_size // micro_batch_size
+
+    prompter = Prompter(prompt_template_name)
+
+    device_map = "auto"
+    world_size = int(os.environ.get("WORLD_SIZE", 1))
+    ddp = world_size != 1
+    if ddp:
+        device_map = {"": int(os.environ.get("LOCAL_RANK") or 0)}
+        gradient_accumulation_steps = gradient_accumulation_steps // world_size
+
+    # Check if parameter passed or if set within environ
+    use_wandb = len(wandb_project) > 0 or (
+        "WANDB_PROJECT" in os.environ and len(os.environ["WANDB_PROJECT"]) > 0
+    )
+    # Only overwrite environ if wandb param passed
+    if len(wandb_project) > 0:
+        os.environ["WANDB_PROJECT"] = wandb_project
+    if len(wandb_watch) > 0:
+        os.environ["WANDB_WATCH"] = wandb_watch
+    if len(wandb_log_model) > 0:
+        os.environ["WANDB_LOG_MODEL"] = wandb_log_model
+
+    model = LlamaForCausalLM.from_pretrained(
+        base_model,
+        load_in_8bit=True,
+        torch_dtype=torch.float16,
+        device_map=device_map,
+    )
+
+    tokenizer = LlamaTokenizer.from_pretrained(base_model)
+
+    tokenizer.pad_token_id = (
+        0  # unk. we want this to be different from the eos token
+    )
+    tokenizer.padding_side = "left"  # Allow batched inference
+
+    def tokenize(prompt, add_eos_token=True):
+        # there's probably a way to do this with the tokenizer settings
+        # but again, gotta move fast
+        result = tokenizer(
+            prompt,
+            truncation=True,
+            max_length=cutoff_len,
+            padding=False,
+            return_tensors=None,
+        )
+        if (
+            result["input_ids"][-1] != tokenizer.eos_token_id
+            and len(result["input_ids"]) < cutoff_len
+            and add_eos_token
+        ):
+            result["input_ids"].append(tokenizer.eos_token_id)
+            result["attention_mask"].append(1)
+
+        result["labels"] = result["input_ids"].copy()
+
+        return result
+
+    def generate_and_tokenize_prompt(data_point):
+        full_prompt = prompter.generate_prompt(
+            # For Alpaca-Lora
+            # data_point["instruction"],
+            # data_point["input"],
+            # data_point["output"],
+            # For SQL Create Context
+            # data_point["question"],
+            # data_point["context"],
+            # data_point["answer"],
+            # For Spider
+            instruction = data_point["question"],
+            input = data_point["db_context"],
+            label = data_point["query"],
+        )
+        # print("FULL PROMPT")
+        # print(full_prompt) # String with instruction, inputs, and response
+
+        tokenized_full_prompt = tokenize(full_prompt)
+        if not train_on_inputs:
+            print("SHOULDNT BE HERE")
+            user_prompt = prompter.generate_prompt(
+            # For Alpaca-Lora
+                # data_point["instruction"], data_point["input"]
+            # For SQL Create Context
+                # data_point["question"], data_point["context"]
+            # For Spider
+                data_point["question"], data_point["db_context"]
+            )
+            tokenized_user_prompt = tokenize(
+                user_prompt, add_eos_token=add_eos_token
+            )
+            user_prompt_len = len(tokenized_user_prompt["input_ids"])
+
+            if add_eos_token:
+                user_prompt_len -= 1
+
+            tokenized_full_prompt["labels"] = [
+                -100
+            ] * user_prompt_len + tokenized_full_prompt["labels"][
+                user_prompt_len:
+            ]  # could be sped up, probably
+        return tokenized_full_prompt
+
+    model = prepare_model_for_int8_training(model)
+
+    config = LoraConfig(
+        r=lora_r,
+        lora_alpha=lora_alpha,
+        target_modules=lora_target_modules,
+        lora_dropout=lora_dropout,
+        bias="none",
+        task_type="CAUSAL_LM",
+    )
+    model = get_peft_model(model, config)
+
+    if data_path.endswith(".json") or data_path.endswith(".jsonl"):
+        data = load_dataset("json", data_files=data_path)
+    else:
+        data = load_dataset(data_path)
+
+    # For SPIDER only !!!!! -------------------------------------
+    data = load_dataset("csv", data_files={"train": "dataset_train_augmented.csv", "validation": "dataset_val_augmented.csv"})
+
+    # End of code for SPIDER only !!!! --------------------------
+
+    if resume_from_checkpoint:
+        # Check the available weights and load them
+        checkpoint_name = os.path.join(
+            resume_from_checkpoint, "pytorch_model.bin"
+        )  # Full checkpoint
+        if not os.path.exists(checkpoint_name):
+            checkpoint_name = os.path.join(
+                resume_from_checkpoint, "adapter_model.bin"
+            )  # only LoRA model - LoRA config above has to fit
+            resume_from_checkpoint = (
+                False  # So the trainer won't try loading its state
+            )
+        # The two files above have a different name depending on how they were saved, but are actually the same.
+        if os.path.exists(checkpoint_name):
+            print(f"Restarting from {checkpoint_name}")
+            adapters_weights = torch.load(checkpoint_name)
+            set_peft_model_state_dict(model, adapters_weights)
+        else:
+            print(f"Checkpoint {checkpoint_name} not found")
+
+    model.print_trainable_parameters()  # Be more transparent about the % of trainable params.
+
+    if val_set_size > 0:
+        train_val = data["train"].train_test_split(
+            test_size=val_set_size, shuffle=True, seed=42
+        )
+        train_data = (
+            train_val["train"].shuffle().map(generate_and_tokenize_prompt)
+        )
+        print("TRAIN DATA")
+        print(train_data[0])
+        val_data = (
+            train_val["test"].shuffle().map(generate_and_tokenize_prompt)
+        )
+    else:
+        train_data = data["train"].shuffle().map(generate_and_tokenize_prompt)
+        val_data = None
+
+    if not ddp and torch.cuda.device_count() > 1:
+        # keeps Trainer from trying its own DataParallelism when more than 1 gpu is available
+        model.is_parallelizable = True
+        model.model_parallel = True
+
+    trainer = transformers.Trainer(
+        model=model,
+        train_dataset=train_data,
+        eval_dataset=val_data,
+        args=transformers.TrainingArguments(
+            per_device_train_batch_size=micro_batch_size,
+            gradient_accumulation_steps=gradient_accumulation_steps,
+            warmup_steps=100,
+            num_train_epochs=num_epochs,
+            learning_rate=learning_rate,
+            fp16=True,
+            logging_steps=10,
+            optim="adamw_torch",
+            evaluation_strategy="steps" if val_set_size > 0 else "no",
+            save_strategy="steps",
+            eval_steps=200 if val_set_size > 0 else None,
+            save_steps=200,
+            output_dir=output_dir,
+            save_total_limit=3,
+            load_best_model_at_end=True if val_set_size > 0 else False,
+            ddp_find_unused_parameters=False if ddp else None,
+            group_by_length=group_by_length,
+            report_to="wandb" if use_wandb else None,
+            run_name=wandb_run_name if use_wandb else None,
+        ),
+        data_collator=transformers.DataCollatorForSeq2Seq(
+            tokenizer, pad_to_multiple_of=8, return_tensors="pt", padding=True
+        ),
+    )
+    model.config.use_cache = False
+
+    old_state_dict = model.state_dict
+    model.state_dict = (
+        lambda self, *_, **__: get_peft_model_state_dict(
+            self, old_state_dict()
+        )
+    ).__get__(model, type(model))
+
+    if torch.__version__ >= "2" and sys.platform != "win32":
+        model = torch.compile(model)
+
+    trainer.train(resume_from_checkpoint=resume_from_checkpoint)
+
+    model.save_pretrained(output_dir)
+
+    print(
+        "\n If there's a warning about missing keys above, please disregard :)"
+    )
+
+
+if __name__ == "__main__":
+    fire.Fire(train)

file_copies_from_project_root/generate.py

@@ -0,0 +1,220 @@
+import os
+import sys
+
+import fire
+import gradio as gr
+import torch
+import transformers
+from peft import PeftModel
+from transformers import GenerationConfig, LlamaForCausalLM, LlamaTokenizer
+
+from utils.callbacks import Iteratorize, Stream
+from utils.prompter import Prompter
+
+if torch.cuda.is_available():
+    device = "cuda"
+else:
+    device = "cpu"
+
+try:
+    if torch.backends.mps.is_available():
+        device = "mps"
+except:  # noqa: E722
+    pass
+
+
+def main(
+    load_8bit: bool = False,
+    base_model: str = "",
+    lora_weights: str = "tloen/alpaca-lora-7b",
+    prompt_template: str = "",  # The prompt template to use, will default to alpaca.
+    server_name: str = "0.0.0.0",  # Allows to listen on all interfaces by providing '0.
+    share_gradio: bool = False,
+):
+    base_model = base_model or os.environ.get("BASE_MODEL", "")
+    assert (
+        base_model
+    ), "Please specify a --base_model, e.g. --base_model='huggyllama/llama-7b'"
+
+    prompter = Prompter(prompt_template)
+    tokenizer = LlamaTokenizer.from_pretrained(base_model)
+    if device == "cuda":
+        model = LlamaForCausalLM.from_pretrained(
+            base_model,
+            load_in_8bit=load_8bit,
+            torch_dtype=torch.float16,
+            # device_map="auto",
+            device_map={'':0},
+        )
+        model = PeftModel.from_pretrained(
+            model,
+            lora_weights,
+            torch_dtype=torch.float16,
+            device_map={'':0},
+        )
+    elif device == "mps":
+        model = LlamaForCausalLM.from_pretrained(
+            base_model,
+            device_map={"": device},
+            torch_dtype=torch.float16,
+        )
+        model = PeftModel.from_pretrained(
+            model,
+            lora_weights,
+            device_map={"": device},
+            torch_dtype=torch.float16,
+        )
+    else:
+        model = LlamaForCausalLM.from_pretrained(
+            base_model, device_map={"": device}, low_cpu_mem_usage=True
+        )
+        model = PeftModel.from_pretrained(
+            model,
+            lora_weights,
+            device_map={"": device},
+        )
+
+    # unwind broken decapoda-research config
+    model.config.pad_token_id = tokenizer.pad_token_id = 0  # unk
+    model.config.bos_token_id = 1
+    model.config.eos_token_id = 2
+
+    if not load_8bit:
+        model.half()  # seems to fix bugs for some users.
+
+    model.eval()
+    if torch.__version__ >= "2" and sys.platform != "win32":
+        model = torch.compile(model)
+
+    def evaluate(
+        instruction,
+        input=None,
+        temperature=0.1,
+        top_p=0.75,
+        top_k=40,
+        num_beams=4,
+        max_new_tokens=128,
+        stream_output=False,
+        **kwargs,
+    ):
+        prompt = prompter.generate_prompt(instruction, input)
+        inputs = tokenizer(prompt, return_tensors="pt")
+        input_ids = inputs["input_ids"].to(device)
+        generation_config = GenerationConfig(
+            temperature=temperature,
+            top_p=top_p,
+            top_k=top_k,
+            num_beams=num_beams,
+            **kwargs,
+        )
+
+        generate_params = {
+            "input_ids": input_ids,
+            "generation_config": generation_config,
+            "return_dict_in_generate": True,
+            "output_scores": True,
+            "max_new_tokens": max_new_tokens,
+        }
+
+        if stream_output:
+            # Stream the reply 1 token at a time.
+            # This is based on the trick of using 'stopping_criteria' to create an iterator,
+            # from https://github.com/oobabooga/text-generation-webui/blob/ad37f396fc8bcbab90e11ecf17c56c97bfbd4a9c/modules/text_generation.py#L216-L243.
+
+            def generate_with_callback(callback=None, **kwargs):
+                kwargs.setdefault(
+                    "stopping_criteria", transformers.StoppingCriteriaList()
+                )
+                kwargs["stopping_criteria"].append(
+                    Stream(callback_func=callback)
+                )
+                with torch.no_grad():
+                    model.generate(**kwargs)
+
+            def generate_with_streaming(**kwargs):
+                return Iteratorize(
+                    generate_with_callback, kwargs, callback=None
+                )
+
+            with generate_with_streaming(**generate_params) as generator:
+                for output in generator:
+                    # new_tokens = len(output) - len(input_ids[0])
+                    decoded_output = tokenizer.decode(output)
+
+                    if output[-1] in [tokenizer.eos_token_id]:
+                        break
+
+                    yield prompter.get_response(decoded_output)
+            return  # early return for stream_output
+
+        # Without streaming
+        with torch.no_grad():
+            generation_output = model.generate(
+                input_ids=input_ids,
+                generation_config=generation_config,
+                return_dict_in_generate=True,
+                output_scores=True,
+                max_new_tokens=max_new_tokens,
+            )
+        s = generation_output.sequences[0]
+        output = tokenizer.decode(s)
+        yield prompter.get_response(output)
+
+    gr.Interface(
+        fn=evaluate,
+        inputs=[
+            gr.components.Textbox(
+                lines=2,
+                label="Instruction",
+                placeholder="Tell me about alpacas.",
+            ),
+            gr.components.Textbox(lines=2, label="Input", placeholder="none"),
+            gr.components.Slider(
+                minimum=0, maximum=1, value=0.1, label="Temperature"
+            ),
+            gr.components.Slider(
+                minimum=0, maximum=1, value=0.75, label="Top p"
+            ),
+            gr.components.Slider(
+                minimum=0, maximum=100, step=1, value=40, label="Top k"
+            ),
+            gr.components.Slider(
+                minimum=1, maximum=4, step=1, value=4, label="Beams"
+            ),
+            gr.components.Slider(
+                minimum=1, maximum=2000, step=1, value=128, label="Max tokens"
+            ),
+            gr.components.Checkbox(label="Stream output"),
+        ],
+        outputs=[
+            gr.inputs.Textbox(
+                lines=5,
+                label="Output",
+            )
+        ],
+        title="🦙🌲 Alpaca-LoRA",
+        description="Alpaca-LoRA is a 7B-parameter LLaMA model finetuned to follow instructions. It is trained on the [Stanford Alpaca](https://github.com/tatsu-lab/stanford_alpaca) dataset and makes use of the Huggingface LLaMA implementation. For more information, please visit [the project's website](https://github.com/tloen/alpaca-lora).",  # noqa: E501
+    ).queue().launch(server_name="0.0.0.0", share=share_gradio)
+    # Old testing code follows.
+
+    """
+    # testing code for readme
+    for instruction in [
+        "Tell me about alpacas.",
+        "Tell me about the president of Mexico in 2019.",
+        "Tell me about the king of France in 2019.",
+        "List all Canadian provinces in alphabetical order.",
+        "Write a Python program that prints the first 10 Fibonacci numbers.",
+        "Write a program that prints the numbers from 1 to 100. But for multiples of three print 'Fizz' instead of the number and for the multiples of five print 'Buzz'. For numbers which are multiples of both three and five print 'FizzBuzz'.",  # noqa: E501
+        "Tell me five words that rhyme with 'shock'.",
+        "Translate the sentence 'I have no mouth but I must scream' into Spanish.",
+        "Count up from 1 to 500.",
+    ]:
+        print("Instruction:", instruction)
+        print("Response:", evaluate(instruction))
+        print()
+    """
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

file_copies_from_project_root/generate_eval.py

@@ -0,0 +1,325 @@
+import os
+import sys
+
+import fire
+import gradio as gr
+import torch
+import transformers
+from peft import PeftModel
+from transformers import GenerationConfig, LlamaForCausalLM, LlamaTokenizer
+
+from utils.callbacks import Iteratorize, Stream
+from utils.prompter import Prompter
+from datasets import load_dataset
+
+if torch.cuda.is_available():
+    device = "cuda"
+else:
+    device = "cpu"
+
+try:
+    if torch.backends.mps.is_available():
+        device = "mps"
+except:  # noqa: E722
+    pass
+
+
+def main(
+    load_8bit: bool = False,
+    base_model: str = "",
+    lora_weights: str = "tloen/alpaca-lora-7b",
+    prompt_template: str = "",  # The prompt template to use, will default to alpaca.
+    server_name: str = "0.0.0.0",  # Allows to listen on all interfaces by providing '0.
+    share_gradio: bool = False,
+    gold_file: str = "gold.txt",
+    predict_file: str = "predict.txt",
+    num_tests: int = 10
+):
+    base_model = base_model or os.environ.get("BASE_MODEL", "")
+    assert (
+        base_model
+    ), "Please specify a --base_model, e.g. --base_model='huggyllama/llama-7b'"
+
+    prompter = Prompter(prompt_template)
+    tokenizer = LlamaTokenizer.from_pretrained(base_model)
+    if device == "cuda":
+        model = LlamaForCausalLM.from_pretrained(
+            base_model,
+            load_in_8bit=load_8bit,
+            torch_dtype=torch.float16,
+            # device_map="auto",
+            device_map={'':0},
+        )
+        model = PeftModel.from_pretrained(
+            model,
+            lora_weights,
+            torch_dtype=torch.float16,
+            device_map={'':0},
+        )
+    elif device == "mps":
+        model = LlamaForCausalLM.from_pretrained(
+            base_model,
+            device_map={"": device},
+            torch_dtype=torch.float16,
+        )
+        model = PeftModel.from_pretrained(
+            model,
+            lora_weights,
+            device_map={"": device},
+            torch_dtype=torch.float16,
+        )
+    else:
+        model = LlamaForCausalLM.from_pretrained(
+            base_model, device_map={"": device}, low_cpu_mem_usage=True
+        )
+        model = PeftModel.from_pretrained(
+            model,
+            lora_weights,
+            device_map={"": device},
+        )
+
+    # unwind broken decapoda-research config
+    model.config.pad_token_id = tokenizer.pad_token_id = 0  # unk
+    model.config.bos_token_id = 1
+    model.config.eos_token_id = 2
+
+    if not load_8bit:
+        model.half()  # seems to fix bugs for some users.
+
+    model.eval()
+    if torch.__version__ >= "2" and sys.platform != "win32":
+        model = torch.compile(model)
+
+    '''
+    def evaluate(
+        instruction,
+        input=None,
+        temperature=0.1,
+        top_p=0.75,
+        top_k=40,
+        num_beams=4,
+        max_new_tokens=128,
+        stream_output=False,
+        **kwargs,
+    ):
+        print("CHECKING")
+        prompt = prompter.generate_prompt(instruction, input)
+        inputs = tokenizer(prompt, return_tensors="pt")
+        input_ids = inputs["input_ids"].to(device)
+        generation_config = GenerationConfig(
+            temperature=temperature,
+            top_p=top_p,
+            top_k=top_k,
+            num_beams=num_beams,
+            **kwargs,
+        )
+        print("CHECKING")
+
+        generate_params = {
+            "input_ids": input_ids,
+            "generation_config": generation_config,
+            "return_dict_in_generate": True,
+            "output_scores": True,
+            "max_new_tokens": max_new_tokens,
+        }
+
+        if stream_output:
+            # Stream the reply 1 token at a time.
+            # This is based on the trick of using 'stopping_criteria' to create an iterator,
+            # from https://github.com/oobabooga/text-generation-webui/blob/ad37f396fc8bcbab90e11ecf17c56c97bfbd4a9c/modules/text_generation.py#L216-L243.
+
+            def generate_with_callback(callback=None, **kwargs):
+                kwargs.setdefault(
+                    "stopping_criteria", transformers.StoppingCriteriaList()
+                )
+                kwargs["stopping_criteria"].append(
+                    Stream(callback_func=callback)
+                )
+                with torch.no_grad():
+                    model.generate(**kwargs)
+
+            def generate_with_streaming(**kwargs):
+                return Iteratorize(
+                    generate_with_callback, kwargs, callback=None
+                )
+
+            with generate_with_streaming(**generate_params) as generator:
+                for output in generator:
+                    # new_tokens = len(output) - len(input_ids[0])
+                    decoded_output = tokenizer.decode(output)
+
+                    if output[-1] in [tokenizer.eos_token_id]:
+                        break
+
+                    yield prompter.get_response(decoded_output)
+            print("EARLY RETURN")
+            return  # early return for stream_output
+
+        # Without streaming
+        with torch.no_grad():
+            generation_output = model.generate(
+                input_ids=input_ids,
+                generation_config=generation_config,
+                return_dict_in_generate=True,
+                output_scores=True,
+                max_new_tokens=max_new_tokens,
+            )
+        s = generation_output.sequences[0]
+        output = tokenizer.decode(s)
+        print("WHERE I SHOULD BE")
+        print("OUTPUT")
+        print(output)
+        yield prompter.get_response(output)
+    '''
+
+    
+    # Modification --------------------------------------------------------------------
+    # data = load_dataset("csv", "dataset_val_augmented.csv")
+    data = load_dataset("csv", data_files={"train": "dataset_train_augmented.csv", "validation": "dataset_val_augmented.csv"})
+    # data_val = data["validation"]
+    data_val = data["validation"].shuffle(seed=42)
+    print(data_val)
+    datalength = len(data_val)
+    print(datalength)
+    golds = []
+    predicts = []
+    for idx in range(num_tests):
+        print(idx)
+        instr = data_val["question"][idx]
+        inp = data_val["db_context"][idx]
+        new_gold = data_val["query"][idx] + "\t" + data_val["db_id"][idx]
+        golds.append(new_gold)
+        print("INSTRUCTION")
+        print(instr)
+        print("INPUT")
+        print(inp)
+        print("GOLD")
+        print(new_gold)
+
+        # response = evaluate(instr, input=inp, temperature=0.1, top_p=0.75, top_k=10, num_beams=1, max_new_tokens=128)
+        # evaluate(instr, input=inp, temperature=0.1, top_p=0.75, top_k=10, num_beams=1, max_new_tokens=128)
+        # print(response)
+        # evaluate(instruction, input=None, temperature=0.1, top_p=0.75, top_k=40, num_beams=4, max_new_tokens=128, stream_output=False, **kwargs,):
+
+        prompt = prompter.generate_prompt(instr, inp)
+        inputs = tokenizer(prompt, return_tensors="pt")
+        input_ids = inputs["input_ids"].to(device)
+        generation_config = GenerationConfig(
+            temperature=0.1,
+            top_p=0.75,
+            top_k=40,
+            num_beams=4,
+            # eos_token_id=[0, 2] # including pad token
+            # eos_token_id=2
+        )
+        max_new_tokens = 128
+
+        generate_params = {
+            "input_ids": input_ids,
+            "generation_config": generation_config,
+            "return_dict_in_generate": True,
+            "output_scores": True,
+            "max_new_tokens": max_new_tokens,
+        }
+
+        # Without streaming
+        with torch.no_grad():
+            generation_output = model.generate(
+                input_ids=input_ids,
+                generation_config=generation_config,
+                return_dict_in_generate=True,
+                output_scores=True,
+                max_new_tokens=max_new_tokens,
+            )
+        s = generation_output.sequences[0]
+        output = tokenizer.decode(s)
+        # print(output)
+        # yield prompter.get_response(output)
+        # USE ONE OF THE FOLLOWING TWO 
+        # prediction =  prompter.get_response(output).split("<unk>")[0].split("\n")[0]
+        prediction =  prompter.get_response(output).split("<unk>")[0]
+
+        # FOR TESTING BASE LLAMA / ALPACA MODELS
+        # prediction = output.strip()
+        # prediction =  output.split("### Answer:")[1].strip().split("<unk>")[0].split("\n")[0]
+        # try:
+            # prediction =  prompter.get_response(output)
+        # except:
+            # prediction = "no response"
+        print("OUTPUT")
+        print(prediction)
+        predicts.append(prediction)
+
+    with open(gold_file, 'w') as file:
+        for g in golds:
+            file.write(g + '\n')
+
+    with open(predict_file, 'w') as file:
+        for p in predicts:
+            file.write(p + '\n')
+                    
+    # fg = open(gold_file, 'w')
+    # fp = open(predict_file, 'w')
+
+    # No need for gradio interface
+    '''
+    gr.Interface(
+        fn=evaluate,
+        inputs=[
+            gr.components.Textbox(
+                lines=2,
+                label="Instruction",
+                placeholder="Tell me about alpacas.",
+            ),
+            gr.components.Textbox(lines=2, label="Input", placeholder="none"),
+            gr.components.Slider(
+                minimum=0, maximum=1, value=0.1, label="Temperature"
+            ),
+            gr.components.Slider(
+                minimum=0, maximum=1, value=0.75, label="Top p"
+            ),
+            gr.components.Slider(
+                minimum=0, maximum=100, step=1, value=40, label="Top k"
+            ),
+            gr.components.Slider(
+                minimum=1, maximum=4, step=1, value=4, label="Beams"
+            ),
+            gr.components.Slider(
+                minimum=1, maximum=2000, step=1, value=128, label="Max tokens"
+            ),
+            gr.components.Checkbox(label="Stream output"),
+        ],
+        outputs=[
+            gr.inputs.Textbox(
+                lines=5,
+                label="Output",
+            )
+        ],
+        title="🦙🌲 Alpaca-LoRA",
+        description="Alpaca-LoRA is a 7B-parameter LLaMA model finetuned to follow instructions. It is trained on the [Stanford Alpaca](https://github.com/tatsu-lab/stanford_alpaca) dataset and makes use of the Huggingface LLaMA implementation. For more information, please visit [the project's website](https://github.com/tloen/alpaca-lora).",  # noqa: E501
+    ).queue().launch(server_name="0.0.0.0", share=share_gradio)
+    '''
+
+    # Old testing code follows.
+
+    """
+    # testing code for readme
+    for instruction in [
+        "Tell me about alpacas.",
+        "Tell me about the president of Mexico in 2019.",
+        "Tell me about the king of France in 2019.",
+        "List all Canadian provinces in alphabetical order.",
+        "Write a Python program that prints the first 10 Fibonacci numbers.",
+        "Write a program that prints the numbers from 1 to 100. But for multiples of three print 'Fizz' instead of the number and for the multiples of five print 'Buzz'. For numbers which are multiples of both three and five print 'FizzBuzz'.",  # noqa: E501
+        "Tell me five words that rhyme with 'shock'.",
+        "Translate the sentence 'I have no mouth but I must scream' into Spanish.",
+        "Count up from 1 to 500.",
+    ]:
+        print("Instruction:", instruction)
+        print("Response:", evaluate(instruction))
+        print()
+    """
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

file_copies_from_project_root/gold3.txt

@@ -0,0 +1,200 @@
+SELECT count(*) ,  country_code FROM players GROUP BY country_code	wta_1
+SELECT count(*) FROM Documents AS T1 JOIN Templates AS T2 ON T1.Template_ID  =  T2.Template_ID WHERE T2.Template_Type_Code  =  'PPT'	cre_Doc_Template_Mgt
+SELECT count(*) FROM Cartoon WHERE Written_by = "Joseph Kuhr";	tvshow
+SELECT name ,  country ,  age FROM singer ORDER BY age DESC	concert_singer
+SELECT Name FROM country WHERE continent  =  "Europe" AND Population  =  "80000"	world_1
+SELECT count(*) FROM Highschooler WHERE grade  =  9 OR grade  =  10	network_1
+SELECT COUNT(*) FROM (SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  "English" INTERSECT SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  "Dutch")	world_1
+select name from teacher where hometown != "little lever urban district"	course_teach
+SELECT T1.address_id ,  T1.line_1 ,  T1.line_2 FROM Addresses AS T1 JOIN Students AS T2 ON T1.address_id  =  T2.current_address_id GROUP BY T1.address_id ORDER BY count(*) DESC LIMIT 1	student_transcripts_tracking
+SELECT Orchestra FROM orchestra WHERE Orchestra_ID NOT IN (SELECT Orchestra_ID FROM performance)	orchestra
+SELECT DISTINCT T1.model FROM MODEL_LIST AS T1 JOIN CAR_NAMES AS T2 ON T1.Model  =  T2.Model JOIN CARS_DATA AS T3 ON T2.MakeId  =  T3.Id JOIN CAR_MAKERS AS T4 ON T1.Maker  =  T4.Id WHERE T3.weight  <  3500 AND T4.FullName != 'Ford Motor Company';	car_1
+SELECT count(*) FROM conductor	orchestra
+select t2.name ,  t2.capacity from concert as t1 join stadium as t2 on t1.stadium_id  =  t2.stadium_id where t1.year  >  2013 group by t2.stadium_id order by count(*) desc limit 1	concert_singer
+SELECT grade FROM Highschooler WHERE name  =  "Kyle"	network_1
+SELECT DISTINCT T2.semester_id FROM Degree_Programs AS T1 JOIN Student_Enrolment AS T2 ON T1.degree_program_id  =  T2.degree_program_id WHERE degree_summary_name  =  'Master' INTERSECT SELECT DISTINCT T2.semester_id FROM Degree_Programs AS T1 JOIN Student_Enrolment AS T2 ON T1.degree_program_id  =  T2.degree_program_id WHERE degree_summary_name  =  'Bachelor'	student_transcripts_tracking
+SELECT avg(LifeExpectancy) FROM country WHERE Continent  =  "Africa" AND GovernmentForm  =  "Republic"	world_1
+SELECT T2.Language FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.HeadOfState  =  "Beatrix" AND T2.IsOfficial  =  "T"	world_1
+SELECT first_name ,  middle_name ,  last_name FROM Students ORDER BY date_first_registered ASC LIMIT 1	student_transcripts_tracking
+SELECT count(*) FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid WHERE T1.age  >  20	pets_1
+SELECT section_name FROM Sections ORDER BY section_name DESC	student_transcripts_tracking
+SELECT count(*) FROM Paragraphs AS T1 JOIN Documents AS T2 ON T1.document_ID  =  T2.document_ID WHERE T2.document_name  =  'Summer Show'	cre_Doc_Template_Mgt
+SELECT document_id FROM Paragraphs GROUP BY document_id HAVING count(*)  >=  2	cre_Doc_Template_Mgt
+SELECT country FROM TV_Channel EXCEPT SELECT T1.country FROM TV_Channel AS T1 JOIN cartoon AS T2 ON T1.id = T2.Channel WHERE T2.written_by  =  'Todd Casey'	tvshow
+select t1.name from country as t1 join countrylanguage as t2 on t1.code  =  t2.countrycode where t2.language  =  "english" and isofficial  =  "t" union select t1.name from country as t1 join countrylanguage as t2 on t1.code  =  t2.countrycode where t2.language  =  "dutch" and isofficial  =  "t"	world_1
+SELECT count(*) FROM CAR_MAKERS AS T1 JOIN MODEL_LIST AS T2 ON T1.Id  =  T2.Maker WHERE T1.FullName  =  'American Motor Company';	car_1
+SELECT name FROM shop WHERE shop_id NOT IN (SELECT shop_id FROM hiring)	employee_hire_evaluation
+SELECT template_id FROM Templates EXCEPT SELECT template_id FROM Documents	cre_Doc_Template_Mgt
+SELECT DISTINCT T1.course_name FROM Courses AS T1 JOIN Student_Enrolment_Courses AS T2 ON T1.course_id  =  T2.course_id	student_transcripts_tracking
+SELECT T2.template_id FROM Ref_template_types AS T1 JOIN Templates AS T2 ON T1.template_type_code  = T2.template_type_code WHERE T1.template_type_description  =  "Presentation"	cre_Doc_Template_Mgt
+SELECT count(*) FROM teacher	course_teach
+SELECT template_type_code FROM Templates GROUP BY template_type_code HAVING count(*)  <  3	cre_Doc_Template_Mgt
+SELECT Record_Company ,  COUNT(*) FROM orchestra GROUP BY Record_Company	orchestra
+SELECT Name FROM people WHERE People_ID NOT IN (SELECT People_ID FROM poker_player)	poker_player
+SELECT T2.property_type_description FROM Properties AS T1 JOIN Ref_Property_Types AS T2 ON T1.property_type_code  =  T2.property_type_code GROUP BY T1.property_type_code	real_estate_properties
+SELECT T3.Name ,  T2.Course FROM course_arrange AS T1 JOIN course AS T2 ON T1.Course_ID  =  T2.Course_ID JOIN teacher AS T3 ON T1.Teacher_ID  =  T3.Teacher_ID ORDER BY T3.Name	course_teach
+SELECT template_type_code FROM Ref_template_types WHERE template_type_description  =  "Book"	cre_Doc_Template_Mgt
+SELECT T1.FullName ,  T1.Id FROM CAR_MAKERS AS T1 JOIN MODEL_LIST AS T2 ON T1.Id  =  T2.Maker GROUP BY T1.Id HAVING count(*)  >  3;	car_1
+SELECT T2.feature_type_name FROM Other_Available_Features AS T1 JOIN Ref_Feature_Types AS T2 ON T1.feature_type_code  =  T2.feature_type_code WHERE T1.feature_name  =  "AirCon"	real_estate_properties
+SELECT department_description FROM Departments WHERE department_name LIKE '%computer%'	student_transcripts_tracking
+select t1.first_name from students as t1 join addresses as t2 on t1.permanent_address_id  =  t2.address_id where t2.country  =  'haiti' or t1.cell_mobile_number  =  '09700166582'	student_transcripts_tracking
+SELECT Airline FROM AIRLINES WHERE Abbreviation  =  "UAL"	flight_2
+SELECT charge_type ,  charge_amount FROM Charges	dog_kennels
+SELECT professional_id ,  last_name ,  cell_number FROM Professionals WHERE state  =  'Indiana' UNION SELECT T1.professional_id ,  T1.last_name ,  T1.cell_number FROM Professionals AS T1 JOIN Treatments AS T2 ON T1.professional_id  =  T2.professional_id GROUP BY T1.professional_id HAVING count(*)  >  2	dog_kennels
+SELECT count(*) FROM pets WHERE weight  >  10	pets_1
+SELECT first_name ,  birth_date FROM players WHERE country_code  =  'USA'	wta_1
+SELECT T1.model FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id WHERE T2.Weight  <  (SELECT avg(Weight) FROM CARS_DATA)	car_1
+SELECT T1.name ,  T2.date_of_treatment FROM Dogs AS T1 JOIN Treatments AS T2 ON T1.dog_id  =  T2.dog_id WHERE T1.breed_code  =  ( SELECT breed_code FROM Dogs GROUP BY breed_code ORDER BY count(*) ASC LIMIT 1 )	dog_kennels
+SELECT Name FROM conductor WHERE Nationality != 'USA'	orchestra
+SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline WHERE T2.SourceAirport  =  "CVO" EXCEPT SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline WHERE T2.SourceAirport  =  "APG"	flight_2
+SELECT T1.lname FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pet_age  =  3 AND T3.pettype  =  'cat'	pets_1
+SELECT Population ,  LifeExpectancy FROM country WHERE Name  =  "Brazil"	world_1
+SELECT Name FROM country WHERE Continent  =  "Africa"  AND population  <  (SELECT max(population) FROM country WHERE Continent  =  "Asia")	world_1
+SELECT count(*) ,  District FROM city WHERE Population  >  (SELECT avg(Population) FROM city) GROUP BY District	world_1
+SELECT max(Final_Table_Made) FROM poker_player WHERE Earnings  <  200000	poker_player
+SELECT avg(Weight) ,  YEAR FROM CARS_DATA GROUP BY YEAR;	car_1
+SELECT name ,  grade FROM Highschooler	network_1
+SELECT grade FROM Highschooler WHERE name  =  "Kyle"	network_1
+SELECT avg(horsepower) FROM CARS_DATA WHERE YEAR  <  1980;	car_1
+SELECT t2.visitor_id ,  t1.name ,  t1.Level_of_membership FROM visitor AS t1 JOIN visit AS t2 ON t1.id  =  t2.visitor_id GROUP BY t2.visitor_id ORDER BY sum(t2.Total_spent) DESC LIMIT 1	museum_visit
+SELECT T2.name FROM singer_in_concert AS T1 JOIN singer AS T2 ON T1.singer_id  =  T2.singer_id JOIN concert AS T3 ON T1.concert_id  =  T3.concert_id WHERE T3.year  =  2014	concert_singer
+SELECT DISTINCT T2.Model FROM CAR_NAMES AS T1 JOIN MODEL_LIST AS T2 ON T1.Model  =  T2.Model JOIN CAR_MAKERS AS T3 ON T2.Maker  =  T3.Id JOIN CARS_DATA AS T4 ON T1.MakeId  =  T4.Id WHERE T3.FullName  =  'General Motors' OR T4.weight  >  3500;	car_1
+SELECT country_code FROM players GROUP BY country_code ORDER BY count(*) DESC LIMIT 1	wta_1
+SELECT T1.model FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id WHERE T2.Weight  <  (SELECT avg(Weight) FROM CARS_DATA)	car_1
+SELECT Country FROM AIRLINES WHERE Airline  =  "JetBlue Airways"	flight_2
+SELECT count(*) FROM Professionals WHERE professional_id NOT IN ( SELECT professional_id FROM Treatments )	dog_kennels
+SELECT T1.series_name FROM TV_Channel AS T1 JOIN TV_series AS T2 ON T1.id = T2.Channel WHERE T2.Episode = "A Love of a Lifetime";	tvshow
+SELECT avg(T2.Earnings) FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID WHERE T1.Height  >  200	poker_player
+SELECT T1.City FROM AIRPORTS AS T1 JOIN FLIGHTS AS T2 ON T1.AirportCode  =  T2.DestAirport GROUP BY T1.City ORDER BY count(*) DESC LIMIT 1	flight_2
+SELECT DISTINCT T1.Maker FROM CAR_MAKERS AS T1 JOIN MODEL_LIST AS T2 ON T1.Id  =  T2.Maker JOIN CAR_NAMES AS T3 ON T2.model  =  T3.model JOIN CARS_DATA AS T4 ON T3.MakeId  =  T4.id WHERE T4.year  =  '1970';	car_1
+SELECT professional_id ,  last_name ,  cell_number FROM Professionals WHERE state  =  'Indiana' UNION SELECT T1.professional_id ,  T1.last_name ,  T1.cell_number FROM Professionals AS T1 JOIN Treatments AS T2 ON T1.professional_id  =  T2.professional_id GROUP BY T1.professional_id HAVING count(*)  >  2	dog_kennels
+SELECT DISTINCT T1.first_name ,  T1.last_name FROM Professionals AS T1 JOIN Treatments AS T2 WHERE cost_of_treatment  <  ( SELECT avg(cost_of_treatment) FROM Treatments )	dog_kennels
+SELECT first_name ,  birth_date FROM players WHERE country_code  =  'USA'	wta_1
+SELECT Citizenship FROM singer WHERE Birth_Year  <  1945 INTERSECT SELECT Citizenship FROM singer WHERE Birth_Year  >  1955	singer
+SELECT song_name ,  song_release_year FROM singer ORDER BY age LIMIT 1	concert_singer
+SELECT Name FROM country ORDER BY Population DESC LIMIT 3	world_1
+SELECT DISTINCT T1.first_name ,  T1.last_name FROM Professionals AS T1 JOIN Treatments AS T2 WHERE cost_of_treatment  <  ( SELECT avg(cost_of_treatment) FROM Treatments )	dog_kennels
+SELECT sum(ranking_points) ,  T1.first_name FROM players AS T1 JOIN rankings AS T2 ON T1.player_id  =  T2.player_id GROUP BY T1.first_name	wta_1
+SELECT count(*) FROM MODEL_LIST AS T1 JOIN CAR_MAKERS AS T2 ON T1.Maker  =  T2.Id JOIN COUNTRIES AS T3 ON T2.Country  =  T3.CountryId WHERE T3.CountryName  =  'usa';	car_1
+SELECT Episode FROM TV_series ORDER BY rating	tvshow
+SELECT Title FROM Cartoon ORDER BY title	tvshow
+SELECT count(*) FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id WHERE T2.name  =  "Kyle"	network_1
+SELECT count(DISTINCT pettype) FROM pets	pets_1
+SELECT email_address ,  cell_number ,  home_phone FROM professionals	dog_kennels
+SELECT major ,  age FROM student WHERE stuid NOT IN (SELECT T1.stuid FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pettype  =  'cat')	pets_1
+SELECT Name FROM conductor ORDER BY Age ASC	orchestra
+SELECT winner_name ,  loser_name FROM matches ORDER BY minutes DESC LIMIT 1	wta_1
+SELECT country FROM TV_Channel EXCEPT SELECT T1.country FROM TV_Channel AS T1 JOIN cartoon AS T2 ON T1.id = T2.Channel WHERE T2.written_by  =  'Todd Casey'	tvshow
+SELECT Country ,  count(*) FROM TV_Channel GROUP BY Country ORDER BY count(*) DESC LIMIT 1;	tvshow
+SELECT T2.Language FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.GovernmentForm  =  "Republic" GROUP BY T2.Language HAVING COUNT(*)  =  1	world_1
+SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline WHERE T2.DestAirport  =  "AHD"	flight_2
+SELECT DISTINCT T2.Name FROM country AS T1 JOIN city AS T2 ON T2.CountryCode  =  T1.Code WHERE T1.Continent  =  'Europe' AND T1.Name NOT IN (SELECT T3.Name FROM country AS T3 JOIN countrylanguage AS T4 ON T3.Code  =  T4.CountryCode WHERE T4.IsOfficial  =  'T' AND T4.Language  =  'English')	world_1
+SELECT section_description FROM Sections WHERE section_name  =  'h'	student_transcripts_tracking
+SELECT T1.series_name FROM TV_Channel AS T1 JOIN Cartoon AS T2 ON T1.id = T2.Channel WHERE T2.Title = "The Rise of the Blue Beetle!";	tvshow
+SELECT Name FROM conductor WHERE Nationality != 'USA'	orchestra
+SELECT T1.Continent FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode GROUP BY T1.Continent ORDER BY COUNT(*) DESC LIMIT 1	world_1
+SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  "English" INTERSECT SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  "French"	world_1
+SELECT T1.FlightNo FROM FLIGHTS AS T1 JOIN AIRLINES AS T2 ON T2.uid  =  T1.Airline WHERE T2.Airline  =  "United Airlines"	flight_2
+SELECT LANGUAGE ,  count(*) FROM TV_Channel GROUP BY LANGUAGE	tvshow
+SELECT T2.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id GROUP BY T1.student_id HAVING count(*)  >=  3	network_1
+SELECT Air_Date FROM TV_series WHERE Episode = "A Love of a Lifetime";	tvshow
+SELECT count(*) FROM Dogs WHERE age  <  ( SELECT avg(age) FROM Dogs )	dog_kennels
+SELECT T1.Continent FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode GROUP BY T1.Continent ORDER BY COUNT(*) DESC LIMIT 1	world_1
+SELECT name FROM shop WHERE number_products  >  (SELECT avg(number_products) FROM shop)	employee_hire_evaluation
+SELECT count(*) FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T2.Airline  =  T1.uid WHERE T1.Airline  =  "United Airlines" AND T2.SourceAirport  =  "AHD"	flight_2
+SELECT template_type_code ,  count(*) FROM Templates GROUP BY template_type_code	cre_Doc_Template_Mgt
+SELECT Title FROM Cartoon WHERE Directed_by = "Ben Jones" OR Directed_by = "Brandon Vietti";	tvshow
+SELECT T1.paragraph_id ,   T1.paragraph_text FROM Paragraphs AS T1 JOIN Documents AS T2 ON T1.document_id  =  T2.document_id WHERE T2.Document_Name  =  'Welcome to NY'	cre_Doc_Template_Mgt
+SELECT DISTINCT T1.template_type_description FROM Ref_template_types AS T1 JOIN Templates AS T2 ON T1.template_type_code  = T2.template_type_code JOIN Documents AS T3 ON T2.Template_ID  =  T3.template_ID	cre_Doc_Template_Mgt
+SELECT contestant_name FROM contestants WHERE contestant_name != 'Jessie Alloway'	voter_1
+SELECT T2.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id WHERE T2.grade  >  5 GROUP BY T1.student_id HAVING count(*)  >=  2	network_1
+SELECT count(*) FROM FLIGHTS WHERE SourceAirport  =  "APG"	flight_2
+SELECT DISTINCT T3.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode JOIN city AS T3 ON T1.Code  =  T3.CountryCode WHERE T2.IsOfficial  =  'T' AND T2.Language  =  'Chinese' AND T1.Continent  =  "Asia"	world_1
+SELECT T1.fname ,  T1.sex FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid GROUP BY T1.stuid HAVING count(*)  >  1	pets_1
+SELECT T1.FlightNo FROM FLIGHTS AS T1 JOIN AIRPORTS AS T2 ON T1.DestAirport   =  T2.AirportCode WHERE T2.City  =  "Aberdeen"	flight_2
+SELECT avg(ranking) ,  T1.first_name FROM players AS T1 JOIN rankings AS T2 ON T1.player_id  =  T2.player_id GROUP BY T1.first_name	wta_1
+SELECT T2.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id INTERSECT SELECT T2.name FROM Likes AS T1 JOIN Highschooler AS T2 ON T1.liked_id  =  T2.id	network_1
+SELECT count(*) FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id WHERE T2.name  =  "Kyle"	network_1
+SELECT Model FROM CAR_NAMES GROUP BY Model ORDER BY count(*) DESC LIMIT 1;	car_1
+SELECT count(DISTINCT loser_name) FROM matches	wta_1
+SELECT T1.owner_id ,  T2.first_name ,  T2.last_name FROM Dogs AS T1 JOIN Owners AS T2 ON T1.owner_id  =  T2.owner_id GROUP BY T1.owner_id ORDER BY count(*) DESC LIMIT 1	dog_kennels
+SELECT weight FROM pets ORDER BY pet_age LIMIT 1	pets_1
+SELECT T1.student_id ,  T1.first_name ,  T1.middle_name ,  T1.last_name ,  count(*) ,  T1.student_id FROM Students AS T1 JOIN Student_Enrolment AS T2 ON T1.student_id  =  T2.student_id GROUP BY T1.student_id ORDER BY count(*) DESC LIMIT 1	student_transcripts_tracking
+SELECT Hometown ,  COUNT(*) FROM teacher GROUP BY Hometown	course_teach
+SELECT T2.document_name FROM Templates AS T1 JOIN Documents AS T2 ON T1.template_id  =  T2.template_id WHERE T1.template_type_code  =  "BK"	cre_Doc_Template_Mgt
+SELECT T1.Name FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID WHERE T2.Earnings  >  300000	poker_player
+SELECT winner_name FROM matches WHERE YEAR  =  2013 INTERSECT SELECT winner_name FROM matches WHERE YEAR  =  2016	wta_1
+SELECT DISTINCT T1.Name FROM singer AS T1 JOIN song AS T2 ON T1.Singer_ID  =  T2.Singer_ID WHERE T2.Sales  >  300000	singer
+SELECT Record_Company FROM orchestra GROUP BY Record_Company ORDER BY COUNT(*) DESC LIMIT 1	orchestra
+SELECT avg(Weight) ,  YEAR FROM CARS_DATA GROUP BY YEAR;	car_1
+SELECT DISTINCT CountryCode FROM countrylanguage WHERE LANGUAGE != "English"	world_1
+SELECT Count(*) ,  T2.FullName ,  T2.id FROM MODEL_LIST AS T1 JOIN CAR_MAKERS AS T2 ON T1.Maker  =  T2.Id GROUP BY T2.id;	car_1
+SELECT count(*) ,  YEAR FROM matches GROUP BY YEAR	wta_1
+SELECT count(*) FROM CARS_DATA WHERE horsepower  >  150;	car_1
+SELECT other_student_details FROM Students ORDER BY other_student_details DESC	student_transcripts_tracking
+SELECT count(*) FROM country WHERE GovernmentForm  =  "Republic"	world_1
+SELECT T1.semester_name ,  T1.semester_id FROM Semesters AS T1 JOIN Student_Enrolment AS T2 ON T1.semester_id  =  T2.semester_id GROUP BY T1.semester_id ORDER BY count(*) DESC LIMIT 1	student_transcripts_tracking
+SELECT document_id FROM Paragraphs WHERE paragraph_text  =  'Brazil' INTERSECT SELECT document_id FROM Paragraphs WHERE paragraph_text  =  'Ireland'	cre_Doc_Template_Mgt
+SELECT T1.series_name FROM TV_Channel AS T1 JOIN TV_series AS T2 ON T1.id = T2.Channel WHERE T2.Episode = "A Love of a Lifetime";	tvshow
+SELECT name FROM Highschooler EXCEPT SELECT T2.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id	network_1
+SELECT T2.name FROM Likes AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id GROUP BY T1.student_id ORDER BY count(*) DESC LIMIT 1	network_1
+SELECT name FROM employee WHERE Employee_ID NOT IN (SELECT Employee_ID FROM evaluation)	employee_hire_evaluation
+select production_code ,  channel from cartoon order by original_air_date desc limit 1	tvshow
+select avg(horsepower) from cars_data where year  <  1980;	car_1
+SELECT Record_Company FROM orchestra WHERE Year_of_Founded  <  2003 INTERSECT SELECT Record_Company FROM orchestra WHERE Year_of_Founded  >  2003	orchestra
+SELECT T2.name ,  count(*) FROM singer_in_concert AS T1 JOIN singer AS T2 ON T1.singer_id  =  T2.singer_id GROUP BY T2.singer_id	concert_singer
+SELECT name ,  LOCATION ,  district FROM shop ORDER BY number_products DESC	employee_hire_evaluation
+SELECT count(*) FROM players	wta_1
+SELECT student_id FROM Friend INTERSECT SELECT liked_id FROM Likes	network_1
+SELECT T1.series_name FROM TV_Channel AS T1 JOIN Cartoon AS T2 ON T1.id = T2.Channel WHERE T2.Title = "The Rise of the Blue Beetle!";	tvshow
+SELECT T2.Language FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.GovernmentForm  =  "Republic" GROUP BY T2.Language HAVING COUNT(*)  =  1	world_1
+SELECT document_id ,  template_id ,  Document_Description FROM Documents WHERE document_name  =  "Robbin CV"	cre_Doc_Template_Mgt
+SELECT T1.owner_id ,  T1.last_name FROM Owners AS T1 JOIN Dogs AS T2 ON T1.owner_id  =  T2.owner_id JOIN Treatments AS T3 ON T2.dog_id  =  T3.dog_id GROUP BY T1.owner_id ORDER BY count(*) DESC LIMIT 1	dog_kennels
+SELECT DISTINCT template_type_code FROM Templates	cre_Doc_Template_Mgt
+SELECT count(*) FROM CARS_DATA WHERE Cylinders  >  4;	car_1
+SELECT ID FROM Highschooler WHERE name  =  "Kyle"	network_1
+SELECT Name FROM teacher WHERE Teacher_id NOT IN (SELECT Teacher_id FROM course_arrange)	course_teach
+SELECT first_name ,  middle_name ,  last_name FROM Students ORDER BY date_left ASC LIMIT 1	student_transcripts_tracking
+SELECT T1.Name FROM conductor AS T1 JOIN orchestra AS T2 ON T1.Conductor_ID  =  T2.Conductor_ID GROUP BY T2.Conductor_ID ORDER BY COUNT(*) DESC LIMIT 1	orchestra
+SELECT T1.treatment_type_description FROM Treatment_types AS T1 JOIN Treatments AS T2 ON T1.treatment_type_code  =  T2.treatment_type_code GROUP BY T1.treatment_type_code ORDER BY sum(cost_of_treatment) ASC LIMIT 1	dog_kennels
+SELECT T1.Model FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id WHERE T2.Cylinders  =  4 ORDER BY T2.horsepower DESC LIMIT 1;	car_1
+SELECT Country FROM AIRLINES WHERE Airline  =  "JetBlue Airways"	flight_2
+SELECT template_type_code FROM Templates GROUP BY template_type_code ORDER BY count(*) DESC LIMIT 1	cre_Doc_Template_Mgt
+SELECT DISTINCT T2.Model FROM CAR_NAMES AS T1 JOIN MODEL_LIST AS T2 ON T1.Model  =  T2.Model JOIN CAR_MAKERS AS T3 ON T2.Maker  =  T3.Id JOIN CARS_DATA AS T4 ON T1.MakeId  =  T4.Id WHERE T3.FullName  =  'General Motors' OR T4.weight  >  3500;	car_1
+SELECT T1.Model FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id ORDER BY T2.horsepower ASC LIMIT 1;	car_1
+SELECT count(*) ,  T1.stuid FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid GROUP BY T1.stuid	pets_1
+SELECT document_id FROM Paragraphs WHERE paragraph_text  =  'Brazil' INTERSECT SELECT document_id FROM Paragraphs WHERE paragraph_text  =  'Ireland'	cre_Doc_Template_Mgt
+SELECT template_id FROM Templates WHERE template_type_code  =  "PP" OR template_type_code  =  "PPT"	cre_Doc_Template_Mgt
+select contestant_name from contestants where contestant_name like "%al%"	voter_1
+SELECT count(DISTINCT Nationality) FROM people	poker_player
+SELECT count(*) FROM employee	employee_hire_evaluation
+SELECT Title FROM Cartoon WHERE Directed_by = "Ben Jones";	tvshow
+SELECT student_id FROM Friend INTERSECT SELECT liked_id FROM Likes	network_1
+SELECT grade FROM Highschooler	network_1
+SELECT Nationality FROM people GROUP BY Nationality HAVING COUNT(*)  >=  2	poker_player
+SELECT count(*) FROM CAR_MAKERS AS T1 JOIN COUNTRIES AS T2 ON T1.Country  =  T2.CountryId WHERE T2.CountryName  =  'france';	car_1
+SELECT T2.horsepower ,  T1.Make FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id WHERE T2.cylinders  =  3 ORDER BY T2.horsepower DESC LIMIT 1;	car_1
+SELECT FlightNo FROM FLIGHTS WHERE SourceAirport  =  "APG"	flight_2
+SELECT country FROM singer WHERE age  >  40 INTERSECT SELECT country FROM singer WHERE age  <  30	concert_singer
+SELECT winner_name ,  winner_rank_points FROM matches GROUP BY winner_name ORDER BY count(*) DESC LIMIT 1	wta_1
+SELECT grade ,  count(*) FROM Highschooler GROUP BY grade	network_1
+SELECT T1.owner_id ,  T1.zip_code FROM Owners AS T1 JOIN Dogs AS T2 ON T1.owner_id  =  T2.owner_id JOIN Treatments AS T3 ON T2.dog_id  =  T3.dog_id GROUP BY T1.owner_id ORDER BY sum(T3.cost_of_treatment) DESC LIMIT 1	dog_kennels
+SELECT count(*) FROM FLIGHTS	flight_2
+SELECT AirportCode ,  AirportName FROM AIRPORTS WHERE city  =  "Anthony"	flight_2
+SELECT T1.killed ,  T1.injured FROM death AS T1 JOIN ship AS t2 ON T1.caused_by_ship_id  =  T2.id WHERE T2.tonnage  =  't'	battle_death
+SELECT YEAR FROM matches GROUP BY YEAR ORDER BY count(*) DESC LIMIT 1	wta_1
+SELECT DISTINCT T1.Name FROM singer AS T1 JOIN song AS T2 ON T1.Singer_ID  =  T2.Singer_ID WHERE T2.Sales  >  300000	singer
+SELECT T2.Name FROM course_arrange AS T1 JOIN teacher AS T2 ON T1.Teacher_ID  =  T2.Teacher_ID GROUP BY T2.Name HAVING COUNT(*)  >=  2	course_teach
+SELECT count(*) FROM CARS_DATA WHERE horsepower  >  150;	car_1
+SELECT count(*) ,  LOCATION FROM shop GROUP BY LOCATION	employee_hire_evaluation
+SELECT T1.Name FROM singer AS T1 JOIN song AS T2 ON T1.Singer_ID  =  T2.Singer_ID GROUP BY T1.Name HAVING COUNT(*)  >  1	singer
+SELECT T1.last_name FROM Students AS T1 JOIN Addresses AS T2 ON T1.current_address_id  =  T2.address_id WHERE T2.state_province_county  =  'NorthCarolina' EXCEPT SELECT DISTINCT T3.last_name FROM Students AS T3 JOIN Student_Enrolment AS T4 ON T3.student_id  =  T4.student_id	student_transcripts_tracking
+SELECT t1.name FROM visitor AS t1 JOIN visit AS t2 ON t1.id  =  t2.visitor_id JOIN museum AS t3 ON t3.Museum_ID  =  t2.Museum_ID WHERE t3.open_year  <  2009 INTERSECT SELECT t1.name FROM visitor AS t1 JOIN visit AS t2 ON t1.id  =  t2.visitor_id JOIN museum AS t3 ON t3.Museum_ID  =  t2.Museum_ID WHERE t3.open_year  >  2011	museum_visit
+SELECT COUNT(T2.Language) FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.Name  =  "Aruba"	world_1
+SELECT Episode FROM TV_series ORDER BY rating	tvshow
+SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline GROUP BY T1.Airline ORDER BY count(*) DESC LIMIT 1	flight_2
+SELECT count(*) FROM COUNTRIES;	car_1
+SELECT count(*) FROM Courses	student_transcripts_tracking
+SELECT grade FROM Highschooler GROUP BY grade ORDER BY count(*) DESC LIMIT 1	network_1
+SELECT count(*) FROM country WHERE GovernmentForm  =  "Republic"	world_1
+SELECT T1.fname ,  T1.age FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pettype  =  'dog' AND T1.stuid NOT IN (SELECT T1.stuid FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pettype  =  'cat')	pets_1

file_copies_from_project_root/gold_base.txt

@@ -0,0 +1,10 @@
+SELECT count(*) ,  country_code FROM players GROUP BY country_code	wta_1
+SELECT count(*) FROM Documents AS T1 JOIN Templates AS T2 ON T1.Template_ID  =  T2.Template_ID WHERE T2.Template_Type_Code  =  'PPT'	cre_Doc_Template_Mgt
+SELECT count(*) FROM Cartoon WHERE Written_by = "Joseph Kuhr";	tvshow
+SELECT name ,  country ,  age FROM singer ORDER BY age DESC	concert_singer
+SELECT Name FROM country WHERE continent  =  "Europe" AND Population  =  "80000"	world_1
+SELECT count(*) FROM Highschooler WHERE grade  =  9 OR grade  =  10	network_1
+SELECT COUNT(*) FROM (SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  "English" INTERSECT SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  "Dutch")	world_1
+select name from teacher where hometown != "little lever urban district"	course_teach
+SELECT T1.address_id ,  T1.line_1 ,  T1.line_2 FROM Addresses AS T1 JOIN Students AS T2 ON T1.address_id  =  T2.current_address_id GROUP BY T1.address_id ORDER BY count(*) DESC LIMIT 1	student_transcripts_tracking
+SELECT Orchestra FROM orchestra WHERE Orchestra_ID NOT IN (SELECT Orchestra_ID FROM performance)	orchestra

file_copies_from_project_root/predict3_processed.txt

@@ -0,0 +1,200 @@
+SELECT country_code ,  count(*) FROM loser GROUP BY country_code
+SELECT count(*) FROM Documents AS T1 JOIN Templates AS T2 ON T1.template_id  =  T2.template_id WHERE T2.template_type_code  =  "PPT"
+SELECT count(*) FROM cartoon WHERE Directed_by  =  "Joseph Kuhr"�,  Title  =  "18 49 Rating Share"
+SELECT T1.Name ,  T1.Country ,  T2.Age FROM concert_record AS T1 JOIN singer AS T2 ON T1.Concert_ID  =  T2.Concert_ID ORDER BY T2.Age
+SELECT name FROM country_codes WHERE country_code  =  "Europe" AND population  =  80000
+SELECT count(*) FROM list AS t1 JOIN students AS t2 ON t1.stuid  =  t2.stuid WHERE t1.name  =  "high schooler" AND t1.grade  =  9 OR t1.grade  =  10
+SELECT count(DISTRICT) FROM city WHERE language  =  'English' AND language  =  'Dutch'
+SELECT T2.name FROM list AS T1 JOIN teacher AS T2 ON T1.Teacher_ID  =  T2.Teacher_ID WHERE T1.hometown != 'Little Lever Urban District'
+SELECT T1.address_id ,  T1.line_1 ,  T1.line_2 ,  max(T2.course_id) FROM addresses AS T1 JOIN course_sections AS T2 ON T1.address_id  =  T2.current_address_id GROUP BY T1.address_id ORDER BY count(T2.course_id) DESC LIMIT 1;
+SELECT T1.Orchestra FROM performance AS T1 JOIN orchestra AS T2 ON T1.Orchestra_ID  =  T2.Orchestra_ID WHERE T1.Performance NOT IN (SELECT T1.Performance FROM performance AS T1 JOIN performance_detail AS T2 ON T1.Performance_ID  =  T2.Performance_ID GROUP BY T1.Performance_ID HAVING COUNT(*)  =  1)
+SELECT model FROM motor_cycle WHERE mpg  <  3500 EXCEPT SELECT T1.model FROM model AS T1 JOIN motor_cycle AS T2 ON T1.id  =  T2.model WHERE T2.maker  =  'Ford Motor Company'
+SELECT count(DISTINCT Conductor_ID) FROM conductor
+SELECT T1.name ,  T1.capacity FROM concert AS T1 JOIN stadium AS T2 ON T1.Stadium_ID  =  T2.Stadium_ID GROUP BY T2.Stadium_ID HAVING count(*)  >=  (SELECT count(*) FROM concert WHERE YEAR  >=  2013) ORDER BY T2.capacity DESC LIMIT 1
+SELECT grade FROM students AS T1 JOIN friends AS T2 ON T1.StuID  =  T2.StuID WHERE T1.name  =  "Kyle"
+SELECT semester_name FROM student_enrolments GROUP BY semester_name UNION SELECT semester_name FROM student_enrolments_masters GROUP BY semester_name UNION SELECT semester_name FROM student_enrolments_baccalaureate
+SELECT avg(life_expectancy) FROM country WHERE country_code  =  "Africa" AND government_form  =  "Republic"
+SELECT Language FROM country WHERE Head_of_State  =  "Beatrix"
+SELECT T1.first_name ,  T1.middle_name ,  T1.last_name FROM student_registrations AS T1 JOIN addresses AS T2 ON T1.current_address_id  =  T2.address_id WHERE T1.student_id  =  (SELECT T1.student_id FROM student_registrations AS T1 JOIN addresses AS T2 ON T1.current_address_id  =  T2.address_id ORDER BY T1.student_id DESC LIMIT 1)
+SELECT count(*) FROM pet WHERE age  >  20 AND student_id IN (SELECT student_id FROM pet_has_pet WHERE pet_id IN (SELECT pet_id FROM pet_has_pet WHERE student_id  >  20) ,  pet_id IN (SELECT pet_id FROM pet_has_pet WHERE student_id  >  20) ,  pet_id IN (SELECT pet_id FROM pet_has_pet WHERE student_id  >  20) ,  pet_id IN (SELECT pet_id FROM pet_has_pet WHERE student_
+SELECT section_name FROM course_sections ORDER BY section_name LIMIT 10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+SELECT count(*) FROM Documents AS T1 JOIN Paragraphs AS T2 ON T1.document_id  =  T2.paragraph_id JOIN Documents AS T3 ON T2.paragraph_id  =  T3.paragraph_id WHERE T3.document_name  =  "Summer Show"
+SELECT T1.document_id FROM Document_contents AS T1 JOIN Paragraphs AS T2 ON T1.paragraph_id  =  T2.paragraph_id GROUP BY T1.document_id HAVING count(*)  >=  2
+SELECT country FROM channel WHERE channel_id NOT IN (SELECT id FROM 18_49_Rating_Share)
+SELECT country_code ,  country_code FROM city WHERE language  =  'English' OR language  =  'Dutch'
+SELECT count(*) FROM maker WHERE full_name  =  "American Motor Company"
+SELECT name FROM shop EXCEPT SELECT T1.name FROM shop AS T1 JOIN employee AS T2 ON T1.shop_id  =  T2.shop_id WHERE T2.employee_id  =  (SELECT employee_id FROM shop WHERE name NOT IN (SELECT T1.name FROM shop AS T1 JOIN employee AS T2 ON T1.shop_id  =  T2.shop_id WHERE T2.employee_id  =  (SELECT T1.employee_id FROM shop AS T1 JOIN employee AS T2 ON T1.shop_id  =
+SELECT t1.id FROM templates AS t1 JOIN document_templates AS t2 ON t1.template_id  =  t2.template_id EXCEPT SELECT t1.id FROM document_templates AS t1 JOIN documents AS t2 ON t1.template_id  =  t2.template_id�
+SELECT course_name FROM course_enrolments GROUP BY course_name HAVING count(*)  >  1
+SELECT t1.template_id FROM templates AS t1 JOIN document_templates AS t2 ON t1.template_id  =  t2.template_id WHERE t2.document_name  =  "Presentation"�
+SELECT count(DISTINCT Teacher_ID) FROM course
+SELECT T2.template_type_code FROM template_details AS T1 JOIN template_type AS T2 ON T1.template_type_code  =  T2.template_type_code GROUP BY T2.template_type_code HAVING count(*)  <  3
+SELECT T1.conductor_id ,  count(*) FROM performance AS T1 JOIN orchestra AS T2 ON T1.orchestra_id  =  T2.orchestra_id JOIN record_company AS T3 ON T1.record_company  =  T3.record_company GROUP BY T1.conductor_id ,  T3.record_companyC, T2.orchestra_id
+SELECT Name FROM people EXCEPT SELECT T1.Name FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID WHERE T2.Poker_Player_ID NOT IN (SELECT T1.People_ID FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID WHERE T2.Poker_Player_ID IN (SELECT T1.People_ID FROM people AS T1 JOIN poker_player AS T2
+SELECT T2.feature_type_description FROM features AS T1 JOIN property_features AS T2 ON T1.feature_id  =  T2.feature_id WHERE T1.feature_type_code  =  "APT_FEATURE_3"
+SELECT T1.name ,  T2.course FROM course AS T1 JOIN teacher AS T2 ON T1.course  =  T2.course_id ORDER BY T1.name ASC
+SELECT T2.template_type_code FROM TEMPLATES AS T1 JOIN TEMPLATE_TYPES AS T2 ON T1.template_type_code  =  T2.template_type_code WHERE T2.template_type_description  =  "Book"F
+SELECT DISTINCT T1.Full_name ,  T1.Maker FROM maker AS T1 JOIN models AS T2 ON T1.Maker  =  T2.Maker GROUP BY T1.Maker HAVING count(*)  >  3�, T2.Model
+SELECT feature_type_name FROM Features WHERE feature_type_code  =  "AirCon"
+SELECT department_description FROM course_sections WHERE course_name LIKE "%computer%"
+SELECT T1.first_name FROM addresses AS T1 JOIN student_addresses AS T2 ON T1.address_id  =  T2.current_address_id WHERE T2.cell_mobile_number  =  09700166582 OR T2.cell_mobile_number  =  09700166582 AND T2.course_id  =  1000000000000000000000000000000000
+SELECT t1.abbreviation FROM airlines AS t1 JOIN flight AS t2 ON t1.airline  =  t2.source_airline WHERE t1.abbreviation  =  "UAL"
+SELECT T2.charge_type_code ,  T2.charge_amount FROM CHARGES AS T1 JOIN TREATMENTS AS T2 ON T1.treatment_id  =  T2.treatment_id GROUP BY T2.charge_type_code ,  T2.charge_amount
+SELECT T1.dog_id ,  T1.name ,  T1.cell_number FROM dog AS T1 JOIN treatment AS T2 ON T1.dog_id  =  T2.dog_id WHERE T2.treatment_type_code  =  "Treatment_Type_Professional" INTERSECT SELECT T1.dog_id ,  T1.name ,  T1.cell_number FROM dog AS T1 JOIN treatment AS T2 ON T1.dog_id  =  T2.dog_id WHERE T2.
+SELECT pet_id FROM pet WHERE pet_weight  >  10+
+SELECT T1.first_name ,  T1.birth_date FROM player AS T1 JOIN loser AS T2 ON T1.player_id  =  T2.loser_id WHERE T2.country_code  =  'USA'
+SELECT Model FROM car WHERE Weight  <  (SELECT avg(Weight) FROM car)
+SELECT Name ,  Date_of_Treatment FROM dog WHERE Treatment_Type_Code  =  (SELECT Treatment_Type_Code FROM dog_treatment_type WHERE breed_code  =  (SELECT breed_code FROM dog ORDER BY RANDOM()) ) ORDER BY RANDOM(), DATE_OF_TREATMENT DESC LIMIT 1
+SELECT T1.Name FROM conductor AS T1 JOIN orchestra AS T2 ON T1.Conductor_ID  =  T2.Conductor_ID WHERE T1.Nationality != "USA"
+SELECT DISTINCT T1.airline_id FROM airlines AS T1 JOIN flight AS T2 ON T1.airline_id  =  T2.source_airline WHERE T2.destination_airport  =  'CVO' EXCEPT SELECT DISTINCT T1.airline_id FROM airlines AS T1 JOIN flight AS T2 ON T1.airline_id  =  T2.source_airline WHERE T2.destination_airport  =  'APG'
+SELECT T3.lname FROM pet AS T1 JOIN has_pet AS T2 ON T1.petid  =  T2.petid JOIN student AS T3 ON T2.stuid  =  T3.stuid WHERE T2.age  =  3 AND T1.pettype  =  "cat" AND T1.petid  =  select petid FROM has_pet WHERE pettype  =  "cat" GROUP BY petid ORDER BY petid DESC LIMIT 1
+SELECT Population ,  Life_Expectancy FROM district WHERE country_code  =  'Brazil'
+SELECT T1.country_code FROM country AS T1 JOIN city AS T2 ON T1.country_code  =  T2.country_code WHERE T2.population  <  (SELECT min(T3.population) FROM city AS T3 JOIN country AS T4 ON T3.country_code  =  T4.country_code WHERE T4.continent  =  'Asia' AND T3.district  =  T4.district_code)C, T2.district_code IN (SELECT district_code FROM city WHERE country_
+SELECT district ,  count(*) FROM city GROUP BY district HAVING count(*)  >  (SELECT avg(count()) FROM city)
+SELECT max(T1.Best_Finish) FROM POKER_Player AS T1 JOIN Final_Table AS T2 ON T1.POKER_Player_ID  =  T2.Final_Table_Made WHERE T1.Earnings  <  200000�, T2.people_id NOT IN (SELECT people_id FROM POKER_Player WHERE Nationality  =  'Britain' OR Nationality  =  'United States of America' OR Nationality  =  'Canada' OR Nationality  =  '
+SELECT avg(weight) ,  YEAR FROM motorcycle GROUP BY YEAR�, MATCH (year  =  (SELECT min(YEAR) FROM motorcycle) OR YEAR  =  (SELECT max(YEAR) FROM motorcycle) )
+SELECT name ,  grade FROM school WHERE name  =  "johnson high school" UNION SELECT name ,  grade FROM school WHERE name  =  "johnson high school for boys" UNION SELECT name ,  grade FROM school WHERE name  =  "johnson high school for girls"
+SELECT grade FROM students WHERE name  =  "Kyle"
+SELECT avg(Hp) FROM car WHERE YEAR  <  1980
+SELECT T1.mID ,  T1.name ,  T1.level_of_membership FROM customer AS T2 JOIN museum AS T1 ON T2.museum_id  =  T1.mID GROUP BY T2.mID ORDER BY sum(T2.total_spent) DESC LIMIT 1F
+SELECT T1.Name FROM concert AS T1 JOIN singer AS T2 ON T1.Concert_ID  =  T2.Concert_ID WHERE T1.Year  =  2014 AND T2.Is_Male  =  'Yes'F, AND T1.Concert_Name  =  "Concert in 2014"
+SELECT DISTINCT T1.Model FROM maker AS T1 JOIN models AS T2 ON T1.MID  =  T2.MID WHERE T1.Full_Name  =  "General Motors" OR T1.Weigh  >  3500
+SELECT country_code FROM player GROUP BY country_code ORDER BY sum(player_rank_points) DESC LIMIT 1/]{'country_code', 'country code', 'hand', 'birth date', 'country code', 'best of', 'draw size', 'loser age', 'loser entry', 'loser hand', 'loser ht', 'loser id', 'loser ioc', 'loser name', 'loser rank', 'loser rank points', 'loser seed', 'match num', 'minutes', 'round', 'score', 'surface', 't
+SELECT Model FROM car WHERE Weight  <  (SELECT avg(Weight) FROM car)
+SELECT country_abbrev FROM airlines WHERE airline  =  'Jetblue Airways'
+SELECT count(*) FROM dog_treatment WHERE dog_id NOT IN (SELECT dog_id FROM dog_treatment WHERE treatment_type_code  =  "Treatment_Type_Code_for_Professionals_operating_on_dogs")
+SELECT T1.Channel FROM series AS T1 JOIN episode AS T2 ON T1.ID  =  T2.Series_ID WHERE T2.Episodes  =  "A Love of a Lifetime" AND T2.Channel  =  "Channel 100"�
+SELECT avg(EARNINGS) FROM poker_player WHERE People_ID  >  200 AND Height  >  200
+SELECT destination_airport FROM flight GROUP BY destination_airport ORDER BY count(*) DESC LIMIT 1
+SELECT DISTINCT T1.Full_name FROM maker AS T1 JOIN models AS T2 ON T1.Maker_ID  =  T2.Maker_ID WHERE T2.Year  =  1970�, T1.Full_name != 'Toyota' AND T1.Full_name != 'Honda' AND T1.Full_name != 'Suzuki' AND T1.Full_name != 'Kawasaki' AND T1.Full_name != 'Yamaha' AND T1.Full_name != 'KTM
+SELECT T1.dog_id ,  T2.last_name ,  T2.cell_number FROM dog AS T1 JOIN professional AS T2 ON T1.dog_id  =  T2.dog_id WHERE T2.state  =  'Indiana' INTERSECT SELECT T1.dog_id ,  T2.last_name ,  T2.cell_number FROM dog AS T1 JOIN professional AS T2 ON T1.dog_id  =  T2.dog_id GROUP BY T1.dog_id HAVING count(*)
+SELECT T2.first_name ,  T2.last_name FROM dog_treatment AS T1 JOIN professional AS T2 ON T1.professional_id  =  T2.professional_id WHERE T1.treatment_type_code  =  (SELECT cost_of_treatment FROM dog_treatment GROUP BY cost_of_treatment ORDER BY avg(cost_of_treatment) DESC LIMIT 1) AND T1.dog_id NOT IN (SELECT dog_id FROM dog_treatment WHERE cost_of_
+SELECT T1.first_name ,  T1.birth_date FROM loser AS T1 JOIN player AS T2 ON T1.loser_name  =  T2.first_name AND T1.loser_birth_date  =  T2.birth_date AND T1.loser_country_code  =  'USA'/]{'SELECT first_name ,  birth_date FROM loser AS T1 JOIN player AS T2 ON T1.loser_name  =  T2.first_name AND T1.loser_b
+SELECT Citizenship FROM singer WHERE birth_year BETWEEN 1945 AND 1955 INTERSECT SELECT Citizenship FROM singer WHERE birth_year BETWEEN 1955 AND 1965+, AND Singer_id IN (SELECT T1.Singer_id FROM performance AS T1 JOIN singer AS T2 ON T1.Singer_id  =  T2.Singer_id WHERE T2.birth_year BETWEEN 1945 AND 1955)
+SELECT T2.Song_Name ,  T2.Song_Release_Year FROM CONCERT AS T1 JOIN SINGER AS T2 ON T1.Singer_ID  =  T2.Singer_ID ORDER BY T1.Year LIMIT 1
+SELECT name FROM country GROUP BY name ORDER BY population DESC LIMIT 3_, 1;
+SELECT T2.first_name ,  T2.last_name FROM dog_treatment AS T1 JOIN professional AS T2 ON T1.professional_id  =  T2.professional_id WHERE T1.cost_of_treatment  <  (SELECT avg(T3.cost_of_treatment) FROM dog_treatment AS T2 JOIN professional AS T3 ON T2.professional_id  =  T3.professional_id GROUP BY T2.professional_id ORDER BY avg(T3.cost_
+SELECT T1.first_name ,  T1.loser_rank FROM loser_record AS T1 JOIN player AS T2 ON T1.loser_id  =  T2.player_id GROUP BY T1.loser_id ORDER BY T2.loser_rank_points DESC LIMIT 10;
+SELECT count(*) FROM model WHERE country_id  =  "United States"
+SELECT T1.Episode FROM series AS T1 JOIN episode AS T2 ON T1.Series_ID  =  T2.Series_ID ORDER BY T2.Rating
+SELECT title FROM cartoons ORDER BY title ASC�, series_name ASC
+SELECT count(*) FROM has_friend AS T1 JOIN student AS T2 ON T1.StuID  =  T2.StuID WHERE T2.name  =  "Kyle"
+SELECT count(DISTINCT pet_type) FROM pet
+SELECT T3.email_address ,  T3.cell_number ,  T3.home_phone FROM dog AS T1 JOIN professional AS T2 ON T1.dog_id  =  T2.dog_id JOIN treatment AS T3 ON T2.treatment_id  =  T3.treatment_id WHERE T1.dog_id  =  (SELECT dog_id FROM dog_treatment ORDER BY date_of_treatment DESC LIMIT 1)
+SELECT Major ,  age FROM STUDENT EXCEPT SELECT T1.Major ,  T1.age FROM STUDENT AS T1 JOIN PET AS T2 ON T1.StuID  =  T2.StuID WHERE T2.PetType  =  'Cat'F
+SELECT T1.Name FROM conductor AS T1 JOIN performance AS T2 ON T1.Conductor_ID  =  T2.Conductor_ID ORDER BY T1.Age ASC+, T2.Date ASC
+SELECT T1.loser_name ,  T2.winner_name FROM loser AS T1 JOIN winner AS T2 ON T1.loser_id  =  T2.winner_id WHERE T1.match_num  =  (SELECT max(match_num) FROM loser) GROUP BY T1.loser_name ,  T2.winner_name ORDER BY T1.loser_rank_points DESC,  T2.winner_rank_points DESC
+SELECT Country FROM channel EXCEPT SELECT T1.Country FROM channel AS T1 JOIN cartoon AS T2 ON T1.Channel  =  T2.Channel WHERE T2.Written_by  =  'Todd Casey'/]{'SELECT Country FROM channel EXCEPT SELECT T1.Country FROM channel AS T1 JOIN cartoon AS T2 ON T1.Channel  =  T2.Channel WHERE T2.Written_by  =  'Todd Casey'}{SELECT Country FROM channel EXCEPT SELECT T1.Country FROM channel AS T1 JOIN cartoon AS
+SELECT country ,  count(*) FROM tv_channel GROUP BY country ORDER BY count(*) DESC LIMIT 1
+SELECT language FROM country GROUP BY language HAVING min(id) = 1 AND country_code  =  country_codes_in_republic_governments ORDER BY min(id) DESC LIMIT 1
+SELECT T2.airline FROM flight AS T1 JOIN airline AS T2 ON T1.source_airport  =  T2.airport_code WHERE T1.destination_airport  =  'AHD'
+SELECT name FROM city WHERE country_code  =  'Europe' EXCEPT SELECT name FROM city WHERE language  =  'english'
+SELECT course_description FROM course WHERE course_name  =  "Human Physiology"
+SELECT TV_Channel FROM series AS T1 JOIN content AS T2 ON T1.Channel  =  T2.Channel WHERE T2.Series_Name  =  "The Rise of the Blue Beetle"
+SELECT T1.Name FROM conductor AS T1 JOIN orchestra_conductor AS T2 ON T1.Conductor_ID  =  T2.Conductor_ID WHERE T2.Nationality NOT IN (SELECT Nationality FROM conductor WHERE Conductor_ID NOT IN (SELECT T1.Conductor_ID FROM conductor AS T1 JOIN orchestra_conductor AS T2 ON T1.Conductor_ID  =  T2.Conductor_ID GROUP BY T1.Conductor_ID ORDER BY count(*) DESC LIMIT 1
+SELECT T2.Continent FROM city AS T1 JOIN country AS T2 ON T1.Continent  =  T2.Country_Code GROUP BY T2.Continent ORDER BY COUNT(*) DESC LIMIT 1
+SELECT T1.name FROM country AS T1 JOIN city AS T2 ON T1.country_code  =  T2.country_code WHERE T2.language  =  'French' INTERSECT SELECT T1.name FROM country AS T1 JOIN city AS T2 ON T1.country_code  =  T2.country_code WHERE T2.language  =  'English'
+SELECT flight_number FROM airlines WHERE airline  =  "United Airlines"
+SELECT Language ,  COUNT(*) FROM tv_channel GROUP BY Language
+SELECT name FROM student WHERE grade  >=  3 GROUP BY name HAVING count(*)  >=  3�
+SELECT T1.Air_date FROM series AS T1 JOIN episode AS T2 ON T1.Series_ID  =  T2.Series_ID WHERE T2.Episode  =  "A Love of a Lifetime"
+SELECT count(*) FROM dog WHERE age  <  (SELECT avg(age) FROM dog)
+SELECT T1.continent ,  COUNT(*) FROM city AS T1 JOIN country AS T2 ON T1.continent  =  T2.continent GROUP BY T1.continent ORDER BY COUNT(*) DESC LIMIT 1
+SELECT T2.name FROM shop AS T1 JOIN shop_stock AS T2 ON T1.shop_id  =  T2.shop_id WHERE T2.number_products  >  (SELECT avg(T3.number_products) FROM shop AS T1 JOIN shop_stock AS T2 ON T1.shop_id  =  T2.shop_id JOIN staff AS T3 ON T1.shop_id  =  T3.shop_id WHERE T3.is_full_time  =  'yes' GROUP BY T1.shop_id)
+SELECT count(*) FROM Flight WHERE airline  =  'United Airlines' AND source_airport  =  'AHD Airport'
+SELECT DISTINCT t1.template_type_code ,  count(*) FROM template_details AS t1 JOIN document_contents AS t2 ON t1.template_id  =  t2.template_id GROUP BY t1.template_type_code
+SELECT Title FROM cartoon WHERE Directed_by  =  "Ben Jones" OR Directed_by  =  "Brandon Vietti"�
+SELECT T1.paragraph_id ,  T1.paragraph_text FROM Documents AS T1 JOIN Paragraphs AS T2 ON T1.document_id  =  T2.paragraph_id WHERE T2.document_name  =  'Welcome to NY'
+SELECT DISTINCT T1.document_name FROM Documents AS T1 JOIN Templates AS T2 ON T1.template_id  =  T2.template_id WHERE T2.documents_used  =  'Y'
+SELECT DISTINCT T1.contestant_name FROM contestant AS T1 JOIN vote AS T2 ON T1.contestant_number  =  T2.contestant_number WHERE T1.contestant_name != "Jessie Alloway"
+SELECT name FROM list AS T1 JOIN high_school AS T2 ON T1.hs_id  =  T2.hs_id WHERE T1.grade  >  5 INTERSECT SELECT name FROM list AS T1 JOIN high_school AS T2 ON T1.hs_id  =  T2.hs_id GROUP BY T1.hs_id HAVING count(*)  >=  2
+SELECT count(*) FROM FLIGHTS WHERE Destination_Airport_Code  =  'APG'
+SELECT DISTINCT T1.name FROM city AS T1 JOIN country AS T2 ON T1.district  =  T2.district WHERE T2.country_code  =  'Asia' AND T2.language  =  'Chinese'�, AND T1.city_code  =  T2.district_code
+SELECT T1.fname ,  T1.lname ,  sex FROM student AS T1 JOIN has_pet AS T2 ON T1.StuID  =  T2.StuID GROUP BY T1.StuID HAVING count(DISTINCT T2.PetID)  >  1
+SELECT flight_number FROM FLIGHTS WHERE Destination_Airport_Code  =  'ABZ' AND Source_Airport_Code  =  'Aberdeen', Airport_Name FROM AIRPORTS WHERE City  =  'Aberdeen', Country_Abbrev  =  'United_Kingdom'
+SELECT T1.first_name ,  avg(T2.loser_rank) FROM player AS T1 JOIN loser_rank AS T2 ON T1.player_id  =  T2.loser_id GROUP BY T1.first_name
+SELECT T1.name FROM students AS T1 JOIN has_friends AS T2 ON T1.StuID  =  T2.StuID WHERE T2.Liked_id  =  (SELECT T3.StuID FROM has_friends AS T3 JOIN students AS T4 ON T3.Friend_id  =  T4.StuID GROUP BY T3.Friend_id ORDER BY count(*) DESC LIMIT 1) INTERSECT SELECT T1.name FROM students AS T1 JOIN has_friends AS T2 ON T1.
+SELECT count(*) FROM has_friend AS T1 JOIN student AS T2 ON T1.StuID  =  T2.StuID WHERE T1.name  =  "Kyle" AND T1.grade  =  "Freshman"
+SELECT Model FROM model GROUP BY Model HAVING COUNT(*)  >  (SELECT count(*) FROM model WHERE Model  =  MATCH (SELECT DISTINCT Model FROM model) EXCEPT SELECT Model FROM model WHERE Version  =  MATCH (SELECT DISTINCT Version FROM model) EXCEPT SELECT Model FROM model WHERE Version  =  MATCH (SELECT DISTINCT Version FROM model) EXCEPT SELECT Model FROM model WHERE Version  =  MATCH (SELECT DISTINCT Version FROM model) EXCEPT SELECT Model FROM model WHERE Version  =  MATCH (SELECT DISTINCT Version FROM model
+SELECT count(DISTINCT loser_name) FROM loser
+SELECT T1.owner_id ,  T1.first_name ,  T1.last_name FROM dog AS T1 JOIN dog_treatment AS T2 ON T1.dog_id  =  T2.dog_id GROUP BY T1.dog_id ORDER BY count(*) DESC LIMIT 1
+SELECT pet_age FROM pet WHERE pet_type  =  "Dog" AND pet_age  =  (SELECT min(pet_age) FROM pet WHERE pet_type  =  "Dog")C, pet_id FROM pet WHERE pet_type  =  "Dog" AND pet_age  =  (SELECT min(pet_age) FROM pet WHERE pet_type  =  "Dog" AND pet_id  =  (SELECT pet_id FROM pet WHERE pet_type  =  "Dog" AND pet_age  =  (SELECT min(pet
+SELECT t1.first_name ,  t1.middle_name ,  t1.last_name ,  t1.enrolments FROM student AS t1 JOIN course_enrolment AS t2 ON t1.student_id  =  t2.student_id GROUP BY t1.student_id ORDER BY t2.enrolments DESC LIMIT 1;
+SELECT count(*) FROM hometown AS T1 JOIN teacher AS T2 ON T1.hometown  =  T2.hometown GROUP BY T1.hometown
+SELECT T1.document_name FROM Documents AS T1 JOIN Document_contents AS T2 ON T1.document_id  =  T2.document_id JOIN TEMPLATES AS T3 ON T2.template_id  =  T3.template_id WHERE T3.template_type_code  =  "BK"�
+SELECT T1.Name FROM poker_player AS T1 JOIN people AS T2 ON T1.People_ID  =  T2.People_ID WHERE T1.Earnings  >  300000
+SELECT T1.loser_name FROM loser AS T1 JOIN loser_match AS T2 ON T1.loser_id  =  T2.winner_id WHERE T2.match_num  =  2013 AND T2.match_year  =  2016 INTERSECT SELECT T1.loser_name FROM loser AS T1 JOIN loser_match AS T2 ON T1.loser_id  =  T2.winner_id WHERE T2.match_num  =  2016 AND T
+SELECT DISTINCT T1.Name FROM singer AS T1 JOIN song AS T2 ON T1.Singer_ID  =  T2.Singer_ID WHERE T2.Sales  >  300000�, T1.Name NOT IN (SELECT DISTINCT T1.Name FROM singer AS T1 JOIN song AS T2 ON T1.Singer_ID  =  T2.Singer_ID WHERE T2.Sales  >  300000)
+SELECT Record_Company FROM orchestra GROUP BY Record_Company ORDER BY COUNT(*) DESC LIMIT 1
+SELECT avg(weight) ,  YEAR FROM car GROUP BY YEARF, CONTIDERID
+SELECT country_code FROM people WHERE language != 'english'
+SELECT maker ,  count(*) ,  T1.id ,  T2.full_name FROM models AS T1 JOIN makes AS T2 ON T1.maker  =  T2.id GROUP BY T1.id ,  T2.full_name
+SELECT Tours.Tourney_Date ,  COUNT(*) FROM MATCHES AS T_1 JOIN TOURNEYS AS T_2 ON T_1.Tourney_ID  =  T_2.Tourney_ID GROUP BY T_2.Tourney_Date
+SELECT count(*) FROM car WHERE Horsepower  >  150�, Maker  =  "YAMAHA"
+SELECT T2.other_address_details FROM student_enrolments AS T1 JOIN addresses AS T2 ON T1.current_address_id  =  T2.address_id ORDER BY T1.student_first_name DESC, T1.student_last_name DESC
+SELECT count(DISTRICT) FROM country WHERE government_form  =  "Republic"�, country_code IN (SELECT country_code FROM country_continent_relationship)
+SELECT T1.semester_name ,  T1.semester_id FROM course_registrations AS T1 JOIN students AS T2 ON T1.student_enrolment_id  =  T2.student_enrolment_id GROUP BY T1.semester_name ORDER BY count(*) ,  count(*) DESC LIMIT 1
+SELECT T1.document_id FROM Document_contents AS T1 JOIN Paragraphs AS T2 ON T1.paragraph_id  =  T2.paragraph_id WHERE T2.text  =  'Brazil' INTERSECT SELECT T1.document_id FROM Document_contents AS T1 JOIN Paragraphs AS T2 ON T1.paragraph_id  =  T2.paragraph_id WHERE T2.text  =  'Ireland'
+SELECT series_name FROM series WHERE episode  =  "A Love of a Lifetime"
+SELECT name FROM students WHERE StuID NOT IN (SELECT StuID FROM student_friends)
+SELECT name FROM student WHERE likes  =  (SELECT likes FROM student_friend ORDER BY likes DESC LIMIT 1)
+SELECT name FROM employee EXCEPT SELECT T1.name FROM employee AS T1 JOIN award AS T2 ON T1.employee_id  =  T2.employee_id WHERE T2.YEAR_AWARDED  =  '2016'
+SELECT production_code ,  channel FROM cartoons ORDER BY episode DESC LIMIT 1
+SELECT avg(Horsepower) FROM model WHERE YEAR  <  1980 GROUP BY YEAR ORDER BY avg(Horsepower) DESC UNION SELECT avg(Horsepower) FROM model WHERE YEAR  >  1980 GROUP BY YEAR ORDER BY avg(Horsepower) ASC
+SELECT Record_Company FROM performance GROUP BY Record_Company WHERE YEAR_OF_FOUNDED  <  2003 INTERSECT SELECT Record_Company FROM performance GROUP BY Record_Company WHERE YEAR_OF_FOUNDED  >  2003
+SELECT T2.Name ,  count(*) FROM concert_record AS T1 JOIN singer AS T2 ON T1.Singer_ID  =  T2.Singer_ID GROUP BY T1.Singer_ID
+SELECT t1.name ,  t1.location ,  t1.district FROM shop AS t1 JOIN shop_stock AS t2 ON t1.shop_id  =  t2.shop_id GROUP BY t1.shop_id ORDER BY number_products DESC
+SELECT sum(Tournament_rank_points) FROM loser
+SELECT T2.student_id FROM friends AS T1 JOIN high_school AS T2 ON T1.friend_id  =  T2.student_id WHERE T1.student_id IN (SELECT T1.student_id FROM friends AS T1 JOIN high_school AS T2 ON T1.friend_id  =  T2.student_id) INTERSECT SELECT T1.student_id FROM friends AS T1 JOIN high_school AS T2 ON T1.friend_id  =  T2.student_id GROUP BY T1.student_id H
+SELECT series_name FROM channel WHERE title  =  "The Rise of the Blue Beetle"C
+SELECT language FROM country WHERE government_form  =  "republic" EXCEPT SELECT T2.language FROM country AS T1 JOIN city AS T2 ON T1.district  =  T2.district WHERE T1.country_code  =  country_code AND T1.government_form  =  "republic"
+SELECT t1.document_id ,  t1.document_name ,  t1.document_description FROM Documents AS t1 JOIN Documents_contents AS t2 ON t1.document_id  =  t2.document_id JOIN Templates AS t3 ON t2.template_id  =  t3.template_id WHERE t1.document_name  =  "Robbin CV"
+SELECT T2.owner_id ,  T2.last_name FROM dog_treatment AS T1 JOIN dog AS T2 ON T1.dog_id  =  T2.dog_id GROUP BY T1.dog_id ORDER BY sum(T1.cost_of_treatment) DESC LIMIT 1;
+SELECT DISTINCT T2.template_type_code FROM TEMPLATES AS T1 JOIN PARAGRAPHS AS T2 ON T1.template_id  =  T2.template_id WHERE T1.template_type_code  =  "Marketing"
+SELECT count(*) FROM cylinders WHERE cylinders  >  4
+SELECT T1.id FROM friends AS T1 JOIN students AS T2 ON T1.StuID  =  T2.StuID WHERE T2.name  =  "Kyle"
+SELECT T1.name FROM list AS T1 JOIN course AS T2 ON T1.course_id  =  T2.course_id EXCEPT SELECT T1.name FROM list AS T1 JOIN course AS T2 ON T1.course_id  =  T2.course_id
+SELECT T1.first_name ,  T1.middle_name ,  T1.last_name FROM student AS T1 JOIN course_enrolment AS T2 ON T1.student_id  =  T2.student_enrolment_id WHERE T2.course_id  =  (SELECT course_id FROM course_enrolment WHERE course_name  =  (SELECT course_name FROM course WHERE degree_program_id  =  (SELECT degree_program_id FROM degree_programs ORDER BY course_name DESC LIMIT 1) ) ) ORDER BY T
+SELECT T1.Name FROM conductor AS T1 JOIN orchestra AS T2 ON T1.Conductor_ID  =  T2.Conductor_ID GROUP BY T1.Conductor_ID ORDER BY count(*) DESC LIMIT 1
+SELECT T2.treatment_type_description FROM dog_treatment_types AS T1 JOIN dog_treatments AS T2 ON T1.treatment_type_code  =  T2.treatment_type_code WHERE T1.treatment_type_code  =  (SELECT T3.treatment_type_code FROM dog_treatment_types AS T1 JOIN dog_treatments AS T2 ON T1.treatment_type_code  =  T2.treatment_type_code JOIN dog AS
+SELECT Model FROM cylinders WHERE Cylinders  =  4 AND Horsepower  =  (SELECT max(Horsepower) FROM model WHERE Make  =  "Car" AND Cylinders  =  4)�, Maker FROM cylinders WHERE Cylinders  =  4 AND Horsepower  =  (SELECT max(Horsepower) FROM model WHERE Make  =  "Car" AND Cylinders  =  4)
+SELECT country_abbrev FROM airlines WHERE airline  =  "JetBlue Airways"�
+SELECT T2.template_type_code FROM template_details AS T1 JOIN template_type AS T2 ON T1.template_type_code  =  T2.template_type_code GROUP BY T1.template_type_code ORDER BY count(*) DESC LIMIT 1
+SELECT DISTINCT T1.model FROM car AS T1 JOIN manufacturer AS T2 ON T1.maker  =  T2.id JOIN country AS T3 ON T2.country_id  =  T3.id WHERE T3.name  =  "General Motors" OR T3.weight  >  3500
+SELECT model FROM car WHERE horsepower  =  (SELECT min(HORSEPOWER) FROM car)
+SELECT count(*) ,  T1.stuid FROM pet AS T1 JOIN has_pet AS T2 ON T1.petid  =  T2.petid JOIN student AS T3 ON T3.stuid  =  T1.stuid WHERE T2.has_pet  =  1 AND T3.stuid  =  (SELECT T1.stuid FROM pet AS T1 JOIN has_pet AS T2 ON T1.petid  =  T2.petid WHERE T2.has_pet  =  1 GROUP BY T1.stuid ORDER BY
+SELECT t3.document_id FROM Document_contents AS t1 JOIN Paragraphs AS t2 ON t1.paragraph_id  =  t2.paragraph_id JOIN Documents AS t3 ON t1.document_id  =  t3.document_id WHERE t2.text  =  "Brazil" INTERSECT SELECT t3.document_id FROM Document_contents AS t1 JOIN Paragraphs AS t2 ON t1.paragraph_id  =  t2.paragraph_id JOIN Documents AS t3 ON t1.document_id  =
+SELECT t1.template_id FROM templates AS t1 JOIN document_templates AS t2 ON t1.template_id  =  t2.template_id WHERE t2.template_type_code  =  "PP" OR t2.template_type_code  =  "PPT"F
+SELECT T2.contestant_name FROM contestants AS T1 JOIN vote AS T2 ON T1.contestant_number  =  T2.contestant_number WHERE T1.contestant_name LIKE '%Al%'
+SELECT count(DISTINCT Nationality) FROM POKER_COMPETITOR
+SELECT count(DISTINCT employee_id) FROM employee/]{'employee id ,  name ,  age ,  city ,  sex ,  department id ,  name ,  location ,  district ,  number products ,  manager name ,  department id ,  start date ,  end date ,  senior employee id ,  end date ,  senior employee id ,  is full time ,  hour worked per day ,  hour worked per week ,  date of employment ,  resignation date ,  reason for leaving ,  senior employee id ,  working hours per day ,  working hours per week ,  employ
+SELECT T1.Title FROM cartoon AS T1 JOIN directed_by AS T2 ON T1.ID  =  T2.ID_cartoon WHERE T2.Directed_by  =  "Ben_Jones"
+SELECT T2.StuID FROM friends AS T1 JOIN student AS T2 ON T1.StuID  =  T2.StudentID WHERE T1.FriendID IN (SELECT T3.FriendID FROM friends AS T1 JOIN student AS T2 ON T1.StuID  =  T2.StudentID JOIN liked AS T3 ON T3.LikedID  =  T1.FriendID) INTERSECT SELECT T1.StuID FROM friends AS T1 JOIN student AS T2 ON T1.StuID  =  T
+SELECT name FROM grade WHERE grade  =  "Freshman" INTERSECT SELECT name FROM students WHERE studentsid  =  T1.Student_IDF, SELECT name FROM grade WHERE grade  =  "Sophomore" INTERSECT SELECT name FROM students WHERE studentsid  =  T1.Student_ID, SELECT name FROM grade WHERE grade  =  "Junior" INTERSECT SELECT name FROM students WHERE studentsid  =  T1.Student_ID, SELECT name FROM grade WHERE grade  =  "Senior" INTERSECT
+SELECT Nationality FROM people GROUP BY Nationality HAVING COUNT(*)  >=  2
+SELECT count(*) FROM maker WHERE country_id  =  "FRANCE"
+SELECT max(Horsepower) ,  Maker FROM car WHERE Cylinders  =  3
+SELECT flight_number FROM FLIGHTS WHERE source_airport  =  "APG"
+SELECT DISTINCT T1.Country FROM concert AS T1 JOIN singer AS T2 ON T1.Concert_ID  =  T2.Concert_ID WHERE T2.Age  >  40 INTERSECT SELECT DISTINCT T1.Country FROM concert AS T1 JOIN singer AS T2 ON T1.Concert_ID  =  T2.Concert_ID WHERE T2.Age  <  30
+SELECT winner_name ,  winner_rank_points FROM loser GROUP BY winner_rank_points ORDER BY sum(winner_rank_points) DESC LIMIT 1
+SELECT grade ,  count(*) FROM list WHERE name  =  "high school" GROUP BY grade
+SELECT T1.owner_id ,  T2.zip_code FROM dog_treatment AS T1 JOIN dog AS T2 ON T1.dog_id  =  T2.dog_id GROUP BY T1.dog_id ORDER BY sum(T1.treatment_amount) DESC LIMIT 1
+SELECT count(DISTINCT flight_number) FROM FLIGHTF, AIRCRAFT
+SELECT T1.airport_code ,  T1.airport_name FROM airport AS T1 JOIN flight AS T2 ON T1.airport_code  =  T2.source_airport WHERE T1.city  =  'Anthony', T2.destination_airport  =  'T1.airport_name'
+SELECT T1.Caused_by_ship_id FROM ship AS T1 JOIN killed_in_battle AS T2 ON T1.id  =  T2.Caused_by_ship_id WHERE T1.Tonnage  =  "t"
+SELECT tourney_year FROM loser GROUP BY tourney_year ORDER BY count(*) DESC LIMIT 1�, 1
+SELECT DISTINCT T1.name FROM singer AS T1 JOIN song AS T2 ON T1.singer_id  =  T2.singer_id WHERE T2.sales  >  300000
+SELECT T2.name FROM list AS T1 JOIN course AS T2 ON T1.course_id  =  T2.course_id GROUP BY T1.course_id HAVING count(*)  >=  2
+SELECT count(*) FROM car WHERE Horsepower  >  150�, MpG  >  150
+SELECT LOCATION ,  number_shop FROM shop GROUP BY LOCATION ORDER BY number_shop DESC LIMIT 1
+SELECT T1.name FROM singer AS T1 JOIN song AS T2 ON T1.singer_id  =  T2.singer_id GROUP BY T1.singer_id HAVING count(*)  >  1
+SELECT T1.last_name FROM students AS T1 JOIN addresses AS T2 ON T1.current_address_id  =  T2.address_id WHERE T2.state_province_county  =  "North Carolina" AND T1.last_name NOT IN (SELECT T1.last_name FROM students AS T1 JOIN enrolments AS T2 ON T1.student_id  =  T2.student_id WHERE T2.course_id NOT IN (SELECT T3.course_id FROM student_course_enrolments AS T3 JOIN
+SELECT T1.name FROM museum AS T1 JOIN customer AS T2 ON T1.museum_id  =  T2.museum_id WHERE T1.open_year  <  2009 INTERSECT SELECT T1.name FROM museum AS T1 JOIN customer AS T2 ON T1.museum_id  =  T2.museum_id WHERE T1.open_year  >  2011
+SELECT count(*) FROM city WHERE country_code  =  "Aruba"�, LANGUAGE IN (SELECT Language FROM language_district WHERE district  =  "Aruba" GROUP BY Language ORDER BY count(*) DESC LIMIT 1)
+SELECT T1.Episode FROM episode AS T1 JOIN ratings AS T2 ON T1.Episode  =  T2.Episode ORDER BY T2.Rating
+SELECT T1.airline_name FROM airlines AS T1 JOIN flight AS T2 ON T1.airline_id  =  T2.airline WHERE T2.flight_number  =  (SELECT max(flight_number) FROM airlines AS T1 JOIN flight AS T2 ON T1.airline_id  =  T2.airline WHERE T1.airline_name  =  "All Nippon Airways" AND T2.source_airport  =  "Tokyo" AND T2.destination_airport  =
+SELECT count(DISTINCT country_id) FROM country
+SELECT count(*) FROM COURSES;
+SELECT grade FROM students GROUP BY grade ORDER BY count(*) DESC LIMIT 1�enaid, name FROM friends AS T1 JOIN students AS T2 ON T1.enanid  =  T2.enanid WHERE T1.friendid  =  T2.enanid
+SELECT count(*) FROM country WHERE government_form_code  =  "Republican"F, country_code IN (SELECT country_code FROM country WHERE government_form_code  =  "Republican")
+SELECT DISTINCT T1.fname FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid WHERE T2.pettype !=  'cat' AND T2.pettype !=  'dog'

file_copies_from_project_root/predict_base.txt

@@ -0,0 +1,10 @@
+['player id', 'first name', 'last name', 'hand', 'birth date', 'country code', 'best of', 'draw size', 'loser age', 'loser entry', 'loser hand', 'loser ht', 'loser id', 'loser ioc', 'loser name', 'loser rank', 'loser rank points', 'loser seed', 'match num', 'minutes', 'round', 'score', 'surface', 'tourney date', 'tourney id', 'tourney level', 'tourney name', '
+2
+['id', 'series name', 'country', 'language', 'content', 'pixel aspect ratio par', 'hight definition tv', 'pay per view ppv', 'package option', 'id', 'episode', 'air date', 'rating', 'share', '18 49 rating share', 'viewers m', 'weekly rank', 'channel', 'id', 'title', 'directed by', 'written by', 'original air date', 'production code', 'channel']
+['stadium id', 'location', 'name', 'capacity', 'highest', 'lowest', 'average', 'singer id', 'name', 'country', 'song name', 'song release year', 'age', 'is male', 'concert id', 'concert name', 'theme', 'stadium id', 'year', 'concert id', 'singer id']
+['id', 'name', 'country code', 'district', 'population', 'name', 'seq', 'code', 'name', 'continent', 'region', 'surface area', 'indepdent year', 'population', 'life expectancy', 'gnp', 'gnp old', 'local name', 'government form', 'head of state', 'capital', 'code2', 'countrycode', 'language', 'is official', 'percentage']
+There are 10 high schoolers in grade 9 or 10.
+2 countries speak both English and Dutch: Suriname and Netherlands.
+['course id', 'staring date', 'course', 'teacher id', 'name', 'age', 'hometown', 'course id', 'teacher id', 'grade']
+['address id', 'line 1', 'line 2', 'line 3', 'city', 'zip postcode', 'state province county', 'country', 'other address details', 'course id', 'course name', 'course description', 'other details', 'department id', 'department name', 'department description', 'other details', 'degree program id', 'department id', 'degree summary name', 'degree summary description', 'other details', 'section id', 'course id', 'section name', 'section description', 'other details', 'semester id', '
+['conductor id', 'name', 'age', 'nationality', 'year of work', 'orchestra id', 'orchestra', 'conductor id', 'record company', 'year of founded', 'major record format', 'performance id', 'orchestra id', 'type', 'date', 'official ratings (millions)', 'weekly rank', 'share', 'show id', 'performance id', 'if first show', 'result', 'attendance']

file_copies_from_project_root/spider_get_table.py

@@ -0,0 +1,17 @@
+import json
+import sys
+
+def get_spider_table():
+    with open('tables.json') as f:
+        tables = json.load(f)
+    # tables_dictionary = {table['db_id']: table for table in tables}
+    tables_dictionary = {table['db_id']: [x[1] for x in table["column_names"][1:]] for table in tables}
+    return tables_dictionary
+
+tables_dict = get_spider_table()
+
+if __name__ == "__main__":
+    dbid = sys.argv[1]
+    print("Printing DB ID: ", dbid)
+    print("TABLE: " + str(dbid) + " \nCOLUMNS: " + str(tables_dict[dbid]) + "\n")
+    # print(tables_dict[sys.argv[1]])

file_copies_from_project_root/spider_raw_get_table.py

@@ -0,0 +1,14 @@
+import json
+import sys
+
+def get_raw_table(dbid):
+    with open('tables.json') as f:
+        tables = json.load(f)
+    for table in tables:
+        if table['db_id'] == dbid:
+            return table
+
+if __name__ == "__main__":
+    dbid = sys.argv[1]
+    print("Printing Raw Table for ID: ", dbid)
+    print(get_raw_table(dbid))

file_copies_from_project_root/spider_table_parse.py

@@ -0,0 +1,46 @@
+from datasets import load_dataset
+import json
+import pandas as pd
+
+data_path = "spider"
+tables_dict = {}
+
+def get_spider_table():
+    with open('tables.json') as f:
+        tables = json.load(f)
+    tables_dictionary = {table['db_id']: [x[1] for x in table["column_names"][1:]] for table in tables}
+    return tables_dictionary
+
+tables_dict = get_spider_table()
+# print(tables_dict["perpetrator"])
+# print(tables_dict["perpetrator"]["column_names"])
+
+# newlist = [[x[1] for x in row["column_names"]] for row in tables_dict]
+# print(newlist)
+
+data = load_dataset(data_path)
+data_train = data["train"]
+data_val = data["validation"]
+
+data_train.to_csv("dataset_train_augmented.csv")
+df_train = pd.read_csv("dataset_train_augmented.csv")
+new_col = []
+new_col = [tables_dict[dbid] for dbid in data_train['db_id']]
+# new_col = ["TABLE: " + str(dbid) + " \nCOLUMNS: " + str(tables_dict[dbid]) + "\n" for dbid in data_train['db_id']]
+print(new_col[0])
+print(new_col[-1])
+df_train["db_context"] = new_col
+df_train.to_csv("dataset_train_augmented.csv")
+data_train = load_dataset("csv", data_files="dataset_train_augmented.csv")
+
+data_val.to_csv("dataset_val_augmented.csv")
+df_val = pd.read_csv("dataset_val_augmented.csv")
+new_col = []
+new_col = [tables_dict[dbid] for dbid in data_val['db_id']]
+# new_col = ["TABLE: " + str(dbid) + " \nCOLUMNS: " + str(tables_dict[dbid]) + "\n" for dbid in data_val['db_id']]
+print(new_col[0])
+print(new_col[-1])
+df_val["db_context"] = new_col
+df_val.to_csv("dataset_val_augmented.csv")
+data_val = load_dataset("csv", data_files="dataset_val_augmented.csv")
+