Initial commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,3 @@
+# CodeParrot
+
+CodeParrot (large) is a 1.5B parameter GPT-2 model trained on the [CodeParrot Python code dataset](https://huggingface.co/datasets/transformersbook/codeparrot). The model is trained in Chapter 10: Training Transformers from Scratch in the [NLP with Transformers book](https://learning.oreilly.com/library/view/natural-language-processing/9781098103231/). You can find the full code in the accompanying [Github repository](https://github.com/nlp-with-transformers/notebooks/blob/main/10_transformers-from-scratch.ipynb).

codeparrot_training.py

@@ -0,0 +1,197 @@
+from transformers import GPT2LMHeadModel, AutoTokenizer
+from transformers import AdamW, get_scheduler, set_seed
+from datasets import load_dataset
+from accelerate import Accelerator
+import datasets, transformers
+from huggingface_hub import Repository
+
+from torch.utils.data import IterableDataset
+from torch.utils.data.dataloader import DataLoader
+from torch.utils.tensorboard import SummaryWriter
+from argparse import Namespace
+import torch
+import logging
+import wandb
+
+class ConstantLengthDataset(IterableDataset):
+    
+    def __init__(self, tokenizer, dataset, seq_length=1024,
+                 num_of_sequences=1024, chars_per_token=3.6):
+        self.tokenizer = tokenizer
+        self.concat_token_id = tokenizer.bos_token_id
+        self.dataset = dataset
+        self.seq_length = seq_length
+        self.input_characters = seq_length * chars_per_token * num_of_sequences
+    
+    def __iter__(self):
+        iterator = iter(self.dataset)
+        more_examples = True
+        while more_examples:
+            buffer, buffer_len = [], 0
+            while True:
+                if buffer_len >= self.input_characters:
+                    break
+                try:
+                    buffer.append(next(iterator)['content'])
+                    buffer_len += len(buffer[-1])
+                except StopIteration:
+                    more_examples = False
+                    break
+            tokenized_inputs = tokenizer(buffer, truncation=False)['input_ids']
+            all_token_ids = []
+            for tokenized_input in tokenized_inputs:
+                all_token_ids.extend(tokenized_input + [self.concat_token_id])
+            for i in range(0, len(all_token_ids), self.seq_length):
+                input_ids = all_token_ids[i : i + self.seq_length]
+                if len(input_ids) == self.seq_length:
+                    yield torch.tensor(input_ids)
+
+def setup_logging(project_name):
+    logger = logging.getLogger(__name__)
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO, handlers=[
+        logging.FileHandler(f"log/debug_{accelerator.process_index}.log"),
+        logging.StreamHandler()])
+    if accelerator.is_main_process: # we only want to setup logging once
+        wandb.init(project=project_name, config=args)
+        run_name = wandb.run.name
+        tb_writer = SummaryWriter()
+        tb_writer.add_hparams(vars(args), {'0': 0})
+        logger.setLevel(logging.INFO)
+        datasets.utils.logging.set_verbosity_info()
+        transformers.utils.logging.set_verbosity_info()
+    else:
+        tb_writer = None
+        run_name = ''
+        logger.setLevel(logging.ERROR)
+        datasets.utils.logging.set_verbosity_error()
+        transformers.utils.logging.set_verbosity_error()
+    return logger, tb_writer, run_name
+
+def create_dataloaders(dataset_name, args):
+    ds_kwargs = {"streaming":True, "chunksize":40<<20, "error_bad_chunk":False}
+    train_data = load_dataset(dataset_name+'-train', split='train', **ds_kwargs)
+    train_data = train_data.shuffle(buffer_size=args.shuffle_buffer,
+                                    seed=args.seed)
+    valid_data = load_dataset(dataset_name+'-valid', split="train", **ds_kwargs)
+    train_dataset = ConstantLengthDataset(tokenizer, train_data,
+                                          seq_length=args.seq_length)
+    valid_dataset = ConstantLengthDataset(tokenizer, valid_data,
+                                          seq_length=args.seq_length)
+    train_dataloader=DataLoader(train_dataset, batch_size=args.train_batch_size)
+    eval_dataloader=DataLoader(valid_dataset, batch_size=args.valid_batch_size)
+    return train_dataloader, eval_dataloader
+
+def get_grouped_params(model, args, no_decay=["bias", "LayerNorm.weight"]):
+    params_with_wd, params_without_wd = [], []
+    for n, p in model.named_parameters():
+        if any(nd in n for nd in no_decay): params_without_wd.append(p)
+        else: params_with_wd.append(p)
+    return [{'params': params_with_wd, 'weight_decay': args.weight_decay},
+            {'params': params_without_wd, 'weight_decay': 0.0}]
+
+def log_metrics(step, metrics):
+    logger.info(f"Step {step}: {metrics}")
+    if accelerator.is_main_process:
+        wandb.log(metrics)
+        [tb_writer.add_scalar(k, v, step) for k, v in metrics.items()]
+
+def evaluate(args):
+    model.eval()
+    losses = []
+    for step, batch in enumerate(eval_dataloader):
+        with torch.no_grad():
+            outputs = model(batch, labels=batch)
+        loss = outputs.loss.repeat(args.valid_batch_size)
+        losses.append(accelerator.gather(loss))
+        if args.max_eval_steps > 0 and step >= args.max_eval_steps: break
+    loss = torch.mean(torch.cat(losses))
+    try: perplexity = torch.exp(loss)
+    except OverflowError: perplexity = float("inf")
+    return loss.item(), perplexity.item()
+
+# Accelerator
+accelerator = Accelerator(dispatch_batches=True)
+acc_state = {str(k): str(v) for k, v in accelerator.state.__dict__.items()}
+# Hyperparameters
+project_name = 'transformersbook/codeparrot'
+dataset_name = '../llama-from-scratch/codeparrot'
+config = {"train_batch_size": 2,
+          "valid_batch_size": 2,
+          "weight_decay": 0.1,
+          "shuffle_buffer": 1_000,
+          "learning_rate": 2e-4,
+          "lr_scheduler_type": "cosine",
+          "num_warmup_steps": 750,
+          "gradient_accumulation_steps": 16,
+          "max_train_steps": 50_000,
+          "max_eval_steps": -1,
+          "seq_length": 1024,
+          "seed": 1,
+          "save_checkpoint_steps": 50_000}
+args = Namespace(**config, **acc_state)
+samples_per_step = accelerator.state.num_processes * args.train_batch_size
+set_seed(args.seed)
+
+# Logging
+logger, tb_writer, run_name = setup_logging(project_name.split("/")[1])
+logger.info(accelerator.state)
+
+# Load model and tokenizer
+if accelerator.is_main_process:
+    hf_repo = Repository("./", clone_from=project_name, revision=run_name)
+model = GPT2LMHeadModel.from_pretrained("./", gradient_checkpointing=True)
+tokenizer = AutoTokenizer.from_pretrained("./")
+
+# Load dataset and dataloader
+train_dataloader, eval_dataloader = create_dataloaders(dataset_name, args)
+
+# Prepare the optimizer and learning rate scheduler
+optimizer = AdamW(get_grouped_params(model, args), lr=args.learning_rate)
+lr_scheduler = get_scheduler(name=args.lr_scheduler_type, optimizer=optimizer,
+                             num_warmup_steps=args.num_warmup_steps,
+                             num_training_steps=args.max_train_steps,)
+def get_lr(): return optimizer.param_groups[0]['lr']
+
+# Prepare everything with our `accelerator`.
+model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
+    model, optimizer, train_dataloader, eval_dataloader)
+
+# Train model
+model.train()
+completed_steps = 0
+for step, batch in enumerate(train_dataloader, start=1):
+    loss = model(batch, labels=batch, use_cache=False).loss
+    log_metrics(step, {'lr': get_lr(), 'samples': step*samples_per_step,
+                       'steps': completed_steps, 'loss/train': loss.item()})
+    loss = loss / args.gradient_accumulation_steps
+    accelerator.backward(loss)
+    if step % args.gradient_accumulation_steps == 0:
+        accelerator.clip_grad_norm_(model.parameters(), 1.0)
+        optimizer.step()
+        lr_scheduler.step()
+        optimizer.zero_grad()
+        completed_steps += 1
+    if step % args.save_checkpoint_steps == 0:
+        logger.info('Evaluating and saving model checkpoint')
+        eval_loss, perplexity = evaluate(args)
+        log_metrics(step, {'loss/eval': eval_loss, 'perplexity': perplexity})
+        accelerator.wait_for_everyone()
+        unwrapped_model = accelerator.unwrap_model(model)
+        if accelerator.is_main_process:
+            unwrapped_model.save_pretrained("./")
+            hf_repo.push_to_hub(commit_message=f'step {step}')
+        model.train()
+    if completed_steps >= args.max_train_steps:
+        break
+
+# Evaluate and save the last checkpoint
+logger.info('Evaluating and saving model after training')
+eval_loss, perplexity = evaluate(args)
+log_metrics(step, {'loss/eval': eval_loss, 'perplexity': perplexity})
+accelerator.wait_for_everyone()
+unwrapped_model = accelerator.unwrap_model(model)
+if accelerator.is_main_process:
+    unwrapped_model.save_pretrained("./")
+    hf_repo.push_to_hub(commit_message=f'final model')

config.json

@@ -0,0 +1,41 @@
+{
+  "_name_or_path": "./",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 0,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 0,
+  "gradient_checkpointing": false,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 1600,
+  "n_head": 25,
+  "n_inner": null,
+  "n_layer": 48,
+  "n_positions": 1024,
+  "output_past": true,
+  "reorder_and_upcast_attn": true,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": true,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.11.0.dev0",
+  "use_cache": true,
+  "vocab_size": 32768
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d702c81fb5bf32236d7359a16fe6a59f68565cc551405b544bb009c18fc72af
+size 6169094681

requirements.txt

@@ -0,0 +1,7 @@
+# torch==1.9.0
+wandb
+tensorboard
+git+https://github.com/huggingface/huggingface_hub.git
+git+https://github.com/huggingface/transformers.git
+git+https://github.com/huggingface/datasets.git@json-dont-raise
+git+https://github.com/huggingface/accelerate.git

runs/Aug27_13-51-10_desktop2.xfact.net/1724734270.8848336/events.out.tfevents.1724734270.desktop2.xfact.net.283940.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bb5f71c3a698d57525a3cf9e5ec798d5207b5830cf6a239ea3201b6b36d7183
+size 1702

runs/Aug27_13-51-10_desktop2.xfact.net/events.out.tfevents.1724734270.desktop2.xfact.net.283940.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f51f7148276851dd5d67d3ff3bc0e45b13d6509185aa286ef9ee1c88bdc436ad
+size 88

runs/Sep20_14-28-12_leandro-16x-v100/1632148092.8874874/events.out.tfevents.1632148092.leandro-16x-v100.8660.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:222b05fb22ccb39b7d43f507f7c672d8c741e4281e65c71c12d98b19c1d3ff1f
+size 1373

runs/Sep20_14-28-12_leandro-16x-v100/events.out.tfevents.1632148092.leandro-16x-v100.8660.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3110a4850e2eba17c258d67eacb63ff2acca8af1b29362e28d3c328621a5391d
+size 147135683

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "thomwolf/codeparrot", "tokenizer_class": "GPT2Tokenizer"}

wandb/debug-internal.log

@@ -0,0 +1 @@
+run-20240827_135059-uy3qxte5/logs/debug-internal.log

wandb/debug.log

@@ -0,0 +1 @@
+run-20240827_135059-uy3qxte5/logs/debug.log

wandb/latest-run

@@ -0,0 +1 @@
+run-20240827_135059-uy3qxte5