import csv import torch import torch.nn as nn from torch.utils.data import Dataset, DataLoader from torch.nn.utils.rnn import pad_sequence import math import progressbar import os Path=os.path.dirname(os.path.abspath(__file__))+"\\" device="cuda" def CreateBar(): global bar bar = progressbar.ProgressBar(maxval=100, \ widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()]) bar.start() tokens = list("azertyuiopqsdfghjklmwxcvbnäüöß—– ") tokensdict = {} for i in range(len(tokens)): tokensdict.update({tokens[i]: [0] * i + [0] * (len(tokens) - (i + 1))}) # Ouvrir le fichier CSV with open(Path+"top-german-verbs.csv", 'r', encoding="utf-8") as file: # Créer un objet lecteur CSV reader = [i for i in csv.reader(file)][1:] class CSVDataset(Dataset): def __init__(self, features, labels): self.features = features self.labels = labels def __len__(self): return len(self.features) def __getitem__(self, idx): sample = self.features[idx], self.labels[idx] return sample features = [] labels = [] padding=len(tokens) for i in reader: k = [] for j in i[2]: k += [tokens.index(j)] #k += [-1] * (25 - len(k)) features += [torch.Tensor(k)] k = [len(tokens)+1] for j in i[8]: k += [tokens.index(j)] #k += [-1] * (25 - len(k)) labels += [torch.Tensor(k)] MyDataset = CSVDataset(features=features, labels=labels) class TransformerModel(nn.Module): def __init__(self, vocab_size, emb_dim, nhead, num_encoder_layers, num_decoder_layers, dim_feedforward, dropout=0.1): super().__init__() self.custom_embedding = nn.Embedding(vocab_size, emb_dim, padding_idx=padding).to(device) self.pos_encoder = PositionalEncoding(emb_dim, dropout).to(device) encoder_layer = nn.TransformerEncoderLayer(emb_dim, nhead, dim_feedforward, dropout, batch_first=True).to(device) self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_encoder_layers) decoder_layer = nn.TransformerDecoderLayer(emb_dim, nhead, dim_feedforward, dropout, batch_first=True).to(device) self.transformer_decoder = nn.TransformerDecoder(decoder_layer, num_decoder_layers) self.output_layer = nn.Linear(emb_dim, vocab_size).to(device) def forward(self, src, tgt, src_mask=None, tgt_mask=None, memory_mask=None, src_key_padding_mask=None, tgt_key_padding_mask=None, memory_key_padding_mask=None): #print("Source:", src) #print("Target:", tgt) src_emb = self.custom_embedding(src.long()) src_emb = self.pos_encoder(src_emb) #print("Source Embedding:", src_emb.shape) tgt_emb = self.custom_embedding(tgt.long()) #print("Target Embedding:", tgt_emb.shape) tgt_emb = self.pos_encoder(tgt_emb) #print("Target Embedding:", tgt_emb.shape) encoder_output = self.transformer_encoder(src_emb, src_mask, src_key_padding_mask) decoder_output = self.transformer_decoder(tgt_emb, encoder_output, tgt_mask, memory_mask, tgt_key_padding_mask, memory_key_padding_mask) output = self.output_layer(decoder_output[:, -1, :]) #print("Output:",output.shape) return output class PositionalEncoding(nn.Module): def __init__(self, d_model, dropout=0.1, max_len=5000): super(PositionalEncoding, self).__init__() self.dropout = nn.Dropout(p=dropout) pe = torch.zeros(max_len, d_model) position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) pe[:, 0::2] = torch.sin(position * div_term) pe[:, 1::2] = torch.cos(position * div_term) pe = pe.unsqueeze(0) self.register_buffer('pe', pe) def forward(self, x): x = x + self.pe[:, :x.size(1), :] return self.dropout(x) def collate_fn(batch): inputs = [item[0].to(device) for item in batch] targets = [item[1].to(device) for item in batch] inputs = pad_sequence(inputs, batch_first=True, padding_value=padding) targets = pad_sequence(targets, batch_first=True, padding_value=padding) return inputs, targets train_loader = DataLoader(MyDataset, batch_size=32, shuffle=True, collate_fn=collate_fn) #Embedding Dimension on epoch 10 #32:10.49 #64:6.55 #128:6.44 #256:9.63 #Head Number on epoch 15 #32:6.44 #64:5.17 #16:5.9402 #Feed Forward Dimension on epoch 15+ (minimum) #128:5.17 #256:3.49 #512:3.44 #1024:3.23 #Num Encoder Layers on epochs 25 (minimum) #1:3.15 #2:4.01 #Num Decoder Layers on epochs 25 (minimum) #1:3.15 #2:2.14 #3:1.75 #4:1.60 #New model: #Dropout: 0 #Forward Dim: 1024 model = TransformerModel(vocab_size=len(tokens)+2, emb_dim=128, nhead=32, num_encoder_layers=1, num_decoder_layers=1, dim_feedforward=1024,dropout=0) loss_fn = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=0.001) try: model.load_state_dict(torch.load("data/PrateritumGPT.pth")) print("Sucessfully loaded model.") except: pass #print(model(torch.zeros((1,25)).to(device),torch.zeros((1,25)).to(device))) def Prompt(): global tokens global model inp=input("Give me a verb: ") src=[[]] tgt=[[len(tokens)+1]] for i in inp: src[0]+=[tokens.index(i)] str_="" for i in range(100): tgt_=torch.Tensor(tgt) out=model(torch.Tensor(src).to(device),tgt_.to(device)).tolist()[0] Best=0 Best_=tokens.index(" ") for k,f in enumerate(out): if f>Best: Best=f Best_=k if Best_==len(tokens): break str_+=tokens[Best_] tgt[0]+=[Best_] print(str_) if eval(input('Train? ')): epochs=eval(input("epochs ")) else: while True: Prompt() for epoch in range(epochs): total_loss = 0.0 CreateBar() for batch_idx, (inputs, targets) in enumerate(train_loader): #print("",inputs,targets) targets.to(device) inputs.to(device) for i in range(1, targets.shape[1]): optimizer.zero_grad() output = model(inputs, targets[:, :i]) # Shifted targets #print(output.shape) loss = loss_fn(output, targets[:, i].long()) # Reshape targets loss.backward() optimizer.step() total_loss += loss.item() mask = targets[:, i] != padding targets = targets[mask] inputs = inputs[mask] bar.update((batch_idx+1)/len(train_loader)*100) #print(f"Epoch {epoch + 1}/{epochs}, Batch {batch_idx}/{len(train_loader)}, Loss: {total_loss / (batch_idx + 1)}") bar.finish() print(f"Epoch {epoch + 1}/{epochs}, Loss: {total_loss / len(train_loader)}") torch.save(model.state_dict(), "data/PrateritumGPT.pth")