import torch import torch.nn as nn from torch.utils.data import DataLoader, Dataset import tensorflow as tf import numpy as np def pad_sequences(sequences, max_seq_len=0): """Pad sequences to max length in sequence.""" max_seq_len = max(max_seq_len, max(len(sequence) for sequence in sequences)) padded_sequences = np.zeros((len(sequences), max_seq_len)) for i, sequence in enumerate(sequences): padded_sequences[i][:len(sequence)] = sequence return padded_sequences class CustomDataSetManager(Dataset): def __init__(self, text_seq, line_num, total_line): self.text_seq = text_seq self.line_num_one_hot = line_num self.total_line_one_hot = total_line def __len__(self): return len(self.text_seq) def __str__(self): return f"" def __getitem__(self, index): X = self.text_seq[index] line_num = self.line_num_one_hot[index] total_line = self.total_line_one_hot[index] return [X, len(X), line_num, total_line] # def collate_fn(self, batch): # """Processing on a batch""" # # Getting Input # batch = np.array(batch) # text_seq = batch[:,0] # seq_lens = batch[:, 1] # line_nums = batch[:, 2] # total_lines = batch[:, 3] # # padding inputs # pad_text_seq = pad_sequences(sequences=text_seq) # max_seq_len=max_length # # converting line nums into one-hot encoding # line_nums = tf.one_hot(line_nums, depth=20) # # converting total lines into one-hot encoding # total_lines = tf.one_hot(total_lines, depth=24) # # converting inputs to tensors # pad_text_seq = torch.LongTensor(pad_text_seq.astype(np.int32)) # seq_lens = torch.LongTensor(seq_lens.astype(np.int32)) # line_nums = torch.tensor(line_nums.numpy()) # total_lines = torch.tensor(total_lines.numpy()) # return pad_text_seq, seq_lens, line_nums, total_lines def collate_fn(self, batch): """Processing on a batch""" # Getting Input text_seq = [item[0] for item in batch] seq_lens = [item[1] for item in batch] line_nums = [item[2] for item in batch] total_lines = [item[3] for item in batch] # padding inputs pad_text_seq = pad_sequences(sequences=text_seq) # max_seq_len=max_length # converting line nums into one-hot encoding line_nums = torch.tensor(tf.one_hot(line_nums, depth=20).numpy()) # converting total lines into one-hot encoding total_lines = torch.tensor(tf.one_hot(total_lines, depth=24).numpy()) # converting inputs to tensors pad_text_seq = torch.tensor(pad_text_seq, dtype=torch.long) seq_lens = torch.tensor(seq_lens, dtype=torch.long) return pad_text_seq, seq_lens, line_nums, total_lines def create_dataloader(self, batch_size, shuffle=False, drop_last=False): dataloader = DataLoader(dataset=self, batch_size=batch_size, collate_fn=self.collate_fn, shuffle=shuffle, drop_last=drop_last, pin_memory=True) return dataloader