import numpy as np import torch import torch.nn as nn import torch.nn.functional as F import math import matplotlib.pyplot as plt import json import yaml class Fundamental_Music_Embedding(nn.Module): def __init__(self, d_model, base, if_trainable = False, if_translation_bias_trainable = True, device='cpu', type = "se",emb_nn=None,translation_bias_type = "nd"): super().__init__() self.d_model = d_model self.device = device self.base = base self.if_trainable = if_trainable #whether the se is trainable if translation_bias_type is not None: self.if_translation_bias = True self.if_translation_bias_trainable = if_translation_bias_trainable #default the 2d vector is trainable if translation_bias_type=="2d": translation_bias = torch.rand((1, 2), dtype = torch.float32) #Returns a tensor filled with random numbers from a uniform distribution on the interval [0, 1)[0,1) elif translation_bias_type=="nd": translation_bias = torch.rand((1, self.d_model), dtype = torch.float32) translation_bias = nn.Parameter(translation_bias, requires_grad=True) self.register_parameter("translation_bias", translation_bias) else: self.if_translation_bias = False i = torch.arange(d_model) angle_rates = 1 / torch.pow(self.base, (2 * (i//2)) / d_model) angle_rates = angle_rates[None, ... ].cuda() if self.if_trainable: angles = nn.Parameter(angle_rates, requires_grad=True) self.register_parameter("angles", angles) else: self.angles = angle_rates def __call__(self, inp): if inp.dim()==2: inp = inp[..., None] #pos (batch, num_pitch, 1) elif inp.dim()==1: inp = inp[None, ..., None] #pos (1, num_pitch, 1) angle_rads = inp*self.angles #(batch, num_pitch)*(1,dim) # apply sin to even indices in the array; 2i angle_rads[:, :, 0::2] = torch.sin(angle_rads.clone()[:, : , 0::2]) # apply cos to odd indices in the array; 2i+1 angle_rads[:, :, 1::2] = torch.cos(angle_rads.clone()[:, :, 1::2]) pos_encoding = angle_rads.to(torch.float32) if self.if_translation_bias: if self.translation_bias.size()[-1]!= self.d_model: translation_bias = self.translation_bias.repeat(1, 1,int(self.d_model/2)) else: translation_bias = self.translation_bias pos_encoding += translation_bias else: self.translation_bias = None return pos_encoding class Music_PositionalEncoding(nn.Module): def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 5000, if_index = True, if_global_timing = True, if_modulo_timing = True, device = 'cuda:0'): super().__init__() self.if_index = if_index self.if_global_timing = if_global_timing self.if_modulo_timing = if_modulo_timing self.dropout = nn.Dropout(p=dropout) self.index_embedding = Fundamental_Music_Embedding(d_model = d_model, base=10000, device = device, if_trainable=False, translation_bias_type = None, if_translation_bias_trainable = False, type = "se").cuda() self.global_time_embedding = Fundamental_Music_Embedding(d_model = d_model, base=10001, device = device, if_trainable=False, translation_bias_type = None, if_translation_bias_trainable = False, type = "se").cuda() self.modulo_time_embedding = Fundamental_Music_Embedding(d_model = d_model, base=10001, device = device, if_trainable=False, translation_bias_type = None, if_translation_bias_trainable = False, type = "se").cuda() position = torch.arange(max_len).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model)) pe = torch.zeros(max_len, 1, d_model) pe[:, 0, 0::2] = torch.sin(position * div_term) pe[:, 0, 1::2] = torch.cos(position * div_term) self.register_buffer('pe', pe) ''' if self.if_global_timing: print("pe add global time") if self.if_modulo_timing: print("pe add modulo time") if self.if_index: print("pe add idx") ''' def forward(self, inp,dur_onset_cumsum = None): if self.if_index: pe_index = self.pe[:inp.size(1)] #[seq_len, batch_size, embedding_dim] pe_index = torch.swapaxes(pe_index, 0, 1) #[batch_size, seq_len, embedding_dim] inp += pe_index if self.if_global_timing: global_timing = dur_onset_cumsum global_timing_embedding = self.global_time_embedding(global_timing) inp += global_timing_embedding if self.if_modulo_timing: modulo_timing = dur_onset_cumsum%4 modulo_timing_embedding = self.modulo_time_embedding(modulo_timing) inp += modulo_timing_embedding return self.dropout(inp) class PositionalEncoding(nn.Module): def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 5000): super().__init__() self.dropout = nn.Dropout(p=dropout) position = torch.arange(max_len).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model)) pe = torch.zeros(max_len, 1, d_model) pe[:, 0, 0::2] = torch.sin(position * div_term) pe[:, 0, 1::2] = torch.cos(position * div_term) self.register_buffer('pe', pe) def forward(self, x): pos = self.pe[:x.size(1)] #[seq_len, batch_size, embedding_dim] pos = torch.swapaxes(pos, 0, 1) #[batch_size, seq_len, embedding_dim] print("huh????", pos.shape, x.shape) x = x + pos return self.dropout(x) class chord_tokenizer(): def __init__(self,seq_len_chord=88,if_pad = True): # self.pitch_dict = {'pad': 0, "None":1, "A": 2, "A#": 3, "Bb":3, "B":4, "C":5, "C#":6, "Db":6, "D": 7, "D#":8, "Eb":8, "E": 9 ,"F":10, "F#":11, "Gb":11, "G":12, "G#":13, "Ab":13} self.pitch_dict = {'pad': 0, "None":1, "N":1, "A": 2, "A#": 3, "Bb":3, "B":4, "Cb": 4, "B#":5, "C":5, "C#":6, "Db":6, "D": 7, "D#":8, "Eb":8, "E": 9 , "Fb": 9, "E#": 10, "F":10, "F#":11, "Gb":11, "G":12, "G#":13, "Ab":13} self.chord_type_dict = {'pad': 0, "None": 1,"N": 1, "maj": 2, "maj7": 3, "m": 4, "m6": 5, "m7": 6, "m7b5": 7, "6": 8, "7": 9, "aug": 10, "dim":11} #, "/": self.chord_inversion_dict = {'pad': 0, "None":1, "N":1,"inv": 2, "no_inv":3} self.seq_len_chord = seq_len_chord self.if_pad = if_pad def __call__(self, chord, chord_time): if len(chord)==0: chord, chord_time = ["N"], [0.] if self.if_pad: pad_len_chord = self.seq_len_chord - len(chord) chord_mask = [True]*len(chord) +[False]*pad_len_chord chord += ["pad"]*pad_len_chord chord_time += [chord_time[-1]]*pad_len_chord else: chord_mask = [True]*len(chord) self.chord_root, self.chord_type, self.chord_inv = self.tokenize_chord_lst(chord) self.chord_time = chord_time self.chord_mask = chord_mask # print("out",self.chord_root, self.chord_type, self.chord_inv, self.chord_time, self.chord_mask) return self.chord_root, self.chord_type, self.chord_inv, self.chord_time, self.chord_mask def get_chord_root_type_inversion_timestamp(self, chord): if chord =="pad": return "pad", "pad", "pad" if chord =="N": return "N", "N", "N" if len(chord.split('/'))>1: chord_inv = "inv" else: chord_inv = "no_inv" chord_wo_inv = chord.split('/')[0] if len(chord_wo_inv)>1: # this part might have a '#' or 'b' if chord_wo_inv[1]=='#' or chord_wo_inv[1]=='b': chord_root=chord_wo_inv[0:2] else: chord_root=chord_wo_inv[0] else: chord_root=chord_wo_inv[0] if len(chord_wo_inv)>len(chord_root): chord_type=chord_wo_inv[len(chord_root):] else: chord_type='maj' return chord_root, chord_type, chord_inv def tokenize_chord_lst(self, chord_lst): out_root = [] out_type = [] out_inv = [] for chord in chord_lst: chord_root, chord_type, chord_inversion= self.get_chord_root_type_inversion_timestamp(chord) out_root.append(self.pitch_dict[chord_root]) out_type.append(self.chord_type_dict[chord_type]) out_inv.append(self.chord_inversion_dict[chord_inversion]) return out_root, out_type, out_inv class beat_tokenizer(): def __init__(self,seq_len_beat=88,if_pad = True): self.beat_dict = {'pad': 0, "None":1, 1.: 2, 2.: 3, 3.:4, 4.:5, 5.:6, 6.:7, 7.:8} self.if_pad = if_pad self.seq_len_beat = seq_len_beat def __call__(self, beat_lst): # beats = [[0.56, 1.1, 1.66, 2.24, 2.8, 3.36, 3.92, 4.48, 5.04, 5.6, 6.16, 6.74, 7.32, 7.9, 8.46, 9.0, 9.58], [3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0]] if self.if_pad: if len(beat_lst[0])==0: beat_mask = [False]*self.seq_len_beat beat_lst = [[0.]*self.seq_len_beat, ["pad"]*self.seq_len_beat] else: pad_len_beat = self.seq_len_beat - len(beat_lst[0]) beat_mask = [True]*len(beat_lst[0]) +[False]*pad_len_beat beat_lst = [beat_lst[0]+[beat_lst[0][-1]]*pad_len_beat, beat_lst[1]+["pad"]*pad_len_beat ] else: beat_mask = [True]*len(beat_lst[0]) self.beat = [self.beat_dict[x] for x in beat_lst[1]] self.beat_timing = beat_lst[0] return self.beat, self.beat_timing, beat_mask # class beat_tokenizer_by_frame(): # def __init__(self, frame_resolution = 0.01, max_len = 10): # def __call__(self, beat_lst): # def timestamp2frame(,frame_resolution, max_len): # def frame2timestamp(frame_resolution, man_len) def l2_norm(a, b): return torch.linalg.norm(a-b, ord = 2, dim = -1) def rounding(x): return x-torch.sin(2.*math.pi*x)/(2.*math.pi) class Chord_Embedding(nn.Module): def __init__(self, FME, PE, d_model = 256, d_oh_type = 12, d_oh_inv = 4): super().__init__() self.FME = FME self.PE = PE self.d_model = d_model self.d_oh_type = d_oh_type self.d_oh_inv = d_oh_inv self.chord_ffn = nn.Linear(d_oh_type + d_oh_inv + d_model + d_model, d_model).cuda() def __call__(self, chord_root, chord_type, chord_inv, chord_timing): #chords: (B, LEN, 4) #Embed root using FME #Embed chord type, chord inversion using OH #Embed timestamps using shared PE chord_root_emb = self.FME(chord_root) # print(chord_root_emb.size()) # print('this is chord root: ', chord_root) # print('this is chord type: ', chord_type) # print('this is chord inv: ', chord_inv) # chord_root_emb = torch.randn((2,20,1024)).cuda() # print(chord_root_emb.device) # chord_root_emb = F.one_hot(chord_type.to(torch.int64), num_classes = self.d_model).to(torch.float32) chord_type_emb = F.one_hot(chord_type.to(torch.int64), num_classes = self.d_oh_type).to(torch.float32) chord_inv_emb = F.one_hot(chord_inv.to(torch.int64), num_classes = self.d_oh_inv).to(torch.float32) chord_time_emb = self.PE.global_time_embedding(chord_timing) chord_emb = self.chord_ffn(torch.cat((chord_root_emb, chord_type_emb, chord_inv_emb, chord_time_emb), dim = -1)) # print("TADY toje", chord_emb.device) return chord_emb class Beat_Embedding(nn.Module): def __init__(self, PE, d_model = 256, d_oh_beat_type = 4): super().__init__() self.PE = PE self.d_model = d_model self.d_oh_beat_type = d_oh_beat_type self.beat_ffn = nn.Linear(d_oh_beat_type+d_model, d_model) def __call__(self, beats, beats_timing): #Embed beat type using OH #Embed time using PE beat_type_emb = F.one_hot(beats.to(torch.int64), num_classes = self.d_oh_beat_type).to(torch.float32) beat_time_emb = self.PE.global_time_embedding(beats_timing) merged_beat = torch.cat((beat_type_emb, beat_time_emb), dim = -1).cuda() beat_emb = self.beat_ffn(merged_beat) return beat_emb if __name__ == "__main__": config_path = "/data/nicolas/TANGO/config/model_embedding_config.yaml" with open (config_path, 'r') as f: cfg = yaml.safe_load(f) beats = [[0.56, 1.1, 1.66, 2.24, 2.8, 3.36, 3.92, 4.48, 5.04, 5.6, 6.16, 6.74, 7.32, 7.9, 8.46, 9.0, 9.58], [3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0]] beats = np.array(beats).T.tolist() chords = [["Gm", 0.464399092], ["Eb", 1.393197278], ["F", 3.157913832], ["Bb", 4.736870748], ["F7", 5.758548752], ["Gm", 6.501587301], ["Eb", 8.173424036], ["F7", 9.938140589]] chord_tokenizer = chord_tokenizer(seq_len_chord=30,if_pad = True) beat_tokenizer = beat_tokenizer(seq_len_beat=17,if_pad = True) #TOKENIZE CHORDS AND BEATS AT DATALOADING PART chord_tokens, chord_masks = chord_tokenizer(chords)#adding batch dimension beat_tokens, beat_masks = beat_tokenizer(beats) chord_tokens, chord_masks, beat_tokens, beat_masks = chord_tokens[None, ...], chord_masks[None, ...], beat_tokens[None, ...], beat_masks[None, ...] #adding batch dimension print("tokeninzing chords and beats", chord_tokens.shape, beat_tokens.shape) #EMBEDDING CHORDS AND BEATS WITHIN THE MODEL FME = Fundamental_Music_Embedding(**cfg["FME_embedding_conf"]) PE = Music_PositionalEncoding(**cfg["Position_encoding_conf"]) chord_embedding_layer = Chord_Embedding(FME, PE, **cfg["Chord_Embedding_conf"]) chord_embedded = chord_embedding_layer(chord_tokens) beat_embedding_layer = Beat_Embedding(PE, **cfg["Beat_Embedding_conf"]) beat_embedded = beat_embedding_layer(beat_tokens) print("embedding tokenized chords and beats", chord_embedded.shape, beat_embedded.shape)