Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import torch | |
| import random | |
| from unidecode import unidecode | |
| from transformers import GPT2LMHeadModel | |
| from samplings import top_p_sampling, temperature_sampling | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| description = """ | |
| <div> | |
| <a style="display:inline-block" href='https://github.com/sander-wood/tunesformer'><img src='https://img.shields.io/github/stars/sander-wood/tunesformer?style=social' /></a> | |
| <a style="display:inline-block" href="https://huggingface.co/datasets/sander-wood/massive_abcnotation_dataset"><img src="https://img.shields.io/badge/huggingface-dataset-ffcc66.svg"></a> | |
| <a style="display:inline-block" href="https://arxiv.org/pdf/2301.02884.pdf"><img src="https://img.shields.io/badge/arXiv-2301.02884-b31b1b.svg"></a> | |
| </div> | |
| ## ℹ️ How to use this demo? | |
| 1. Enter the control codes to set the musical form of the generated music. For details, please refer to the below "Control Codes" section. (optional) | |
| 2. Enter the prefix of the generated music. You can set the ABC header (i.e., note length, tempo, meter, and key) and the beginning of the music. (optional) | |
| 2. You can set the parameters (i.e., number of tunes, maximum length, top-p, temperature, and random seed) for the generation. (optional) | |
| 3. Click "Submit" and wait for the result. | |
| 4. The generated ABC notation can be converted to MIDI or PDF using [EasyABC](https://sourceforge.net/projects/easyabc/), you can also use this [online renderer](https://ldzhangyx.github.io/abc/) to render the ABC notation. | |
| ## 📝 Control Codes | |
| There are 3 types of control codes: section bar `[BARS_NB]`, `[SECS_NS]`, and similarity `[SIM_EDS]`. The control codes can be used to set the musical form of the generated music: | |
| - `[BARS_NB]`: controls the number of bars in a section of the melody. For example, users could specify that they want a section to contain 8 bars, and TunesFormer would generate a section that fits within that structure. It counts on the bar symbol `|`. The value of `NB` ranges from 1 to 32, where 1 means one bar and 32 means thirty-two bars. | |
| - `[SECS_NS]`: controls the number of sections in the entire melody. This can be used to create a sense of structure and coherence within the melody, as different sections can be used to create musical themes or motifs. It counts on several symbols that are commonly used in ABC notation and can be used to represent section boundaries: `[|`,`||`,`|]`,`|:`,`::`, and `:|`. The value of `NS` ranges from 1 to 8, where 1 means one section and 8 means eight sections. | |
| - `[SIM_EDS]`: controls the similarity between the generated music and the prefix. The value of `EDS` is the edit distance similarity level between the current section and a previous section in the melody. The value of EDS ranges from 0 to 10, where 0 means no similarity and 10 means the same section. | |
| To provide a clearer understanding of the control codes, we provide an example below: | |
| `[SECS_3][BARS_8][SIM_3][BARS_8][SIM_10][SIM_3][BARS_8]` | |
| The first control code `[SECS_3]` specifies there are 3 sections in the tune, and the following control code `[BARS_8]` indicates the first section has 8 bars. The next two control codes `[SIM_3]` and `[BAR_8]` indicate that the EDS between section II and section I is approximately 0.3, and section II has 8 bars. The last three control codes `[SIM_10]`, `[SIM_3]` and `[BARS_8]` specify that section III is identical to section I while dissimilar to section II, and has 8 bars. | |
| ## ❕Notice | |
| - If the prefix is given, you must also provide the control codes. | |
| - The order of the ABC header cannot be changed (i.e., note length, tempo, meter, and key), and also do not support other headers (e.g., composer, title, etc.) as they are not used in the model. For details, please refer to the official [ABC notation](https://abcnotation.com/wiki/abc:standard:v2.1#description_of_information_fields). | |
| - The demo is based on GPT2-small and trained from scratch on the [Massive ABC Notation Dataset](https://huggingface.co/datasets/sander-wood/massive_abcnotation_dataset). | |
| - The demo is still in the early stage, and the generated music is not perfect. If you have any suggestions, please feel free to contact me via [email](mailto:shangda@mail.ccom.edu.cn). | |
| """ | |
| class ABCTokenizer(): | |
| def __init__(self): | |
| self.pad_token_id = 0 | |
| self.bos_token_id = 2 | |
| self.eos_token_id = 3 | |
| self.merged_tokens = [] | |
| for i in range(8): | |
| self.merged_tokens.append('[SECS_'+str(i+1)+']') | |
| for i in range(32): | |
| self.merged_tokens.append('[BARS_'+str(i+1)+']') | |
| for i in range(11): | |
| self.merged_tokens.append('[SIM_'+str(i)+']') | |
| def __len__(self): | |
| return 128+len(self.merged_tokens) | |
| def encode(self, text): | |
| encodings = {} | |
| encodings['input_ids'] = torch.tensor(self.txt2ids(text, self.merged_tokens)) | |
| encodings['attention_mask'] = torch.tensor([1]*len(encodings['input_ids'])) | |
| return encodings | |
| def decode(self, ids, skip_special_tokens=False): | |
| txt = "" | |
| for i in ids: | |
| if i>=128: | |
| if not skip_special_tokens: | |
| txt += self.merged_tokens[i-128] | |
| elif i!=self.bos_token_id and i!=self.eos_token_id: | |
| txt += chr(i) | |
| return txt | |
| def txt2ids(self, text, merged_tokens): | |
| ids = ["\""+str(ord(c))+"\"" for c in text] | |
| txt_ids = ' '.join(ids) | |
| for t_idx, token in enumerate(merged_tokens): | |
| token_ids = ["\""+str(ord(c))+"\"" for c in token] | |
| token_txt_ids = ' '.join(token_ids) | |
| txt_ids = txt_ids.replace(token_txt_ids, "\""+str(t_idx+128)+"\"") | |
| txt_ids = txt_ids.split(' ') | |
| txt_ids = [int(i[1:-1]) for i in txt_ids] | |
| return [self.bos_token_id]+txt_ids+[self.eos_token_id] | |
| def generate_abc(control_codes, prefix, num_tunes, max_length, top_p, temperature, seed): | |
| try: | |
| seed = int(seed) | |
| except: | |
| seed = None | |
| prefix = unidecode(control_codes + prefix) | |
| tokenizer = ABCTokenizer() | |
| model = GPT2LMHeadModel.from_pretrained('sander-wood/tunesformer').to(device) | |
| if prefix: | |
| ids = tokenizer.encode(prefix)['input_ids'][:-1] | |
| else: | |
| ids = torch.tensor([tokenizer.bos_token_id]) | |
| random.seed(seed) | |
| tunes = "" | |
| for c_idx in range(num_tunes): | |
| print("\nX:"+str(c_idx+1)+"\n", end="") | |
| print(tokenizer.decode(ids[1:], skip_special_tokens=True), end="") | |
| input_ids = ids.unsqueeze(0) | |
| for t_idx in range(max_length): | |
| if seed!=None: | |
| n_seed = random.randint(0, 1000000) | |
| random.seed(n_seed) | |
| else: | |
| n_seed = None | |
| outputs = model(input_ids=input_ids.to(device)) | |
| probs = outputs.logits[0][-1] | |
| probs = torch.nn.Softmax(dim=-1)(probs).cpu().detach().numpy() | |
| sampled_id = temperature_sampling(probs=top_p_sampling(probs, | |
| top_p=top_p, | |
| seed=n_seed, | |
| return_probs=True), | |
| seed=n_seed, | |
| temperature=temperature) | |
| input_ids = torch.cat((input_ids, torch.tensor([[sampled_id]])), 1) | |
| if sampled_id!=tokenizer.eos_token_id: | |
| print(tokenizer.decode([sampled_id], skip_special_tokens=True), end="") | |
| continue | |
| else: | |
| tune = "X:"+str(c_idx+1)+"\n"+tokenizer.decode(input_ids.squeeze(), skip_special_tokens=True) | |
| tunes += tune+"\n\n" | |
| print("\n") | |
| break | |
| return tunes | |
| input_control_codes = gr.inputs.Textbox(lines=5, label="Control Codes", default="[SECS_2][BARS_9][SIM_3][BARS_9]") | |
| input_prefix = gr.inputs.Textbox(lines=5, label="Prefix", default="L:1/8\nQ:1/4=114\nM:3/4\nK:D\nde | \"D\"") | |
| input_num_tunes = gr.inputs.Slider(minimum=1, maximum=10, step=1, default=1, label="Number of Tunes") | |
| input_max_length = gr.inputs.Slider(minimum=10, maximum=1000, step=10, default=500, label="Max Length") | |
| input_top_p = gr.inputs.Slider(minimum=0.0, maximum=1.0, step=0.05, default=0.9, label="Top P") | |
| input_temperature = gr.inputs.Slider(minimum=0.0, maximum=2.0, step=0.1, default=1.0, label="Temperature") | |
| input_seed = gr.inputs.Textbox(lines=1, label="Seed (int)", default="None") | |
| output_abc = gr.outputs.Textbox(label="Generated Tunes") | |
| gr.Interface(generate_abc, | |
| [input_control_codes, input_prefix, input_num_tunes, input_max_length, input_top_p, input_temperature, input_seed], | |
| output_abc, | |
| title="TunesFormer: Forming Tunes with Control Codes", | |
| description=description).launch() | |