Spaces:
Runtime error
Runtime error
Cédric Colas
commited on
Commit
•
e775f6d
1
Parent(s):
ccc7bf6
initial commit
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- .gitattributes +5 -5
- .gitignore +125 -0
- README.md +6 -6
- app.py +214 -0
- checkpoints/cocktail_representation/cocktails_data.csv +3 -0
- checkpoints/cocktail_representation/handcoded_reps/cocktail_handcoded_reps_minmax_norm-1_1_dim13_customkeys.txt +3 -0
- checkpoints/cocktail_representation/handcoded_reps/cocktail_minmax_dim13_customkeys.txt +3 -0
- checkpoints/cocktail_representation/handcoded_reps/density_ingredients.txt +3 -0
- checkpoints/cocktail_representation/handcoded_reps/distrib_nb_ing.txt +3 -0
- checkpoints/cocktail_representation/handcoded_reps/ingredients_max_quantities.txt +3 -0
- checkpoints/cocktail_representation/handcoded_reps/ingredients_min_quantities.txt +3 -0
- checkpoints/cocktail_representation/handcoded_reps/ingredients_min_quantities_when_present.txt +3 -0
- checkpoints/cocktail_representation/handcoded_reps/nn_model.pickle +3 -0
- checkpoints/cocktail_representation/ingredient_list.csv +3 -0
- checkpoints/cocktail_representation/minmax/max_arousal.txt +3 -0
- checkpoints/cocktail_representation/minmax/max_cocktail_reps_affective.txt +3 -0
- checkpoints/cocktail_representation/minmax/max_dominance.txt +3 -0
- checkpoints/cocktail_representation/minmax/max_valence.txt +3 -0
- checkpoints/cocktail_representation/minmax/min_arousal.txt +3 -0
- checkpoints/cocktail_representation/minmax/min_cocktail_reps_affective.txt +3 -0
- checkpoints/cocktail_representation/minmax/min_dominance.txt +3 -0
- checkpoints/cocktail_representation/minmax/min_valence.txt +3 -0
- checkpoints/cocktail_representation/multi_predictor/checkpoint_best.save +3 -0
- checkpoints/cocktail_representation/multi_predictor/max_ing_quantities.txt +3 -0
- checkpoints/cocktail_representation/multi_predictor/mean_ing_quantities.txt +3 -0
- checkpoints/cocktail_representation/multi_predictor/min_when_present_ing_quantities.txt +3 -0
- checkpoints/cocktail_representation/multi_predictor/normalization_funcs.pickle +3 -0
- checkpoints/cocktail_representation/multi_predictor/params.json +3 -0
- checkpoints/cocktail_representation/multi_predictor/std_ing_quantities.txt +3 -0
- checkpoints/music2cocktails/affects2affect_cluster/cluster_model.pickle +3 -0
- checkpoints/music2cocktails/music2affects/final_best_keys.pickle +3 -0
- checkpoints/music2cocktails/music2affects/max_handcoded_reps.txt +3 -0
- checkpoints/music2cocktails/music2affects/min_handcoded_reps.txt +3 -0
- checkpoints/music2cocktails/music2affects/music2affect_models.pickle +3 -0
- checkpoints/music2cocktails/music2affects/top_10_keys.pickle +3 -0
- checkpoints/music2cocktails/music2flavor/b256_r128_classif001_ld40_meanstd_regground2.5_egg_bubbles/params.json +3 -0
- checkpoints/music2cocktails/music2flavor/b256_r128_classif001_ld40_meanstd_regground2.5_egg_bubbles/results.txt +3 -0
- checkpoints/music_representation/b256_r128_represented/all_reps_unnormalized_sample100.pickle +3 -0
- checkpoints/music_representation/b256_r128_represented/all_reps_unnormalized_sample1000.pickle +3 -0
- checkpoints/music_representation/b256_r128_represented/all_reps_unnormalized_sample200.pickle +3 -0
- checkpoints/music_representation/b256_r128_represented/all_reps_unnormalized_sample2000.pickle +3 -0
- checkpoints/music_representation/b256_r128_represented/all_reps_unnormalized_sample500.pickle +3 -0
- checkpoints/music_representation/b256_r128_represented/all_reps_unnormalized_sample5000.pickle +3 -0
- checkpoints/music_representation/b256_r128_represented/music_reps_normalized_meanstd.pickle +3 -0
- checkpoints/music_representation/b256_r128_represented/music_reps_unnormalized.pickle +3 -0
- checkpoints/music_representation/b256_r128_represented/nn_model.pickle +3 -0
- checkpoints/music_representation/sentence_embedding/smallbert_b256_r128_1/acc.png +3 -0
- checkpoints/music_representation/sentence_embedding/smallbert_b256_r128_1/best_model/1_Pooling/config.json +3 -0
- checkpoints/music_representation/sentence_embedding/smallbert_b256_r128_1/best_model/2_Dense/config.json +3 -0
- checkpoints/music_representation/sentence_embedding/smallbert_b256_r128_1/best_model/2_Dense/pytorch_model.bin +3 -0
.gitattributes
CHANGED
@@ -9,13 +9,9 @@
|
|
9 |
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
10 |
*.model filter=lfs diff=lfs merge=lfs -text
|
11 |
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
12 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
13 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
14 |
*.onnx filter=lfs diff=lfs merge=lfs -text
|
15 |
*.ot filter=lfs diff=lfs merge=lfs -text
|
16 |
*.parquet filter=lfs diff=lfs merge=lfs -text
|
17 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
18 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
19 |
*.pb filter=lfs diff=lfs merge=lfs -text
|
20 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
21 |
*.pth filter=lfs diff=lfs merge=lfs -text
|
@@ -27,5 +23,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
27 |
*.wasm filter=lfs diff=lfs merge=lfs -text
|
28 |
*.xz filter=lfs diff=lfs merge=lfs -text
|
29 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
30 |
-
*.
|
31 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
9 |
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
10 |
*.model filter=lfs diff=lfs merge=lfs -text
|
11 |
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
12 |
*.onnx filter=lfs diff=lfs merge=lfs -text
|
13 |
*.ot filter=lfs diff=lfs merge=lfs -text
|
14 |
*.parquet filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
15 |
*.pb filter=lfs diff=lfs merge=lfs -text
|
16 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
17 |
*.pth filter=lfs diff=lfs merge=lfs -text
|
|
|
23 |
*.wasm filter=lfs diff=lfs merge=lfs -text
|
24 |
*.xz filter=lfs diff=lfs merge=lfs -text
|
25 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
26 |
+
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
27 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
28 |
+
data/* filter=lfs diff=lfs merge=lfs -text
|
29 |
+
data/**/* filter=lfs diff=lfs merge=lfs -text
|
30 |
+
checkpoints/**/* filter=lfs diff=lfs merge=lfs -text
|
31 |
+
checkpoints/* filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Byte-compiled / optimized / DLL files
|
2 |
+
__pycache__/
|
3 |
+
*.py[cod]
|
4 |
+
*$py.class
|
5 |
+
|
6 |
+
# C extensions
|
7 |
+
*.so
|
8 |
+
|
9 |
+
# Distribution / packaging
|
10 |
+
.Python
|
11 |
+
build/
|
12 |
+
develop-eggs/
|
13 |
+
dist/
|
14 |
+
downloads/
|
15 |
+
eggs/
|
16 |
+
.eggs/
|
17 |
+
lib/
|
18 |
+
lib64/
|
19 |
+
parts/
|
20 |
+
sdist/
|
21 |
+
var/
|
22 |
+
wheels/
|
23 |
+
*.egg-info/
|
24 |
+
.installed.cfg
|
25 |
+
*.egg
|
26 |
+
MANIFEST
|
27 |
+
|
28 |
+
# General MAC files
|
29 |
+
.DS_Store
|
30 |
+
.AppleDouble
|
31 |
+
.LSOverride
|
32 |
+
|
33 |
+
# PyInstaller
|
34 |
+
# Usually these files are written by a python script from a template
|
35 |
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
36 |
+
*.manifest
|
37 |
+
*.spec
|
38 |
+
|
39 |
+
# Installer logs
|
40 |
+
pip-log.txt
|
41 |
+
pip-delete-this-directory.txt
|
42 |
+
|
43 |
+
# Unit test / coverage reports
|
44 |
+
htmlcov/
|
45 |
+
.tox/
|
46 |
+
.coverage
|
47 |
+
.coverage.*
|
48 |
+
.cache
|
49 |
+
nosetests.xml
|
50 |
+
coverage.xml
|
51 |
+
*.cover
|
52 |
+
.hypothesis/
|
53 |
+
.pytest_cache/
|
54 |
+
|
55 |
+
# Translations
|
56 |
+
*.mo
|
57 |
+
*.pot
|
58 |
+
|
59 |
+
# Django stuff:
|
60 |
+
*.log
|
61 |
+
local_settings.py
|
62 |
+
db.sqlite3
|
63 |
+
|
64 |
+
# Flask stuff:
|
65 |
+
instance/
|
66 |
+
.webassets-cache
|
67 |
+
|
68 |
+
# Scrapy stuff:
|
69 |
+
.scrapy
|
70 |
+
|
71 |
+
# Sphinx documentation
|
72 |
+
docs/_build/
|
73 |
+
|
74 |
+
# PyBuilder
|
75 |
+
target/
|
76 |
+
|
77 |
+
# Jupyter Notebook
|
78 |
+
.ipynb_checkpoints
|
79 |
+
|
80 |
+
# pyenv
|
81 |
+
.python-version
|
82 |
+
|
83 |
+
# celery beat schedule file
|
84 |
+
celerybeat-schedule
|
85 |
+
|
86 |
+
# SageMath parsed files
|
87 |
+
*.sage.py
|
88 |
+
|
89 |
+
# Environments
|
90 |
+
.env
|
91 |
+
.venv
|
92 |
+
env/
|
93 |
+
venv/
|
94 |
+
ENV/
|
95 |
+
env.bak/
|
96 |
+
venv.bak/
|
97 |
+
|
98 |
+
# Spyder project settings
|
99 |
+
.spyderproject
|
100 |
+
.spyproject
|
101 |
+
|
102 |
+
# Rope project settings
|
103 |
+
.ropeproject
|
104 |
+
|
105 |
+
# mkdocs documentation
|
106 |
+
/site
|
107 |
+
|
108 |
+
# mypy
|
109 |
+
.mypy_cache/
|
110 |
+
|
111 |
+
#Pycharm
|
112 |
+
.idea/
|
113 |
+
|
114 |
+
# data
|
115 |
+
moodboard/*
|
116 |
+
misc/*
|
117 |
+
experiments/*
|
118 |
+
**/saved_models/*
|
119 |
+
*.pt
|
120 |
+
*.zip
|
121 |
+
*.npy
|
122 |
+
*.jbl
|
123 |
+
|
124 |
+
|
125 |
+
/experiments/
|
README.md
CHANGED
@@ -1,12 +1,12 @@
|
|
1 |
---
|
2 |
-
title:
|
3 |
-
emoji:
|
4 |
-
colorFrom:
|
5 |
-
colorTo:
|
6 |
sdk: streamlit
|
7 |
-
sdk_version: 1.
|
8 |
app_file: app.py
|
9 |
pinned: false
|
10 |
---
|
11 |
|
12 |
-
Check out the configuration reference at https://huggingface.co/docs/hub/spaces
|
|
|
1 |
---
|
2 |
+
title: TastyPiano
|
3 |
+
emoji: 🍹
|
4 |
+
colorFrom: pink
|
5 |
+
colorTo: purple
|
6 |
sdk: streamlit
|
7 |
+
sdk_version: 1.2.0
|
8 |
app_file: app.py
|
9 |
pinned: false
|
10 |
---
|
11 |
|
12 |
+
Check out the configuration reference at https://huggingface.co/docs/hub/spaces#reference
|
app.py
ADDED
@@ -0,0 +1,214 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import time
|
2 |
+
import os
|
3 |
+
import pickle
|
4 |
+
import sys
|
5 |
+
sys.path.append('/')
|
6 |
+
from src.music.pipeline.music_pipeline import encode_music
|
7 |
+
from src.music2cocktailrep.pipeline.music2cocktailrep import music2cocktailrep, setup_translation_models, debug_translation
|
8 |
+
from src.cocktails.pipeline.cocktailrep2recipe import cocktailrep2recipe
|
9 |
+
from src.debugger import Debugger
|
10 |
+
from datetime import datetime
|
11 |
+
from shutil import copy
|
12 |
+
import streamlit as st
|
13 |
+
from src.music.config import AUDIO_PATH, MIDI_PATH
|
14 |
+
from pretty_midi import PrettyMIDI
|
15 |
+
import numpy as np
|
16 |
+
import pydub
|
17 |
+
from PIL import Image
|
18 |
+
|
19 |
+
|
20 |
+
st.set_page_config(
|
21 |
+
page_title="TastyPiano",
|
22 |
+
page_icon="🎹",
|
23 |
+
)
|
24 |
+
|
25 |
+
st.title('TastyPiano')
|
26 |
+
|
27 |
+
synestesia_path = 'data/synesthesia/'
|
28 |
+
debugger = Debugger(verbose=False)
|
29 |
+
|
30 |
+
def centered_module(func, text, args=None):
|
31 |
+
_, col2, _ = st.columns([1, 2, 1])
|
32 |
+
with col2:
|
33 |
+
return func(text)
|
34 |
+
|
35 |
+
def centered_button(func, text, args=None):
|
36 |
+
_, _, _, col3, _, _, _ = st.columns([1, 1, 1, 1, 1, 1, 1])
|
37 |
+
with col3:
|
38 |
+
return func(text)
|
39 |
+
|
40 |
+
def setup_streamlite():
|
41 |
+
setup_translation_models()
|
42 |
+
image = Image.open('./data/pianocktail.jpg')
|
43 |
+
st.image(image, caption='Pianocktail by Compagnie la Rumeur')
|
44 |
+
|
45 |
+
st.subheader("Ready to taste music?")
|
46 |
+
st.markdown("TastyPiano generates music--taste synesthetic experiences by letting you turn any piano song into a cocktail. It is inspired by the"
|
47 |
+
" *Pianocktail*, a literary invention from the French novelist, jazz musician, singer and engineer Boris Vian. I see TastyPiano as a digital member of the "
|
48 |
+
"Pianocktail species, along with [other](https://www.youtube.com/watch?v=5bdX5i0nAWw) "
|
49 |
+
"[wonderful](https://www.youtube.com/watch?v=pzsDOH-xtrs&list=PLC196AA37A2D1C066&index=3) [machines](https://www.youtube.com/watch?v=y0RJg7I2x34).")
|
50 |
+
st.markdown("But TastyPiano is different from existing pianocktails. While existing version merely map played notes to drops of corresponding ingredients, "
|
51 |
+
"TastyPiano listens to the song, analyzes the music, hallucinates matching tastes and flavors, before it starts generating cocktail recipes "
|
52 |
+
"until it finds the one matching its gustatory hallucination. Check my [blog post](https://ccolas.github.io/project/pianocktail) for more information."
|
53 |
+
" Hear and taste it for yourself!")
|
54 |
+
st.subheader("How to use it?")
|
55 |
+
st.markdown("Provide a piano solo input and click on the **Taste it!** button.\n"
|
56 |
+
"\nYou can input either: \n* a YouTube url **or**\n* an audio file (.mp3) **or**\n* a midi file (.mid)\n"
|
57 |
+
"All these should be **only piano**, no other instrument.\n"
|
58 |
+
"Note that audio sources are cropped to the first 40s because the process of "
|
59 |
+
"converting it to midi is rather slow (1sec/sec). Midi inputs are taken whole. Please report any bug / buggy url in the community tab.")
|
60 |
+
|
61 |
+
st.subheader("Prepare")
|
62 |
+
col1, col2, col3 = st.columns(3)
|
63 |
+
generate_audio_from_midi = False
|
64 |
+
with col1:
|
65 |
+
st.markdown('**YouTube url**')
|
66 |
+
url = st.text_area('Type it below', 'https://www.youtube.com/watch?v=UGK70IkP830', height=160)
|
67 |
+
with col2:
|
68 |
+
st.markdown('**Audio file**')
|
69 |
+
audio = st.file_uploader("Upload it here (.mp3)", type=['.mp3'])
|
70 |
+
with col3:
|
71 |
+
st.markdown('**Midi file**')
|
72 |
+
midi = st.file_uploader("Upload it here (.mid)", type=['.mid'])
|
73 |
+
generate_audio_from_midi = st.checkbox('Generate audio? Untick if the song is too long (>10min)', value=True)
|
74 |
+
|
75 |
+
def run(unit):
|
76 |
+
setup_and_run(unit=unit, url=url, midi=midi, audio=audio, generate_audio_from_midi=generate_audio_from_midi, extra_code=None)
|
77 |
+
# url = "https://youtu.be/VUCI-1vIbUo"
|
78 |
+
# unit = 'mL'
|
79 |
+
# run(unit)
|
80 |
+
st.markdown('##')
|
81 |
+
unit = st.radio('Pick the units (before pressing "Taste it!", default mL)', ['mL', 'oz'], index=0)
|
82 |
+
button = centered_button(st.button, 'Taste it!')
|
83 |
+
if button:
|
84 |
+
run(unit)
|
85 |
+
|
86 |
+
|
87 |
+
def pianocktail(unit='mL', record=False, url=None, midi=None, audio=None, processed=None, crop=40, verbose=False, debug=False, level=0):
|
88 |
+
assert url is not None or midi is not None or audio is not None or processed is not None
|
89 |
+
if verbose: print('------\nNew synesthetic exploration!')
|
90 |
+
init_time = time.time()
|
91 |
+
try:
|
92 |
+
with st.spinner("Listening to the song (~20s).."):
|
93 |
+
music_ai_rep, music_handcoded_rep, all_paths, error = encode_music(record=record, url=url, audio_path=audio, midi_path=midi, nb_aug=0, noise_injection=False,
|
94 |
+
augmentation=False, processed_path=processed, crop=crop, apply_filtering=False, verbose=verbose,
|
95 |
+
level=level+2)
|
96 |
+
if music_ai_rep is not None:
|
97 |
+
with st.spinner(text="Thinking about corresponding flavors.."):
|
98 |
+
cocktail_rep, affective_cluster_id, affect = music2cocktailrep(music_ai_rep, music_handcoded_rep, verbose=verbose, level=level+2)
|
99 |
+
with st.spinner("Trying recipes (15s).."):
|
100 |
+
cocktail_recipes, scores = cocktailrep2recipe(cocktail_rep, unit=unit, target_affective_cluster=affective_cluster_id, verbose=verbose, full_verbose=verbose, \
|
101 |
+
level=level+2)
|
102 |
+
|
103 |
+
cocktail_recipe = cocktail_recipes[0]
|
104 |
+
recipe_score = scores[0]
|
105 |
+
if debug:
|
106 |
+
music_reconstruction = debug_translation(music_ai_rep)
|
107 |
+
debugger.extract_info(all_paths, affective_cluster_id, affect, cocktail_rep, music_reconstruction, recipe_score, verbose=verbose, level=level+2)
|
108 |
+
debug_info = debugger.debug_dict
|
109 |
+
else:
|
110 |
+
debug_info = None
|
111 |
+
if verbose:
|
112 |
+
print(cocktail_recipe.replace('Recipe', ' ' * (level + 2) + 'Generated recipe:').replace('None ()', ''))
|
113 |
+
debugger.print_debug(level=level+2)
|
114 |
+
print(f'\nEnd of synesthetic exploration ({int(time.time() - init_time)} secs).\n------')
|
115 |
+
st.success('Recipe found!')
|
116 |
+
else:
|
117 |
+
st.error('Error in listening. Is the url valid? the audio an .mp3? the midi a .mid?')
|
118 |
+
cocktail_recipe = None
|
119 |
+
debug_info = None
|
120 |
+
except:
|
121 |
+
st.error('Error: ' + error)
|
122 |
+
cocktail_recipe = None
|
123 |
+
debug_info = None
|
124 |
+
return cocktail_recipe, debug_info
|
125 |
+
|
126 |
+
def setup_and_run(unit='mL', url=None, midi=None, audio=None, generate_audio_from_midi=False, verbose=True, debug=True, extra_code=None):
|
127 |
+
if url is None and midi is None and audio is None:
|
128 |
+
st.error('Please enter a piano input.')
|
129 |
+
assert False
|
130 |
+
st.subheader('Synesthesia')
|
131 |
+
now = datetime.now()
|
132 |
+
folder_name = f'{now.year}-{now.month}-{now.day}_{now.hour}:{now.minute}:{now.second}'
|
133 |
+
folder_path = synestesia_path + folder_name
|
134 |
+
if extra_code is not None:
|
135 |
+
folder_path += '_' + extra_code
|
136 |
+
if os.path.exists(folder_path):
|
137 |
+
folder_path += '_2'
|
138 |
+
folder_path += '/'
|
139 |
+
os.makedirs(folder_path, exist_ok=True)
|
140 |
+
if midi is not None:
|
141 |
+
st.write(f' \tReading from midi file: {midi.name}')
|
142 |
+
midi_path = MIDI_PATH + 'from_url_midi/' + midi.name[:-4] + '_midi.mid'
|
143 |
+
audio_path = AUDIO_PATH + 'from_url/' + midi.name.replace('.mid', '.mp3')
|
144 |
+
with open(midi_path, "wb") as f:
|
145 |
+
f.write(midi.getbuffer())
|
146 |
+
midi = midi_path
|
147 |
+
if generate_audio_from_midi:
|
148 |
+
midi_data = PrettyMIDI(midi_path)
|
149 |
+
audio_data = midi_data.fluidsynth(fs=44100)
|
150 |
+
y = np.int16(audio_data * 2 ** 15)
|
151 |
+
song = pydub.AudioSegment(y.tobytes(), frame_rate=44100, sample_width=2, channels=1)
|
152 |
+
song.export(audio_path, format="mp3", bitrate="320k")
|
153 |
+
# st.write(audio_data)
|
154 |
+
st.audio(audio_path, format='audio/mp3')
|
155 |
+
url = None
|
156 |
+
elif audio is not None:
|
157 |
+
st.write(f' \tReading from audio file: {audio.name}')
|
158 |
+
audio_path = AUDIO_PATH + 'from_url/' + audio.name
|
159 |
+
with open(audio_path, "wb") as f:
|
160 |
+
f.write(audio.getbuffer())
|
161 |
+
audio = audio_path
|
162 |
+
audio_file = open(audio, 'rb')
|
163 |
+
audio_bytes = audio_file.read()
|
164 |
+
st.audio(audio_bytes, format='audio/mp3')
|
165 |
+
url = None
|
166 |
+
else:
|
167 |
+
st.write(f' \tReading from YouTube url: {url}')
|
168 |
+
st.video(url)
|
169 |
+
|
170 |
+
_, col2, _ = st.columns([1, 1, 1])
|
171 |
+
with col2:
|
172 |
+
st.markdown('##')
|
173 |
+
recipe, debug = pianocktail(unit=unit, url=url, midi=midi, audio=audio, verbose=verbose, debug=debug)
|
174 |
+
with open(folder_path + 'debug.pk', 'wb') as f:
|
175 |
+
pickle.dump(debug, f)
|
176 |
+
with open(folder_path + 'recipe.txt', 'w') as f:
|
177 |
+
f.write(recipe)
|
178 |
+
paths = debug['all_paths']
|
179 |
+
if paths['url'] is not None:
|
180 |
+
with open(folder_path + 'url.txt', 'w') as f:
|
181 |
+
f.write(paths['url'])
|
182 |
+
for k in ['audio_path', 'midi_path']:
|
183 |
+
origin = paths[k]
|
184 |
+
if origin is not None:
|
185 |
+
copy(origin, folder_path + origin.split('/')[-1])
|
186 |
+
|
187 |
+
st.subheader('Recipe')
|
188 |
+
recipe = recipe.replace(' Enjoy!', ' \nEnjoy!').replace('\n', ' \n')
|
189 |
+
st.text(recipe)
|
190 |
+
|
191 |
+
st.markdown('**About this synesthesia**')
|
192 |
+
closest_songs = [debug['nn_music'][i][:-26].split('structured_')[1].replace('_', ' ') for i in range(3)]
|
193 |
+
str_songs = 'These are the closest song I know: '
|
194 |
+
str_songs += ' '.join([f' \n* {closest_songs[i]}' for i in range(3)])
|
195 |
+
st.markdown(str_songs + '.')
|
196 |
+
str_cocktails = 'These are existing cocktails that are close to the taste of this song:'
|
197 |
+
str_cocktails += ' '.join([f' \n* {cocktail_name}: {cocktail_url}' for cocktail_name, cocktail_url in zip(debug['nearest_cocktail_names'][:3],
|
198 |
+
debug['nearest_cocktail_urls'][:3])])
|
199 |
+
st.markdown(str_cocktails + '.')
|
200 |
+
|
201 |
+
if __name__ == '__main__':
|
202 |
+
setup_streamlite()
|
203 |
+
# urls = ["https://www.youtube.com/watch?v=PLFVGwGQcB0",
|
204 |
+
# "https://www.youtube.com/watch?v=VQmuAr93OlI",
|
205 |
+
# "https://www.youtube.com/watch?v=Nv2GgV34qIg&list=PLO9E3V4rGLD8_iWrCioJRWZXJJE3Fzu_J&index=4",
|
206 |
+
# "https://www.youtube.com/watch?v=qAEIjWYdoYc&list=PLO9E3V4rGLD8_iWrCioJRWZXJJE3Fzu_J&index=1",
|
207 |
+
# "https://www.youtube.com/watch?v=M73x3O7dhmg&list=PLO9E3V4rGLD8_iWrCioJRWZXJJE3Fzu_J&index=5"]
|
208 |
+
# setup_translation_models()
|
209 |
+
# setup_and_run(url=urls[0], verbose=True, debug=True)
|
210 |
+
# recipes = []
|
211 |
+
# for url in urls:
|
212 |
+
# recipe = pianocktail(url=url, verbose=True, debug=True)[0]
|
213 |
+
# recipes.append(recipe)
|
214 |
+
# stop = 1
|
checkpoints/cocktail_representation/cocktails_data.csv
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:1d3bbceb3696969a87c615c7e32e1e985c77c21f376eef1d6fee1a91dc4cccd9
|
3 |
+
size 513959
|
checkpoints/cocktail_representation/handcoded_reps/cocktail_handcoded_reps_minmax_norm-1_1_dim13_customkeys.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:7b6608088da85fbaea3cc051212483f743db72a9736a13acfe004f53758bda92
|
3 |
+
size 209186
|
checkpoints/cocktail_representation/handcoded_reps/cocktail_minmax_dim13_customkeys.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:d9145908cb34334384b5600b78087e8be1412ca051eac92cf8f399691ac1b215
|
3 |
+
size 651
|
checkpoints/cocktail_representation/handcoded_reps/density_ingredients.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:73b3c6f30619e6bf8d129c4eea33fa84412666bff057a8d3ea19f342267d48fb
|
3 |
+
size 852
|
checkpoints/cocktail_representation/handcoded_reps/distrib_nb_ing.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:39d6fa912402828441f01639106fd6ae9f23c962a639c7b5b14b86f906d06963
|
3 |
+
size 225
|
checkpoints/cocktail_representation/handcoded_reps/ingredients_max_quantities.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:d7e2dcfd30a4c9e71e5162dd54d7be4fdaac128747b5a4247a4edb01ba3216f2
|
3 |
+
size 875
|
checkpoints/cocktail_representation/handcoded_reps/ingredients_min_quantities.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:2d06bb88733125b018552a4041f07724d09b759d52fd2c0bb9ef628dfa6eddac
|
3 |
+
size 875
|
checkpoints/cocktail_representation/handcoded_reps/ingredients_min_quantities_when_present.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:bce522acc70b778fe97ae50011bf8e0a3815c106ec963bbc97bc55c126ff30be
|
3 |
+
size 875
|
checkpoints/cocktail_representation/handcoded_reps/nn_model.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:3a898513c6cebe267725c2be2a59e626e69669640b5a75a96f216f997083dac4
|
3 |
+
size 78303
|
checkpoints/cocktail_representation/ingredient_list.csv
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b41bc1ff60d50addb62d15c385ff8f0efa299a4591626a268b22014568fb437f
|
3 |
+
size 4561
|
checkpoints/cocktail_representation/minmax/max_arousal.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b52578e6a6e8e1f83c35f8823fca8d804ba15483def5e61580a8efc65aec1305
|
3 |
+
size 25
|
checkpoints/cocktail_representation/minmax/max_cocktail_reps_affective.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:1ed0cfac52273c0da08db080d36fa0be832cac89cdfe148073e01bc744b83f08
|
3 |
+
size 200
|
checkpoints/cocktail_representation/minmax/max_dominance.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:01ec9307e9d5afa3b65bc5bbaa9e5d6e25b64b40d22c54fb3497084148c5188f
|
3 |
+
size 25
|
checkpoints/cocktail_representation/minmax/max_valence.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:a67328add2fc55ba06616300a1bc05ccb455edef04cef4aa0979c3cd92cbebe1
|
3 |
+
size 25
|
checkpoints/cocktail_representation/minmax/min_arousal.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:7fca11f8bc207745e78ed0381a3dedf031d7c4c609f6bc3764df9ed8384ec4e7
|
3 |
+
size 26
|
checkpoints/cocktail_representation/minmax/min_cocktail_reps_affective.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:d438cf2fb225d02e12287a36cc1a147ee05554ea803cb3fbd23136e95826a86b
|
3 |
+
size 201
|
checkpoints/cocktail_representation/minmax/min_dominance.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:0ce9888c43b8e9d9369ed9f1873b21daed71a62c937062006d13af550289bb29
|
3 |
+
size 26
|
checkpoints/cocktail_representation/minmax/min_valence.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:176e2fc0f9b8e23c2d18b9e202cf449497f9ad092609b5b5c12f4d054d886e1b
|
3 |
+
size 26
|
checkpoints/cocktail_representation/multi_predictor/checkpoint_best.save
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:2bbd776254e9fd3913eb3d4720a92ae60be8f2887c64e99b9e775829ce241f23
|
3 |
+
size 29027
|
checkpoints/cocktail_representation/multi_predictor/max_ing_quantities.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:d7e2dcfd30a4c9e71e5162dd54d7be4fdaac128747b5a4247a4edb01ba3216f2
|
3 |
+
size 875
|
checkpoints/cocktail_representation/multi_predictor/mean_ing_quantities.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:3b9e72db830283458a6db3df0dccbae95739f9c4bda94100044e7f89b6156f58
|
3 |
+
size 875
|
checkpoints/cocktail_representation/multi_predictor/min_when_present_ing_quantities.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:0eaa093684ddc7feb18ff84db3a9b783c04b05c44a321f731eb24d53818f7b72
|
3 |
+
size 875
|
checkpoints/cocktail_representation/multi_predictor/normalization_funcs.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:86ada5f5871c88400b5aeca915b2ebe57156754c458be289377109c8a713e3bd
|
3 |
+
size 1112
|
checkpoints/cocktail_representation/multi_predictor/params.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:7f748aa3e430ae5de8b9d934309d5c96b297c688759da63ed89e2a7b81533318
|
3 |
+
size 1857
|
checkpoints/cocktail_representation/multi_predictor/std_ing_quantities.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:d5aa46fe761e4e475e032ec547cf596c03e4a5bd4017ba930cee42bc599cc4d5
|
3 |
+
size 875
|
checkpoints/music2cocktails/affects2affect_cluster/cluster_model.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:9c19903b4bce0aae48abad4a2743afe3ca9e7b6e24a3d4ebf4c7259751a93585
|
3 |
+
size 3404
|
checkpoints/music2cocktails/music2affects/final_best_keys.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:100429e3fe3aa5814815624faf7220db37962ea7c498407e6b44fbe05ecc5670
|
3 |
+
size 331
|
checkpoints/music2cocktails/music2affects/max_handcoded_reps.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:178e37f23c0901e8a3353397c79804160385b6bb7be2522f469b69698676a81d
|
3 |
+
size 425
|
checkpoints/music2cocktails/music2affects/min_handcoded_reps.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:bb2f540dd4790f868c324d7f3b71c412fad6f9593f66c74f2b6c319c865cc63f
|
3 |
+
size 425
|
checkpoints/music2cocktails/music2affects/music2affect_models.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:4397f2bfb54df47864e202fa3681cca2c2dbeee3b83e303329bfb0d1bf8708c2
|
3 |
+
size 1335
|
checkpoints/music2cocktails/music2affects/top_10_keys.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:e7116349d7b415f0c78e36c33d20389e7c7913acbc1b3bd5fe4a292031154308
|
3 |
+
size 466
|
checkpoints/music2cocktails/music2flavor/b256_r128_classif001_ld40_meanstd_regground2.5_egg_bubbles/params.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:ddb8c49d6449c3b4bfacd6595f53ff115d330edd85b41abfbdb7eb08c0fce3d1
|
3 |
+
size 8218
|
checkpoints/music2cocktails/music2flavor/b256_r128_classif001_ld40_meanstd_regground2.5_egg_bubbles/results.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:dd900beda6faa055b2b29b066261f18d29146a40b68a25e42c81117de57a469c
|
3 |
+
size 190680
|
checkpoints/music_representation/b256_r128_represented/all_reps_unnormalized_sample100.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:039a4b7604a13fcf183e3f2281a54796faae11444e6fd36d595e97d6653df8f1
|
3 |
+
size 217453
|
checkpoints/music_representation/b256_r128_represented/all_reps_unnormalized_sample1000.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:6132759881707cb5adc41b510a82a45da1b2a749d3b595b577dae617a740568f
|
3 |
+
size 2172255
|
checkpoints/music_representation/b256_r128_represented/all_reps_unnormalized_sample200.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:e51611311b19236443b294a9cc0b0b08864282f35afb98775e1d90b2510d6e98
|
3 |
+
size 434653
|
checkpoints/music_representation/b256_r128_represented/all_reps_unnormalized_sample2000.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:1d18462db945b39a14ca1c7d112ea2572d774a971d0f85efca6571ece5cd51f6
|
3 |
+
size 4344255
|
checkpoints/music_representation/b256_r128_represented/all_reps_unnormalized_sample500.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:91cde3222075cc1c01255f7fdb9eaacacc5a57d60593857960ae8a058ff9e048
|
3 |
+
size 1086255
|
checkpoints/music_representation/b256_r128_represented/all_reps_unnormalized_sample5000.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:84c05e355fba5dd19ee2fe0bba9c762567025779bc9e9410fef60b60fcda5505
|
3 |
+
size 10860255
|
checkpoints/music_representation/b256_r128_represented/music_reps_normalized_meanstd.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:dd92cda2e94b870978a91ee0d12824061fe0e4a1242d53f30fae81d5f2c2bce1
|
3 |
+
size 34834083
|
checkpoints/music_representation/b256_r128_represented/music_reps_unnormalized.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:3d510c52a4795ab4770453992ba9cec88009b0c0159b8d21817aee3c8064034d
|
3 |
+
size 33924060
|
checkpoints/music_representation/b256_r128_represented/nn_model.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:292eb82c7cd47b485c913660275da9471c2b9994f66c31000e8f987887cd23a4
|
3 |
+
size 33924424
|
checkpoints/music_representation/sentence_embedding/smallbert_b256_r128_1/acc.png
ADDED
Git LFS Details
|
checkpoints/music_representation/sentence_embedding/smallbert_b256_r128_1/best_model/1_Pooling/config.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:a37f83ada23e7887be6b88f4998927dbeac0038af301553c7cd5461413bf1a56
|
3 |
+
size 190
|
checkpoints/music_representation/sentence_embedding/smallbert_b256_r128_1/best_model/2_Dense/config.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f654288d4c31922c6534adef9ae1dfa5f556a0c7683d310c981b4a4f022944f8
|
3 |
+
size 114
|
checkpoints/music_representation/sentence_embedding/smallbert_b256_r128_1/best_model/2_Dense/pytorch_model.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:5d9c88f12b6c64f4ca2b6f7eb7287ffff6d30f4a283f9bc100e39be24f727ed5
|
3 |
+
size 394791
|