File size: 7,061 Bytes
252d087
63c4bae
687e655
 
 
 
 
 
 
 
 
 
c9e9d08
9de3656
59f04fa
e83ff6f
687e655
9de3656
 
 
 
687e655
 
 
 
 
 
 
 
 
 
 
 
 
252d087
 
687e655
 
 
 
 
 
 
 
 
 
 
 
252d087
687e655
 
 
 
 
9fd29b1
 
687e655
252d087
687e655
 
252d087
687e655
 
252d087
687e655
 
252d087
687e655
428e1b4
69f6cc9
 
 
687e655
 
adb8651
 
 
687e655
 
 
 
 
 
69f6cc9
687e655
 
 
6fe43d7
1d1e6d2
687e655
 
 
 
 
41c7860
687e655
 
1d1e6d2
687e655
 
6fe43d7
1d1e6d2
687e655
 
 
 
 
 
 
 
 
 
 
 
1d1e6d2
69f6cc9
687e655
 
6fe43d7
9fd29b1
687e655
1d1e6d2
687e655
 
 
1d1e6d2
687e655
1d1e6d2
687e655
1d1e6d2
59f04fa
 
c9e9d08
 
 
 
 
 
 
 
 
 
 
 
 
 
59f04fa
 
 
264274f
 
ddf423d
 
 
 
264274f
 
 
 
 
 
 
 
ddf423d
 
 
59f04fa
264274f
59f04fa
 
 
 
 
 
 
264274f
59f04fa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
import numpy as numpy

import streamlit as st
import librosa
import soundfile as sf
import librosa.display
from config import CONFIG
import torch
from dataset import MaskGenerator
import onnxruntime, onnx
import matplotlib.pyplot as plt
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from pystoi import stoi
#from pesq import pesq
import pandas as pd
import torchaudio


from torch_pesq import PesqLoss


@st.cache
def load_model():
    path = 'lightning_logs/version_0/checkpoints/frn.onnx'
    onnx_model = onnx.load(path)
    options = onnxruntime.SessionOptions()
    options.intra_op_num_threads = 2
    options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
    session = onnxruntime.InferenceSession(path, options)
    input_names = [x.name for x in session.get_inputs()]
    output_names = [x.name for x in session.get_outputs()]
    return session, onnx_model, input_names, output_names

def inference(re_im, session, onnx_model, input_names, output_names):
    inputs = {input_names[i]: numpy.zeros([d.dim_value for d in _input.type.tensor_type.shape.dim],
                                       dtype=numpy.float32)
              for i, _input in enumerate(onnx_model.graph.input)
              }

    output_audio = []
    for t in range(re_im.shape[0]):
        inputs[input_names[0]] = re_im[t]
        out, prev_mag, predictor_state, mlp_state = session.run(output_names, inputs)
        inputs[input_names[1]] = prev_mag
        inputs[input_names[2]] = predictor_state
        inputs[input_names[3]] = mlp_state
        output_audio.append(out)

    output_audio = torch.tensor(numpy.concatenate(output_audio, 0))
    output_audio = output_audio.permute(1, 0, 2).contiguous()
    output_audio = torch.view_as_complex(output_audio)
    output_audio = torch.istft(output_audio, window, stride, window=hann)
    return output_audio.numpy()

def visualize(hr, lr, recon, sr):
    sr = sr
    window_size = 1024
    window = numpy.hanning(window_size)

    stft_hr = librosa.core.spectrum.stft(hr, n_fft=window_size, hop_length=512, window=window)
    stft_hr = 2 * numpy.abs(stft_hr) / numpy.sum(window)

    stft_lr = librosa.core.spectrum.stft(lr, n_fft=window_size, hop_length=512, window=window)
    stft_lr = 2 * numpy.abs(stft_lr) / numpy.sum(window)

    stft_recon = librosa.core.spectrum.stft(recon, n_fft=window_size, hop_length=512, window=window)
    stft_recon = 2 * numpy.abs(stft_recon) / numpy.sum(window)

    fig, (ax1, ax2, ax3) = plt.subplots(3, 1, sharey=True, sharex=True, figsize=(16, 12))
    ax1.title.set_text('Оригинальный сигнал')
    ax2.title.set_text('Сигнал с потерями')
    ax3.title.set_text('Улучшенный сигнал')

    canvas = FigureCanvas(fig)
    p = librosa.display.specshow(librosa.amplitude_to_db(stft_hr), ax=ax1, y_axis='log', x_axis='time', sr=sr)
    p = librosa.display.specshow(librosa.amplitude_to_db(stft_lr), ax=ax2, y_axis='log', x_axis='time', sr=sr)
    p = librosa.display.specshow(librosa.amplitude_to_db(stft_recon), ax=ax3, y_axis='log', x_axis='time', sr=sr)
    return fig

packet_size = CONFIG.DATA.EVAL.packet_size
window = CONFIG.DATA.window_size
stride = CONFIG.DATA.stride

title = 'Сокрытие потерь пакетов'
st.set_page_config(page_title=title, page_icon=":sound:")
st.title(title)

st.subheader('1. Загрузка аудио')
uploaded_file = st.file_uploader("Загрузите аудио формата (.wav) 48 КГц")

is_file_uploaded = uploaded_file is not None
if not is_file_uploaded:
    uploaded_file = 'sample.wav'

target, sr = librosa.load(uploaded_file)
target = target[:packet_size * (len(target) // packet_size)]

st.text('Ваше аудио')
st.audio(uploaded_file)

st.subheader('2. Выберите желаемый процент потерь')
slider = [st.slider("Ожидаемый процент потерь для генератора потерь цепи Маркова", 0, 100, step=1)]
loss_percent = float(slider[0])/100
mask_gen = MaskGenerator(is_train=False, probs=[(1 - loss_percent, loss_percent)])
lossy_input = target.copy().reshape(-1, packet_size)
mask = mask_gen.gen_mask(len(lossy_input), seed=0)[:, np.newaxis]
lossy_input *= mask
lossy_input = lossy_input.reshape(-1)
hann = torch.sqrt(torch.hann_window(window))
lossy_input_tensor = torch.tensor(lossy_input)
re_im = torch.stft(lossy_input_tensor, window, stride, window=hann, return_complex=False).permute(1, 0, 2).unsqueeze(
    1).numpy().astype(np.float32)
session, onnx_model, input_names, output_names = load_model()

if st.button('Сгенерировать потери'):
    with st.spinner('Ожидайте...'):
        output = inference(re_im, session, onnx_model, input_names, output_names)

        st.subheader('3. Визуализация')
        fig = visualize(target, lossy_input, output, sr)
        st.pyplot(fig)
    st.success('Сделано!')
    sf.write('target.wav', target, sr)
    sf.write('lossy.wav', lossy_input, sr)
    sf.write('enhanced.wav', output, sr)
    st.text('Оригинальное аудио')
    st.audio('target.wav')
    st.text('Аудио с потерями')
    st.audio('lossy.wav')
    st.text('Улучшенное аудио')
    st.audio('enhanced.wav')

    
    data_clean, samplerate = sf.read('target.wav')
    data_lossy, samplerate = sf.read('lossy.wav')
    data_enhanced, samplerate = sf.read('enhanced.wav')
    min_len = min(data_clean.shape[0], data_lossy.shape[0], data_enhanced.shape[0])
    data_clean = data_clean[:min_len]
    data_lossy = data_lossy[:min_len]
    data_enhanced = data_enhanced[:min_len]


    stoi_orig = round(stoi(data_clean, data_clean, samplerate, extended=False),5)
    stoi_lossy  = round(stoi(data_clean, data_lossy , samplerate, extended=False),5)
    stoi_enhanced = round(stoi(data_clean, data_enhanced, samplerate, extended=False),5)
    
    stoi_mass=[stoi_orig, stoi_lossy, stoi_enhanced]



    
    #if samplerate != 16000:
    #    data_lossy = librosa.resample(data_lossy, orig_sr=48000, target_sr=16000)
    #    data_clean = librosa.resample(data_clean, orig_sr=48000, target_sr=16000)
    #    data_enhanced = librosa.resample(data_enhanced, orig_sr=48000, target_sr=16000)
    #

    pesq = PesqLoss(0.5, sample_rate=48000)

    pesq_orig = pesq.mos(data_clean, data_clean)
    pesq_lossy = pesq.mos(data_clean, data_lossy)
    pesq_enhanced= pesq.mos(data_clean, data_enhanced)

    #pesq_orig = pesq(fs = 16000, ref = data_clean, deg = data_clean, mode='nb')
    #pesq_lossy = pesq(fs = 16000, ref = data_clean, deg = data_lossy, mode='nb')
    #pesq_enhanced = pesq(fs = 16000, ref = data_clean, deg = data_enhanced, mode='nb')

    psq_mas=[pesq_orig, pesq_lossy, pesq_enhanced]



    df = pd.DataFrame(columns=['Audio', 'PESQ', 'STOI', 'PLCMOS', 'LSD'])

    df['Audio'] = ['Clean', 'Lossy', 'Enhanced']

    df['PESQ'] = psq_mas

    df['STOI'] = stoi_mass

    st.table(df)