new file: SER_model/config.json

new file: SER_model/preprocessor_config.json
new file: SER_model/pytorch_model.bin
new file: SER_model/training_args.bin
new file: SPT_model/config.json
new file: SPT_model/preprocessor_config.json
new file: SPT_model/pytorch_model.bin
new file: SPT_model/special_tokens_map.json
new file: SPT_model/tokenizer_config.json
new file: SPT_model/vocab.json
new file: app.py
new file: flagged/audio/tmpqx7fc5l_.wav
new file: flagged/log.csv
new file: requirements.txt
new file: uploads/examples_03-01-01-01-01-02-05.wav

SER_model/config.json

@@ -0,0 +1,134 @@
+{
+    "_name_or_path": "./SPT_model",
+    "activation_dropout": 0.0,
+    "adapter_kernel_size": 3,
+    "adapter_stride": 2,
+    "add_adapter": false,
+    "apply_spec_augment": true,
+    "architectures": [
+      "Wav2Vec2ForSequenceClassification"
+    ],
+    "attention_dropout": 0.1,
+    "bos_token_id": 1,
+    "classifier_proj_size": 256,
+    "codevector_dim": 256,
+    "contrastive_logits_temperature": 0.1,
+    "conv_bias": false,
+    "conv_dim": [
+      512,
+      512,
+      512,
+      512,
+      512,
+      512,
+      512
+    ],
+    "conv_kernel": [
+      10,
+      3,
+      3,
+      3,
+      3,
+      2,
+      2
+    ],
+    "conv_stride": [
+      5,
+      2,
+      2,
+      2,
+      2,
+      2,
+      2
+    ],
+    "ctc_loss_reduction": "sum",
+    "ctc_zero_infinity": false,
+    "diversity_loss_weight": 0.1,
+    "do_stable_layer_norm": false,
+    "eos_token_id": 2,
+    "feat_extract_activation": "gelu",
+    "feat_extract_norm": "group",
+    "feat_proj_dropout": 0.1,
+    "feat_quantizer_dropout": 0.0,
+    "final_dropout": 0.0,
+    "freeze_feat_extract_train": true,
+    "hidden_act": "gelu",
+    "hidden_dropout": 0.1,
+    "hidden_size": 768,
+    "id2label": {
+      "0": "angry",
+      "1": "disgust",
+      "2": "fear",
+      "3": "happy",
+      "4": "neutral",
+      "5": "sad"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "label2id": {
+      "angry": "0",
+      "disgust": "1",
+      "fear": "2",
+      "happy": "3",
+      "neutral": "4",
+      "sad": "5"
+    },
+    "layer_norm_eps": 1e-05,
+    "layerdrop": 0.0,
+    "mask_channel_length": 10,
+    "mask_channel_min_space": 1,
+    "mask_channel_other": 0.0,
+    "mask_channel_prob": 0.0,
+    "mask_channel_selection": "static",
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_min_space": 1,
+    "mask_time_other": 0.0,
+    "mask_time_prob": 0.05,
+    "mask_time_selection": "static",
+    "model_type": "wav2vec2",
+    "no_mask_channel_overlap": false,
+    "no_mask_time_overlap": false,
+    "num_adapter_layers": 3,
+    "num_attention_heads": 12,
+    "num_codevector_groups": 2,
+    "num_codevectors_per_group": 320,
+    "num_conv_pos_embedding_groups": 16,
+    "num_conv_pos_embeddings": 128,
+    "num_feat_extract_layers": 7,
+    "num_hidden_layers": 12,
+    "num_negatives": 100,
+    "output_hidden_size": 768,
+    "pad_token_id": 0,
+    "proj_codevector_dim": 256,
+    "tdnn_dilation": [
+      1,
+      2,
+      3,
+      1,
+      1
+    ],
+    "tdnn_dim": [
+      512,
+      512,
+      512,
+      512,
+      1500
+    ],
+    "tdnn_kernel": [
+      5,
+      3,
+      3,
+      1,
+      1
+    ],
+    "torch_dtype": "float32",
+    "transformers_version": "4.25.1",
+    "use_weighted_layer_sum": false,
+    "vocab_size": 32,
+    "xvector_output_dim": 512
+  }
+  

SER_model/preprocessor_config.json

@@ -0,0 +1,9 @@
+{
+    "do_normalize": true,
+    "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+    "feature_size": 1,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": false,
+    "sampling_rate": 16000
+  }

SER_model/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:583aea2bf31f8713200e46bd9f1c6735beed40b5ec0d1f6e2decd45c6446e1b6
+size 378352659

SER_model/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c08295d554ee360c4ed5b8090035b4384199468399ca15720321255bf7bf7b3
+size 3503

SPT_model/config.json

@@ -0,0 +1,113 @@
+{
+    "activation_dropout": 0.1,
+    "adapter_kernel_size": 3,
+    "adapter_stride": 2,
+    "add_adapter": false,
+    "apply_spec_augment": true,
+    "architectures": [
+      "Wav2Vec2ConformerForCTC"
+    ],
+    "attention_dropout": 0.1,
+    "bos_token_id": 1,
+    "classifier_proj_size": 256,
+    "codevector_dim": 768,
+    "conformer_conv_dropout": 0.1,
+    "contrastive_logits_temperature": 0.1,
+    "conv_bias": true,
+    "conv_depthwise_kernel_size": 31,
+    "conv_dim": [
+      512,
+      512,
+      512,
+      512,
+      512,
+      512,
+      512
+    ],
+    "conv_kernel": [
+      10,
+      3,
+      3,
+      3,
+      3,
+      2,
+      2
+    ],
+    "conv_stride": [
+      5,
+      2,
+      2,
+      2,
+      2,
+      2,
+      2
+    ],
+    "ctc_loss_reduction": "sum",
+    "ctc_zero_infinity": false,
+    "diversity_loss_weight": 0.1,
+    "do_stable_layer_norm": true,
+    "eos_token_id": 2,
+    "feat_extract_activation": "gelu",
+    "feat_extract_dropout": 0.0,
+    "feat_extract_norm": "layer",
+    "feat_proj_dropout": 0.1,
+    "feat_quantizer_dropout": 0.0,
+    "final_dropout": 0.1,
+    "gradient_checkpointing": false,
+    "hidden_act": "swish",
+    "hidden_dropout": 0.1,
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "layer_norm_eps": 1e-05,
+    "layerdrop": 0.0,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_source_positions": 5000,
+    "model_type": "wav2vec2-conformer",
+    "num_adapter_layers": 3,
+    "num_attention_heads": 16,
+    "num_codevector_groups": 2,
+    "num_codevectors_per_group": 320,
+    "num_conv_pos_embedding_groups": 16,
+    "num_conv_pos_embeddings": 128,
+    "num_feat_extract_layers": 7,
+    "num_hidden_layers": 24,
+    "num_negatives": 100,
+    "output_hidden_size": 1024,
+    "pad_token_id": 0,
+    "position_embeddings_type": "relative",
+    "proj_codevector_dim": 768,
+    "rotary_embedding_base": 10000,
+    "tdnn_dilation": [
+      1,
+      2,
+      3,
+      1,
+      1
+    ],
+    "tdnn_dim": [
+      512,
+      512,
+      512,
+      512,
+      1500
+    ],
+    "tdnn_kernel": [
+      5,
+      3,
+      3,
+      1,
+      1
+    ],
+    "torch_dtype": "float32",
+    "transformers_version": "4.19.0.dev0",
+    "use_weighted_layer_sum": false,
+    "vocab_size": 32,
+    "xvector_output_dim": 512
+  }

SPT_model/preprocessor_config.json

@@ -0,0 +1,10 @@
+{
+  "do_normalize": true,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0,
+  "processor_class": "Wav2Vec2Processor",
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}

SPT_model/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f8342d778e99445a2a3923b248235e2cb8c2b85ac7574d00495d2329b0fe4b6
+size 2474878081

SPT_model/special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}

SPT_model/tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "do_lower_case": false, "word_delimiter_token": "|", "replace_word_delimiter_char": " ", "tokenizer_class": "Wav2Vec2CTCTokenizer", "processor_class": "Wav2Vec2Processor"}

SPT_model/vocab.json

@@ -0,0 +1 @@
+{"<s>": 1, "<pad>": 0, "</s>": 2, "<unk>": 3, "|": 4, "E": 5, "T": 6, "A": 7, "O": 8, "N": 9, "I": 10, "H": 11, "S": 12, "R": 13, "D": 14, "L": 15, "U": 16, "M": 17, "W": 18, "C": 19, "F": 20, "G": 21, "Y": 22, "P": 23, "B": 24, "V": 25, "K": 26, "'": 27, "X": 28, "J": 29, "Q": 30, "Z": 31}

app.py

@@ -0,0 +1,95 @@
+import os
+from flask import Flask, request, jsonify, flash, redirect, url_for
+import torch
+import torch.nn.functional as F
+import torchaudio
+from transformers import AutoConfig, Wav2Vec2FeatureExtractor, Wav2Vec2ForSequenceClassification, Wav2Vec2Processor, Wav2Vec2ConformerForCTC
+import librosa
+import jellyfish
+from werkzeug.utils import secure_filename
+import gradio as gr
+
+
+def speech_file_to_array_fn(path, sampling_rate):
+    speech_array, _sampling_rate = torchaudio.load(path)
+    resampler = torchaudio.transforms.Resample(_sampling_rate)
+    speech = resampler(speech_array).squeeze().numpy()
+    return speech
+
+def predict(path, sampling_rate, feature_extractor, device, model, config):
+    speech = speech_file_to_array_fn(path, sampling_rate)
+    inputs = feature_extractor(speech, sampling_rate=sampling_rate, return_tensors="pt", padding=True)
+    inputs = {key: inputs[key].to(device) for key in inputs}
+    with torch.no_grad():
+        logits = model(**inputs).logits
+    scores = F.softmax(logits, dim=1).detach().cpu().numpy()[0]
+    outputs = [{"Emotion": config.id2label[i], "Score": f"{round(score * 100, 3):.1f}%"} for i, score in enumerate(scores)]
+    return outputs
+
+def get_speech_to_text(model, processor, audio_path):
+    data, sample_rate = librosa.load(audio_path, sr=16000)
+    input_values = processor(data, return_tensors="pt", padding="longest").input_values
+    logits = model(input_values).logits
+    predicted_ids = torch.argmax(logits, dim=-1)
+    transcription = processor.batch_decode(predicted_ids)
+    return transcription
+
+# def get_percentage_match(transcription, text):
+#     return jellyfish.damerau_levenshtein_distance(transcription, text)
+
+def get_sos_status(transcription, key_phrase):
+    ct = 0
+    for words in key_phrase.split(" "):
+        # print(type(words))
+        if transcription[0].find(words) != -1:
+            ct = ct + 1
+    if ct == 3:
+        sos = 1
+    else:
+        sos = 0
+    return sos
+
+def main(audio):
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    SPT_MODEL = "D:\kaggle_practice\KJSCE_hack\SERModel\SPT_model"
+    model_name_or_path = "D:\kaggle_practice\KJSCE_hack\SERModel\SER_model"
+    config = AutoConfig.from_pretrained(model_name_or_path)
+    feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(model_name_or_path)
+    sampling_rate = feature_extractor.sampling_rate
+    model = Wav2Vec2ForSequenceClassification.from_pretrained(model_name_or_path).to(device)
+    processor = Wav2Vec2Processor.from_pretrained(SPT_MODEL)
+    model_SPT = Wav2Vec2ConformerForCTC.from_pretrained(SPT_MODEL)
+    # path = r'testing_audios\03-01-06-02-02-01-01.wav'
+    outputs = predict(audio, sampling_rate, feature_extractor, device = device, model = model, config = config)
+    transcription = get_speech_to_text(model_SPT, processor, audio_path=audio)
+    key_phrase = "DOGS DOOR SITTING"
+    status = get_sos_status(transcription, key_phrase)
+    max_score = 0
+    emotion = ""
+    for i in outputs:
+        if float(i['Score'][:-1]) > max_score:
+            max_score = float(i['Score'][:-1])
+            emotion = i['Emotion']
+    if emotion in ['disgust', 'fear', 'sadness']:
+        emotion = 'negative'
+    elif emotion == 'neutral':
+        emotion = 'neutral'
+    else:
+        emotion = 'positive'
+    
+    if emotion == 'negative' or status == 1:
+        sos = 1
+    else:
+        sos = 0
+
+    return [emotion, transcription, sos]
+
+gr.Interface(
+    fn=main, 
+    inputs=[
+        gr.inputs.Audio(source="upload", type="filepath")
+    ],
+    outputs=[
+        "textbox"
+    ],
+    live=True).launch(server_port=5000)

flagged/log.csv

@@ -0,0 +1,2 @@
+audio,output,flag,username,timestamp
+D:\kaggle_practice\KJSCE_hack\SERModel\flagged\audio\tmpqx7fc5l_.wav,,,,2023-04-09 15:47:42.154241

requirements.txt

@@ -0,0 +1,9 @@
+flask
+torch==1.13.1 
+torchvision==0.14.1 
+torchaudio==0.13.1
+transformers
+librosa 
+jellyfish
+werkzeug
+panel