pycui's picture
Add RealChar deployment for HuggingFace (V0)
babeaf6
raw
history blame
18.4 kB
/**
* WebSocket Connection
* The client sends and receives messages through this WebSocket connection.
*/
const connectButton = document.getElementById('connect');
const disconnectButton = document.getElementById('disconnect');
const devicesContainer = document.getElementById('devices-container');
let socket;
let clientId = Math.floor(Math.random() * 10000000);
function connectSocket() {
chatWindow.value = "";
var clientId = Math.floor(Math.random() * 1010000);
var ws_scheme = window.location.protocol == "https:" ? "wss" : "ws";
var ws_path = ws_scheme + '://' + window.location.host + `/ws/${clientId}`;
socket = new WebSocket(ws_path);
socket.binaryType = 'arraybuffer';
socket.onopen = (event) => {
console.log("successfully connected");
connectMicrophone(audioDeviceSelection.value);
speechRecognition();
socket.send("web"); // select web as the platform
};
socket.onmessage = (event) => {
if (typeof event.data === 'string') {
const message = event.data;
if (message == '[end]\n') {
chatWindow.value += "\n\n";
chatWindow.scrollTop = chatWindow.scrollHeight;
} else if (message.startsWith('[+]')) {
// [+] indicates the transcription is done. stop playing audio
chatWindow.value += `\nYou> ${message}\n`;
stopAudioPlayback();
} else if (message.startsWith('[=]')) {
// [=] indicates the response is done
chatWindow.value += "\n\n";
chatWindow.scrollTop = chatWindow.scrollHeight;
} else if (message.startsWith('Select')) {
createCharacterGroups(message);
} else {
chatWindow.value += `${event.data}`;
chatWindow.scrollTop = chatWindow.scrollHeight;
// if user interrupts the previous response, should be able to play audios of new response
shouldPlayAudio=true;
}
} else { // binary data
if (!shouldPlayAudio) {
return;
}
audioQueue.push(event.data);
if (audioQueue.length === 1) {
playAudios();
}
}
};
socket.onerror = (error) => {
console.log(`WebSocket Error: ${error}`);
};
socket.onclose = (event) => {
console.log("Socket closed");
};
}
connectButton.addEventListener("click", function() {
connectButton.style.display = "none";
textContainer.textContent = "Select a character";
devicesContainer.style.display = "none";
connectSocket();
talkButton.style.display = 'flex';
textButton.style.display = 'flex';
});
disconnectButton.addEventListener("click", function() {
stopAudioPlayback();
if (radioGroupsCreated) {
destroyRadioGroups();
}
if (mediaRecorder) {
mediaRecorder.stop();
}
if (recognition) {
recognition.stop();
}
textContainer.textContent = "";
disconnectButton.style.display = "none";
playerContainer.style.display = "none";
stopCallButton.style.display = "none";
continueCallButton.style.display = "none";
messageButton.style.display = "none";
sendButton.style.display = "none";
messageInput.style.display = "none";
chatWindow.style.display = "none";
callButton.style.display = "none";
connectButton.style.display = "flex";
devicesContainer.style.display = "flex";
talkButton.disabled = true;
textButton.disabled = true;
chatWindow.value = "";
selectedCharacter = null;
characterSent = false;
callActive = false;
showRecordingStatus();
socket.close();
});
/**
* Devices
* Get the list of media devices
*/
const audioDeviceSelection = document.getElementById('audio-device-selection');
window.addEventListener("load", function() {
navigator.mediaDevices.enumerateDevices()
.then(function(devices) {
// Filter out the audio input devices
let audioInputDevices = devices.filter(function(device) {
return device.kind === 'audioinput';
});
// If there are no audio input devices, display an error and return
if (audioInputDevices.length === 0) {
console.log('No audio input devices found');
return;
}
// Add the audio input devices to the dropdown
audioInputDevices.forEach(function(device, index) {
let option = document.createElement('option');
option.value = device.deviceId;
option.textContent = device.label || `Microphone ${index + 1}`;
audioDeviceSelection.appendChild(option);
});
})
.catch(function(err) {
console.log('An error occurred: ' + err);
});
});
audioDeviceSelection.addEventListener('change', function(e) {
connectMicrophone(e.target.value);
});
/**
* Audio Recording and Transmission
* captures audio from the user's microphone, which is then sent over the
* WebSocket connection then sent over the WebSocket connection to the server
* when the recording stops.
*/
let mediaRecorder;
let chunks = [];
let finalTranscripts = [];
let debug = false;
let audioSent = false;
function connectMicrophone(deviceId) {
navigator.mediaDevices.getUserMedia({
audio: {
deviceId: deviceId ? {exact: deviceId} : undefined,
echoCancellation: true
}
}).then(function(stream) {
mediaRecorder = new MediaRecorder(stream);
mediaRecorder.ondataavailable = function(e) {
chunks.push(e.data);
}
mediaRecorder.onstart = function() {
console.log("recorder starts");
}
mediaRecorder.onstop = function(e) {
console.log("recorder stops");
let blob = new Blob(chunks, {'type' : 'audio/webm'});
chunks = [];
if (debug) {
// Save the audio
let url = URL.createObjectURL(blob);
let a = document.createElement("a");
document.body.appendChild(a);
a.style = "display: none";
a.href = url;
a.download = 'test.webm';
a.click();
}
if (socket && socket.readyState === WebSocket.OPEN) {
if (!audioSent && callActive) {
console.log("sending audio");
socket.send(blob);
}
audioSent = false;
if (callActive) {
mediaRecorder.start();
}
}
}
})
.catch(function(err) {
console.log('An error occurred: ' + err);
});
}
/**
* Speech Recognition
* listens for when the user's speech ends and stops the recording.
*/
let recognition;
let onresultTimeout;
let onspeechTimeout;
let confidence;
function speechRecognition() {
// Initialize SpeechRecognition
window.SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
recognition = new SpeechRecognition();
recognition.interimResults = true;
recognition.maxAlternatives = 1;
recognition.continuous = true;
recognition.onstart = function() {
console.log("recognition starts");
}
recognition.onresult = function(event) {
// Clear the timeout if a result is received
clearTimeout(onresultTimeout);
clearTimeout(onspeechTimeout);
stopAudioPlayback()
const result = event.results[event.results.length - 1];
const transcriptObj = result[0];
const transcript = transcriptObj.transcript;
const ifFinal = result.isFinal;
if (ifFinal) {
console.log(`final transcript: {${transcript}}`);
finalTranscripts.push(transcript);
confidence = transcriptObj.confidence;
socket.send(`[&]${transcript}`);
} else {
console.log(`interim transcript: {${transcript}}`);
}
// Set a new timeout
onresultTimeout = setTimeout(() => {
if (ifFinal) {
return;
}
// If the timeout is reached, send the interim transcript
console.log(`TIMEOUT: interim transcript: {${transcript}}`);
socket.send(`[&]${transcript}`);
}, 500); // 500 ms
onspeechTimeout = setTimeout(() => {
recognition.stop();
}, 2000); // 2 seconds
}
recognition.onspeechend = function() {
console.log("speech ends");
if (socket && socket.readyState === WebSocket.OPEN){
audioSent = true;
mediaRecorder.stop();
if (confidence > 0.8 && finalTranscripts.length > 0) {
console.log("send final transcript");
let message = finalTranscripts.join(' ');
socket.send(message);
chatWindow.value += `\nYou> ${message}\n`;
chatWindow.scrollTop = chatWindow.scrollHeight;
shouldPlayAudio = true;
}
}
finalTranscripts = [];
};
recognition.onend = function() {
console.log("recognition ends");
if (socket && socket.readyState === WebSocket.OPEN && callActive){
recognition.start();
}
};
}
/**
* Voice-based Chatting
* allows users to start a voice chat.
*/
const talkButton = document.getElementById('talk-btn');
const textButton = document.getElementById('text-btn');
const callButton = document.getElementById('call');
const textContainer = document.querySelector('.header p');
const playerContainer = document.getElementById('player-container');
const soundWave = document.getElementById('sound-wave');
const stopCallButton = document.getElementById('stop-call');
const continueCallButton = document.getElementById('continue-call');
let callActive = false;
callButton.addEventListener("click", () => {
playerContainer.style.display = 'flex';
chatWindow.style.display = 'none';
sendButton.style.display = 'none';
messageInput.style.display = "none";
callButton.style.display = "none";
messageButton.style.display = 'flex';
if (callActive) {
stopCallButton.style.display = 'flex';
soundWave.style.display = 'flex';
} else {
continueCallButton.style.display = 'flex';
}
showRecordingStatus();
});
stopCallButton.addEventListener("click", () => {
soundWave.style.display = "none";
stopCallButton.style.display = "none";
continueCallButton.style.display = "flex";
callActive = false;
mediaRecorder.stop();
recognition.stop();
stopAudioPlayback();
showRecordingStatus();
})
continueCallButton.addEventListener("click", () => {
stopCallButton.style.display = "flex";
continueCallButton.style.display = "none";
soundWave.style.display = "flex";
mediaRecorder.start();
recognition.start();
callActive = true;
showRecordingStatus();
});
function showRecordingStatus() {
// show recording status
if (mediaRecorder.state == "recording") {
recordingStatus.style.display = "inline-block";
} else {
recordingStatus.style.display = "none";
}
}
talkButton.addEventListener("click", function() {
if (socket && socket.readyState === WebSocket.OPEN && mediaRecorder && selectedCharacter) {
playerContainer.style.display = "flex";
talkButton.style.display = "none";
textButton.style.display = 'none';
disconnectButton.style.display = "flex";
messageButton.style.display = "flex";
stopCallButton.style.display = "flex";
soundWave.style.display = "flex";
textContainer.textContent = "Hi, my friend, what brings you here today?";
shouldPlayAudio=true;
socket.send(selectedCharacter);
hideOtherCharacters();
mediaRecorder.start();
recognition.start();
callActive = true;
showRecordingStatus();
}
});
textButton.addEventListener("click", function() {
if (socket && socket.readyState === WebSocket.OPEN && mediaRecorder && selectedCharacter) {
messageButton.click();
disconnectButton.style.display = "flex";
textContainer.textContent = "";
shouldPlayAudio=true;
socket.send(selectedCharacter);
hideOtherCharacters();
showRecordingStatus();
}
});
function hideOtherCharacters() {
// Hide the radio buttons that are not selected
const radioButtons = document.querySelectorAll('.radio-buttons input[type="radio"]');
radioButtons.forEach(radioButton => {
if (radioButton.value != selectedCharacter) {
radioButton.parentElement.style.display = 'none';
}
});
}
/**
* Text-based Chatting
* allow users to send text-based messages through the WebSocket connection.
*/
const messageInput = document.getElementById('message-input');
const sendButton = document.getElementById('send-btn');
const messageButton = document.getElementById('message');
const chatWindow = document.getElementById('chat-window');
const recordingStatus = document.getElementById("recording");
let characterSent = false;
messageButton.addEventListener('click', function() {
playerContainer.style.display = 'none';
chatWindow.style.display = 'block';
talkButton.style.display = 'none';
textButton.style.display = 'none';
sendButton.style.display = 'block';
messageInput.style.display = "block";
callButton.style.display = "flex";
messageButton.style.display = 'none';
continueCallButton.style.display = 'none';
stopCallButton.style.display = 'none';
soundWave.style.display = "none";
showRecordingStatus();
});
const sendMessage = () => {
if (socket && socket.readyState === WebSocket.OPEN) {
const message = messageInput.value;
chatWindow.value += `\nYou> ${message}\n`;
chatWindow.scrollTop = chatWindow.scrollHeight;
socket.send(message);
messageInput.value = "";
if (isPlaying) {
stopAudioPlayback();
}
}
}
sendButton.addEventListener("click", sendMessage);
messageInput.addEventListener("keydown", (event) => {
if (event.key === "Enter") {
event.preventDefault();
sendMessage();
}
});
/**
* Character Selection
* parses the initial message from the server that asks the user to select a
* character for the chat. creates radio buttons for the character selection.
*/
let selectedCharacter;
let radioGroupsCreated = false;
function createCharacterGroups(message) {
const options = message.split('\n').slice(1);
// Create a map from character name to image URL
// TODO: store image in database and let server send the image url to client.
const imageMap = {
'Raiden Shogun And Ei': '/static/raiden.svg',
'Loki': '/static/loki.svg',
'Ai Character Helper': '/static/ai_helper.png',
'Reflection Pi': '/static/pi.jpeg',
'Elon Musk': '/static/elon.png',
'Bruce Wayne': '/static/bruce.png',
'Steve Jobs': '/static/jobs.png',
};
const radioButtonDiv = document.getElementsByClassName('radio-buttons')[0];
options.forEach(option => {
const match = option.match(/^(\d+)\s-\s(.+)$/);
if (match) {
const label = document.createElement('label');
label.className = 'custom-radio';
const input = document.createElement('input');
input.type = 'radio';
input.name = 'radio';
input.value = match[1]; // The option number is the value
const span = document.createElement('span');
span.className = 'radio-btn';
span.innerHTML = '<i class="las la-check"></i>';
const hobbiesIcon = document.createElement('div');
hobbiesIcon.className = 'hobbies-icon';
const img = document.createElement('img');
let src = imageMap[match[2]];
if (!src) {
src = '/static/realchar.svg';
}
img.src = src;
// Create a h3 element
const h3 = document.createElement('h4');
h3.textContent = match[2]; // The option name is the text
hobbiesIcon.appendChild(img);
hobbiesIcon.appendChild(h3);
span.appendChild(hobbiesIcon);
label.appendChild(input);
label.appendChild(span);
radioButtonDiv.appendChild(label);
}
});
radioButtonDiv.addEventListener('change', (event) => {
if (event.target.value != "") {
selectedCharacter = event.target.value;
}
talkButton.disabled = false;
textButton.disabled = false;
});
radioGroupsCreated = true;
}
function destroyRadioGroups() {
const radioButtonDiv = document.getElementsByClassName('radio-buttons')[0];
while (radioButtonDiv.firstChild) {
radioButtonDiv.removeChild(radioButtonDiv.firstChild);
}
selectedCharacter = null;
radioGroupsCreated = false;
}
// This function will add or remove the pulse animation
function togglePulseAnimation() {
const selectedRadioButton = document.querySelector('.custom-radio input:checked + .radio-btn');
if (isPlaying && selectedRadioButton) {
// Remove existing pulse animations
selectedRadioButton.classList.remove("pulse-animation-1");
selectedRadioButton.classList.remove("pulse-animation-2");
// Add a new pulse animation, randomly choosing between the two speeds
const animationClass = Math.random() > 0.5 ? "pulse-animation-1" : "pulse-animation-2";
selectedRadioButton.classList.add(animationClass);
} else if (selectedRadioButton) {
selectedRadioButton.classList.remove("pulse-animation-1");
selectedRadioButton.classList.remove("pulse-animation-2");
}
}
/**
* Audio Playback
* playing back audio received from the server.
*/
const audioPlayer = document.getElementById('audio-player')
let audioQueue = [];
let audioContext;
let shouldPlayAudio = false;
let isPlaying = false;
// Function to unlock the AudioContext
function unlockAudioContext(audioContext) {
if (audioContext.state === 'suspended') {
var unlock = function() {
audioContext.resume().then(function() {
document.body.removeEventListener('touchstart', unlock);
document.body.removeEventListener('touchend', unlock);
});
};
document.body.addEventListener('touchstart', unlock, false);
document.body.addEventListener('touchend', unlock, false);
}
}
async function playAudios() {
isPlaying = true;
togglePulseAnimation();
while (audioQueue.length > 0) {
let data = audioQueue[0];
let blob = new Blob([data], { type: 'audio/mp3' });
let audioUrl = URL.createObjectURL(blob);
await playAudio(audioUrl);
audioQueue.shift();
}
isPlaying = false;
togglePulseAnimation();
}
function playAudio(url) {
if (!audioContext) {
audioContext = new (window.AudioContext || window.webkitAudioContext)();
unlockAudioContext(audioContext);
}
if (!audioPlayer) {
audioPlayer = document.getElementById('audio-player');
}
return new Promise((resolve) => {
audioPlayer.src = url;
audioPlayer.muted = true; // Start muted
audioPlayer.play();
audioPlayer.onended = resolve;
audioPlayer.play().then(() => {
audioPlayer.muted = false; // Unmute after playback starts
}).catch(error => alert(`Playback failed because: ${error}`));
});
}
function stopAudioPlayback() {
if (audioPlayer) {
audioPlayer.pause();
shouldPlayAudio = false;
}
audioQueue = [];
isPlaying = false;
togglePulseAnimation();
}