File size: 39,841 Bytes
1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f a1ab948 1873a9f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 |
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## All package installation and libraries imports\n",
"### Packages installation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"id": "rd5vZMt_2wrC"
},
"outputs": [],
"source": [
"#run this cell \n",
"!pip install accelerate\n",
"!pip install bitsandbytes\n",
"!pip install optimum\n",
"!pip install auto-gptq\n",
"!pip install gradio\n",
"\n",
"#text-to-speech and speech to text\n",
"!pip install TTS\n",
"!pip install 'transformers == 4.36'\n",
"!pip install numpy\n",
"!pip install openai-whisper #Whisper models\n",
"\n",
"!pip install geopy\n",
"\n",
"!pip uninstall transformer-engine -y\n",
"\n",
"\n",
"!pip install langchain\n",
"!pip install text_generation"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### libraries import"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"id": "oOnNfKjX4IAV"
},
"outputs": [],
"source": [
"#gradio interface\n",
"import gradio as gr\n",
"\n",
"from transformers import AutoModelForCausalLM,AutoTokenizer\n",
"import torch\n",
"\n",
"#STT (speech to text)\n",
"from transformers import WhisperProcessor, WhisperForConditionalGeneration\n",
"import librosa\n",
"\n",
"#TTS (text to speech)\n",
"import torch\n",
"from TTS.api import TTS\n",
"from IPython.display import Audio\n",
"\n",
"#json request for APIs\n",
"import requests\n",
"import json\n",
"\n",
"#regular expressions\n",
"import re\n",
"\n",
"#langchain and function calling\n",
"from typing import List, Literal, Union\n",
"import requests\n",
"from functools import partial\n",
"from geopy.geocoders import Nominatim\n",
"import math\n",
"\n",
"\n",
"#langchain, not used anymore since I had to find another way fast to stop using the endpoint, but could be interesting to reuse \n",
"from langchain.tools.base import StructuredTool\n",
"from langchain.agents import (\n",
" Tool,\n",
" AgentExecutor,\n",
" LLMSingleActionAgent,\n",
" AgentOutputParser,\n",
")\n",
"from langchain.schema import AgentAction, AgentFinish, OutputParserException\n",
"from langchain.prompts import StringPromptTemplate\n",
"from langchain.llms import HuggingFaceTextGenInference\n",
"from langchain.chains import LLMChain\n",
"\n",
"\n",
"\n",
"from datetime import datetime, timedelta, timezone\n",
"from transformers import pipeline\n",
"import inspect"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Models loads"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"id": "JNALTDb0LT90"
},
"outputs": [],
"source": [
"# load model and processor for speech-to-text\n",
"processor = WhisperProcessor.from_pretrained(\"openai/whisper-small\")\n",
"modelw = WhisperForConditionalGeneration.from_pretrained(\"openai/whisper-small\")\n",
"modelw.config.forced_decoder_ids = None\n",
"\n",
"#load model for text to speech\n",
"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
"tts = TTS(\"tts_models/multilingual/multi-dataset/xtts_v1.1\").to(device)\n",
"\n",
"#load model language recognition\n",
"model_ckpt = \"papluca/xlm-roberta-base-language-detection\"\n",
"pipe_language = pipeline(\"text-classification\", model=model_ckpt)\n",
"\n",
"#load model llama2\n",
"mn = 'stabilityai/StableBeluga-7B' #mn = \"TheBloke/Llama-2-7b-Chat-GPTQ\" --> other possibility \n",
"model = AutoModelForCausalLM.from_pretrained(mn, device_map=0, load_in_4bit=True) #torch_dtype=torch.float16\n",
"tokr = AutoTokenizer.from_pretrained(mn, load_in_4bit=True) #tokenizer\n",
"\n",
"#NexusRaven for function calling\n",
"model_id = \"Nexusflow/NexusRaven-13B\"\n",
"tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
"modelNexus = AutoModelForCausalLM.from_pretrained(model_id, device_map=0, load_in_4bit=True)\n",
"pipe = pipeline(\"text-generation\", model=modelNexus, tokenizer = tokenizer)\n",
"\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Function calling with NexusRaven "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#FUNCTION CALLING \n",
"\n",
"#API keys\n",
"TOMTOM_KEY= \"your_key\" \n",
"WHEATHER_API_KEY = \"your_key\" \n",
"\n",
"##########################################################\n",
"# Step 1: Define the functions you want to articulate. ###\n",
"##########################################################\n",
"\n",
"########################################################################################\n",
"# Functions called in the articulated functions (not directly called by the model): ###\n",
"########################################################################################\n",
"\n",
"geolocator = Nominatim(user_agent=\"MyApp\")\n",
"\n",
"def find_precise_place(lat, lon):\n",
" location = geolocator.reverse(str(lat) +\", \" + str(lon))\n",
" return location.raw.get('display_name', {})\n",
"\n",
"def find_coordinates(address):\n",
" coord = geolocator.geocode(address)\n",
" lat = coord.latitude\n",
" lon = coord.longitude\n",
" return(lat,lon)\n",
"\n",
"\n",
"def check_city_coordinates(lat = \"\", lon = \"\", city = \"\", **kwargs):\n",
" \"\"\"\n",
" :param lat: latitude\n",
" :param lon: longitude\n",
" :param city: name of the city\n",
"\n",
" Checks if the coordinates correspond to the city, if not update the coordinate to correspond to the city\n",
" \"\"\"\n",
" if lat != \"0\" and lon != \"0\":\n",
" reverse = partial(geolocator.reverse, language=\"en\")\n",
" location = reverse(f\"{lat}, {lon}\")\n",
" address = location.raw.get('address', {})\n",
" city = address.get('city') or address.get('town') or address.get('village') or address.get('county')\n",
" else : \n",
" reverse = partial(geolocator.reverse, language=\"en\")\n",
" location = reverse(f\"{lat}, {lon}\")\n",
" address = location.raw.get('address', {})\n",
" city_name = address.get('city') or address.get('town') or address.get('village') or address.get('county')\n",
" if city_name is None :\n",
" city_name = 'not_found'\n",
" print(city_name)\n",
" if city_name.lower() != city.lower():\n",
" coord = geolocator.geocode(city )\n",
" if coord is None:\n",
" coord = geolocator.geocode(city)\n",
" lat = coord.latitude\n",
" lon = coord.longitude\n",
" return lat, lon, city\n",
"\n",
"# Select coordinates at equal distance, including the last one\n",
"def select_equally_spaced_coordinates(coords, number_of_points=10):\n",
" n = len(coords)\n",
" selected_coords = []\n",
" interval = max((n - 1) / (number_of_points - 1), 1)\n",
" for i in range(number_of_points):\n",
" # Calculate the index, ensuring it doesn't exceed the bounds of the list\n",
" index = int(round(i * interval))\n",
" if index < n:\n",
" selected_coords.append(coords[index])\n",
" return selected_coords\n",
"\n",
"###################################################\n",
"# Functions we want to articulate (APIs calls): ###\n",
"###################################################\n",
"\n",
"def search_along_route(latitude_depart, longitude_depart, city_destination, type_of_poi):\n",
" \"\"\"\n",
" Return some of the closest points of interest along the route from the depart point, specified by its coordinates and a city destination.\n",
" :param latitude_depart (string): Required. Latitude of depart location\n",
" :param longitude_depart (string): Required. Longitude of depart location\n",
" :param city_destination (string): Required. City destination\n",
" :param type_of_poi (string): Required. type of point of interest depending on what the user wants to do.\n",
" \"\"\"\n",
" \n",
" lat_dest, lon_dest = find_coordinates(city_destination)\n",
" print(lat_dest)\n",
" \n",
" r = requests.get('https://api.tomtom.com/routing/1/calculateRoute/{0},{1}:{2},{3}/json?key={4}'.format(\n",
" latitude_depart,\n",
" longitude_depart,\n",
" lat_dest,\n",
" lon_dest,\n",
" TOMTOM_KEY\n",
" ))\n",
" \n",
" coord_route = select_equally_spaced_coordinates(r.json()['routes'][0]['legs'][0]['points'])\n",
"\n",
" # The API endpoint for searching along a route\n",
" url = f'https://api.tomtom.com/search/2/searchAlongRoute/{type_of_poi}.json?key={TOMTOM_KEY}&maxDetourTime=700&limit=20&sortBy=detourTime'\n",
"\n",
" # The data payload\n",
" payload = {\n",
" \"route\": {\n",
" \"points\": [\n",
" {\"lat\": float(latitude_depart), \"lon\": float(longitude_depart)},\n",
" {\"lat\": float(coord_route[1]['latitude']), \"lon\": float(coord_route[1]['longitude'])},\n",
" {\"lat\": float(coord_route[2]['latitude']), \"lon\": float(coord_route[2]['longitude'])},\n",
" {\"lat\": float(coord_route[3]['latitude']), \"lon\": float(coord_route[3]['longitude'])},\n",
" {\"lat\": float(coord_route[4]['latitude']), \"lon\": float(coord_route[4]['longitude'])},\n",
" {\"lat\": float(coord_route[5]['latitude']), \"lon\": float(coord_route[5]['longitude'])},\n",
" {\"lat\": float(coord_route[6]['latitude']), \"lon\": float(coord_route[6]['longitude'])},\n",
" {\"lat\": float(coord_route[7]['latitude']), \"lon\": float(coord_route[7]['longitude'])},\n",
" {\"lat\": float(coord_route[8]['latitude']), \"lon\": float(coord_route[8]['longitude'])},\n",
" {\"lat\": float(lat_dest), \"lon\": float(lon_dest)},\n",
" ]\n",
" }\n",
" }\n",
"\n",
" # Make the POST request\n",
" response = requests.post(url, json=payload)\n",
"\n",
" # Check if the request was successful\n",
" if response.status_code == 200:\n",
" # Parse the JSON response\n",
" data = response.json()\n",
" print(json.dumps(data, indent=4))\n",
" else:\n",
" print('Failed to retrieve data:', response.status_code)\n",
" answer = \"\"\n",
" for result in data['results']:\n",
" name = result['poi']['name']\n",
" address = result['address']['freeformAddress']\n",
" detour_time = result['detourTime']\n",
" answer = answer + f\" \\nAlong the route to {city_destination}, there is the {name} at {address} that would represent a detour of {int(detour_time/60)} minutes.\"\n",
" \n",
" return answer\n",
"\n",
"\n",
"def find_points_of_interest(lat=\"0\", lon=\"0\", city=\"\", type_of_poi=\"restaurant\", **kwargs):\n",
" \"\"\"\n",
" Return some of the closest points of interest for a specific location and type of point of interest. The more parameters there are, the more precise.\n",
" :param lat (string): latitude\n",
" :param lon (string): longitude\n",
" :param city (string): Required. city\n",
" :param type_of_poi (string): Required. type of point of interest depending on what the user wants to do.\n",
" \"\"\"\n",
" lat, lon, city = check_city_coordinates(lat,lon,city)\n",
"\n",
" r = requests.get(f'https://api.tomtom.com/search/2/search/{type_of_poi}'\n",
" '.json?key={0}&lat={1}&lon={2}&radius=10000&idxSet=POI&limit=100'.format(\n",
" TOMTOM_KEY,\n",
" lat,\n",
" lon\n",
" ))\n",
"\n",
" # Parse JSON from the response\n",
" data = r.json()\n",
" #print(data)\n",
" # Extract results\n",
" results = data['results']\n",
"\n",
" # Sort the results based on distance\n",
" sorted_results = sorted(results, key=lambda x: x['dist'])\n",
" #print(sorted_results)\n",
"\n",
" # Format and limit to top 5 results\n",
" formatted_results = [\n",
" f\"The {type_of_poi} {result['poi']['name']} is {int(result['dist'])} meters away\"\n",
" for result in sorted_results[:5]\n",
" ]\n",
"\n",
"\n",
" return \". \".join(formatted_results)\n",
"\n",
"def find_route(lat_depart=\"0\", lon_depart=\"0\", city_depart=\"\", address_destination=\"\", depart_time =\"\", **kwargs):\n",
" \"\"\"\n",
" Return the distance and the estimated time to go to a specific destination from the current place, at a specified depart time.\n",
" :param lat_depart (string): latitude of depart\n",
" :param lon_depart (string): longitude of depart\n",
" :param city_depart (string): Required. city of depart\n",
" :param address_destination (string): Required. The destination\n",
" :param depart_time (string): departure hour, in the format '08:00:20'.\n",
" \"\"\"\n",
" print(address_destination)\n",
" date = \"2025-03-29T\"\n",
" departure_time = '2024-02-01T' + depart_time\n",
" lat, lon, city = check_city_coordinates(lat_depart,lon_depart,city_depart)\n",
" lat_dest, lon_dest = find_coordinates(address_destination)\n",
" #print(lat_dest, lon_dest)\n",
" \n",
" #print(departure_time)\n",
"\n",
" r = requests.get('https://api.tomtom.com/routing/1/calculateRoute/{0},{1}:{2},{3}/json?key={4}&departAt={5}'.format(\n",
" lat_depart,\n",
" lon_depart,\n",
" lat_dest,\n",
" lon_dest,\n",
" TOMTOM_KEY,\n",
" departure_time\n",
" ))\n",
"\n",
" # Parse JSON from the response\n",
" data = r.json()\n",
" #print(data)\n",
" \n",
" #print(data)\n",
" \n",
" result = data['routes'][0]['summary']\n",
"\n",
" # Calculate distance in kilometers (1 meter = 0.001 kilometers)\n",
" distance_km = result['lengthInMeters'] * 0.001\n",
"\n",
" # Calculate travel time in minutes (1 second = 1/60 minutes)\n",
" time_minutes = result['travelTimeInSeconds'] / 60\n",
" if time_minutes < 60:\n",
" time_display = f\"{time_minutes:.0f} minutes\"\n",
" else:\n",
" hours = int(time_minutes / 60)\n",
" minutes = int(time_minutes % 60)\n",
" time_display = f\"{hours} hours\" + (f\" and {minutes} minutes\" if minutes > 0 else \"\")\n",
" \n",
" # Extract arrival time from the JSON structure\n",
" arrival_time_str = result['arrivalTime']\n",
"\n",
" # Convert string to datetime object\n",
" arrival_time = datetime.fromisoformat(arrival_time_str)\n",
"\n",
" # Extract and display the arrival hour in HH:MM format\n",
" arrival_hour_display = arrival_time.strftime(\"%H:%M\")\n",
"\n",
"\n",
" # return the distance and time\n",
" return(f\"The route to go to {address_destination} is {distance_km:.2f} km and {time_display}. Leaving now, the arrival time is estimated at {arrival_hour_display} \" )\n",
"\n",
" \n",
" # Sort the results based on distance\n",
" #sorted_results = sorted(results, key=lambda x: x['dist'])\n",
"\n",
" #return \". \".join(formatted_results)\n",
"\n",
"#current weather API\n",
"def get_weather(city_name:str= \"\", **kwargs):\n",
" \"\"\"\n",
" Returns the CURRENT weather in a specified city.\n",
" Args:\n",
" city_name (string) : Required. The name of the city.\n",
" \"\"\"\n",
" # The endpoint URL provided by WeatherAPI\n",
" url = f\"http://api.weatherapi.com/v1/current.json?key={WEATHER_API_KEY}&q={city_name}&aqi=no\"\n",
"\n",
" # Make the API request\n",
" response = requests.get(url)\n",
"\n",
" if response.status_code == 200:\n",
" # Parse the JSON response\n",
" weather_data = response.json()\n",
"\n",
" # Extracting the necessary pieces of data\n",
" location = weather_data['location']['name']\n",
" region = weather_data['location']['region']\n",
" country = weather_data['location']['country']\n",
" time = weather_data['location']['localtime']\n",
" temperature_c = weather_data['current']['temp_c']\n",
" condition_text = weather_data['current']['condition']['text']\n",
" wind_mph = weather_data['current']['wind_mph']\n",
" humidity = weather_data['current']['humidity']\n",
" feelslike_c = weather_data['current']['feelslike_c']\n",
"\n",
" # Formulate the sentences\n",
" weather_sentences = (\n",
" f\"The current weather in {location}, {region}, {country} is {condition_text} \"\n",
" f\"with a temperature of {temperature_c}°C that feels like {feelslike_c}°C. \"\n",
" f\"Humidity is at {humidity}%. \"\n",
" f\"Wind speed is {wind_mph} mph.\"\n",
" )\n",
" return weather_sentences\n",
" else:\n",
" # Handle errors\n",
" return f\"Failed to get weather data: {response.status_code}, {response.text}\"\n",
" \n",
"#forecast API\n",
"def get_forecast(city_name:str= \"\", when = 0, **kwargs):\n",
" \"\"\"\n",
" Returns the weather forecast in a specified number of days for a specified city .\n",
" Args:\n",
" city_name (string) : Required. The name of the city.\n",
" when (int) : Required. in number of days (until the day for which we want to know the forecast) (example: tomorrow is 1, in two days is 2, etc.)\n",
" \"\"\"\n",
" #print(when)\n",
" when +=1\n",
" # The endpoint URL provided by WeatherAPI\n",
" url = f\"http://api.weatherapi.com/v1/forecast.json?key={WEATHER_API_KEY}&q={city_name}&days={str(when)}&aqi=no\"\n",
"\n",
"\n",
" # Make the API request\n",
" response = requests.get(url)\n",
"\n",
" if response.status_code == 200:\n",
" # Parse the JSON response\n",
" data = response.json()\n",
" \n",
" # Initialize an empty string to hold our result\n",
" forecast_sentences = \"\"\n",
"\n",
" # Extract city information\n",
" location = data.get('location', {})\n",
" city_name = location.get('name', 'the specified location')\n",
" \n",
" #print(data)\n",
" \n",
"\n",
" # Extract the forecast days\n",
" forecast_days = data.get('forecast', {}).get('forecastday', [])[when-1:]\n",
" #number = 0\n",
" \n",
" #print (forecast_days)\n",
"\n",
" for day in forecast_days:\n",
" date = day.get('date', 'a specific day')\n",
" conditions = day.get('day', {}).get('condition', {}).get('text', 'weather conditions')\n",
" max_temp_c = day.get('day', {}).get('maxtemp_c', 'N/A')\n",
" min_temp_c = day.get('day', {}).get('mintemp_c', 'N/A')\n",
" chance_of_rain = day.get('day', {}).get('daily_chance_of_rain', 'N/A')\n",
" \n",
" if when == 1:\n",
" number_str = 'today'\n",
" elif when == 2:\n",
" number_str = 'tomorrow'\n",
" else:\n",
" number_str = f'in {when-1} days'\n",
"\n",
" # Generate a sentence for the day's forecast\n",
" forecast_sentence = f\"On {date} ({number_str}) in {city_name}, the weather will be {conditions} with a high of {max_temp_c}°C and a low of {min_temp_c}°C. There's a {chance_of_rain}% chance of rain. \"\n",
" \n",
" #number = number + 1\n",
" # Add the sentence to the result\n",
" forecast_sentences += forecast_sentence\n",
" return forecast_sentences\n",
" else:\n",
" # Handle errors\n",
" print( f\"Failed to get weather data: {response.status_code}, {response.text}\")\n",
" return f'error {response.status_code}'\n",
"\n",
"\n",
"#############################################################\n",
"# Step 2: Let's define some utils for building the prompt ###\n",
"#############################################################\n",
"\n",
"\n",
"def format_functions_for_prompt(*functions):\n",
" formatted_functions = []\n",
" for func in functions:\n",
" source_code = inspect.getsource(func)\n",
" docstring = inspect.getdoc(func)\n",
" formatted_functions.append(\n",
" f\"OPTION:\\n<func_start>{source_code}<func_end>\\n<docstring_start>\\n{docstring}\\n<docstring_end>\"\n",
" )\n",
" return \"\\n\".join(formatted_functions)\n",
"\n",
"\n",
"##############################\n",
"# Step 3: Construct Prompt ###\n",
"##############################\n",
"\n",
"\n",
"def construct_prompt(user_query: str, context):\n",
" formatted_prompt = format_functions_for_prompt(get_weather, find_points_of_interest, find_route, get_forecast, search_along_route)\n",
" formatted_prompt += f'\\n\\nContext : {context}'\n",
" formatted_prompt += f\"\\n\\nUser Query: Question: {user_query}\\n\"\n",
"\n",
" prompt = (\n",
" \"<human>:\\n\"\n",
" + formatted_prompt\n",
" + \"Please pick a function from the above options that best answers the user query and fill in the appropriate arguments.<human_end>\"\n",
" )\n",
" return prompt\n",
"\n",
"#######################################\n",
"# Step 4: Execute the function call ###\n",
"#######################################\n",
"\n",
"\n",
"def execute_function_call(model_output):\n",
" # Ignore everything after \"Reflection\" since that is not essential.\n",
" function_call = (\n",
" model_output[0][\"generated_text\"]\n",
" .strip()\n",
" .split(\"\\n\")[1]\n",
" .replace(\"Initial Answer:\", \"\")\n",
" .strip()\n",
" )\n",
"\n",
" try:\n",
" return eval(function_call)\n",
" except Exception as e:\n",
" return str(e)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# might be deleted\n",
"# Compute a Simple equation\n",
"print(\"before everything: \")\n",
"!nvidia-smi\n",
"prompt = construct_prompt(\"What restaurants are there on the road from Luxembourg Gare, which coordinates are lat 49.5999681, lon 6.1342493, to Thionville?\", \"\")\n",
"print(\"after creating prompt: \")\n",
"!nvidia-smi\n",
"model_output = pipe(\n",
" prompt, do_sample=False, max_new_tokens=300, return_full_text=False\n",
" )\n",
"print(model_output[0][\"generated_text\"])\n",
"\n",
"print(\"creating the pipe of model output: \")\n",
"!nvidia-smi\n",
"result = execute_function_call(model_output)\n",
"print(\"execute function call: \")\n",
"!nvidia-smi\n",
"del model_output\n",
"import gc # garbage collect library\n",
"gc.collect()\n",
"torch.cuda.empty_cache() \n",
"\n",
"#print(\"Model Output:\", model_output)\n",
"print(\"Execution Result:\", result)\n",
"\n",
"\n",
"#execute_function_call(pipe(construct_prompt(\"Is it raining in Belval, ?\"), do_sample=False, max_new_tokens=300, return_full_text=False))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## functions to process the anwser and the question"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#generation of text with Stable beluga \n",
"def gen(p, maxlen=15, sample=True):\n",
" toks = tokr(p, return_tensors=\"pt\")\n",
" res = model.generate(**toks.to(\"cuda\"), max_new_tokens=maxlen, do_sample=sample).to('cpu')\n",
" return tokr.batch_decode(res)\n",
"\n",
"#to have a prompt corresponding to the specific format required by the fine-tuned model Stable Beluga\n",
"def mk_prompt(user, syst=\"### System:\\nYou are a useful AI assistant in a car, that follows instructions extremely well. Help as much as you can. Answer questions concisely and do not mention what you base your reply on.\\n\\n\"): return f\"{syst}### User: {user}\\n\\n### Assistant:\\n\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "yAJI0WyOLE8G"
},
"outputs": [],
"source": [
"def car_answer_only(complete_answer, general_context):\n",
" \"\"\"returns only the AI assistant answer, without all context, to reply to the user\"\"\"\n",
" pattern = r\"Assistant:\\\\n(.*)(</s>|[.!?](\\s|$))\" #pattern = r\"Assistant:\\\\n(.*?)</s>\"\n",
"\n",
" match = re.search(pattern, complete_answer, re.DOTALL)\n",
"\n",
" if match:\n",
" # Extracting the text\n",
" model_answer = match.group(1)\n",
" #print(complete_answer)\n",
" else:\n",
" #print(complete_answer)\n",
" model_answer = \"There has been an error with the generated response.\" \n",
"\n",
" general_context += model_answer\n",
" return (model_answer, general_context)\n",
"#print(model_answer)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "ViCEgogaENNV"
},
"outputs": [],
"source": [
"def FnAnswer(general_context, ques, place, time, delete_history, state):\n",
" \"\"\"function to manage the two different llms (function calling and basic answer) and call them one after the other\"\"\"\n",
" # Initialize state if it is None\n",
" if delete_history == \"Yes\":\n",
" state = None\n",
" if state is None:\n",
" conv_context = []\n",
" conv_context.append(general_context)\n",
" state = {}\n",
" state['context'] = conv_context\n",
" state['number'] = 0\n",
" state['last_question'] = \"\"\n",
" \n",
" if type(ques) != str: \n",
" ques = ques[0]\n",
" \n",
" place = definePlace(place) #which on the predefined places it is\n",
" \n",
" formatted_context = '\\n'.join(state['context'])\n",
" \n",
" #updated at every question\n",
" general_context = f\"\"\"\n",
" Recent conversation history: '{formatted_context}' (If empty, this indicates the beginning of the conversation).\n",
"\n",
" Previous question from the user: '{state['last_question']}' (This may or may not be related to the current question).\n",
"\n",
" User information: The user is inside a car in {place[0]}, with latitude {place[1]} and longitude {place[2]}. The user is mobile and can drive to different destinations. It is currently {time}\n",
"\n",
" \"\"\"\n",
" #first llm call (function calling model, NexusRaven)\n",
" model_output= pipe(construct_prompt(ques, general_context), do_sample=False, max_new_tokens=300, return_full_text=False)\n",
" call = execute_function_call(model_output) #call variable is formatted to as a call to a specific function with the required parameters\n",
" print(call)\n",
" #this is what will erase the model_output from the GPU memory to free up space\n",
" del model_output\n",
" import gc # garbage collect library\n",
" gc.collect()\n",
" torch.cuda.empty_cache() \n",
" \n",
" #updated at every question\n",
" general_context += f'This information might be of help, use if it seems relevant, and ignore if not relevant to reply to the user: \"{call}\". '\n",
" \n",
" #question formatted for the StableBeluga llm (second llm), using the output of the first llm as context in general_context\n",
" question=f\"\"\"Reply to the user and answer any question with the help of the provided context.\n",
"\n",
" ## Context\n",
"\n",
" {general_context} .\n",
"\n",
" ## Question\n",
"\n",
" {ques}\"\"\"\n",
"\n",
" complete_answer = str(gen(mk_prompt(question), 100)) #answer generation with StableBeluga (2nd llm)\n",
"\n",
" model_answer, general_context= car_answer_only(complete_answer, general_context) #to retrieve only the car answer \n",
" \n",
" language = pipe_language(model_answer, top_k=1, truncation=True)[0]['label'] #detect the language of the answer, to modify the text-to-speech consequently\n",
" \n",
" state['last_question'] = ques #add the current question as 'last question' for the next question's context\n",
" \n",
" state['number']= state['number'] + 1 #adds 1 to the number of interactions with the car\n",
"\n",
" state['context'].append(str(state['number']) + '. User question: '+ ques + ', Model answer: ' + model_answer) #modifies the context\n",
" \n",
" #print(\"contexte : \" + '\\n'.join(state['context']))\n",
" \n",
" if len(state['context'])>5: #6 questions maximum in the context to avoid having too many information\n",
" state['context'] = state['context'][1:]\n",
"\n",
" return model_answer, state['context'], state, language"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "9WQlYePVLrTN"
},
"outputs": [],
"source": [
"def transcript(general_context, link_to_audio, voice, place, time, delete_history, state):\n",
" \"\"\"this function manages speech-to-text to input Fnanswer function and text-to-speech with the Fnanswer output\"\"\"\n",
" # load audio from a specific path\n",
" audio_path = link_to_audio\n",
" audio_array, sampling_rate = librosa.load(link_to_audio, sr=16000) # \"sr=16000\" ensures that the sampling rate is as required\n",
"\n",
"\n",
" # process the audio array\n",
" input_features = processor(audio_array, sampling_rate, return_tensors=\"pt\").input_features\n",
"\n",
"\n",
" predicted_ids = modelw.generate(input_features)\n",
"\n",
" transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)\n",
"\n",
" quest_processing = FnAnswer(general_context, transcription, place, time, delete_history, state)\n",
" \n",
" state=quest_processing[2]\n",
" \n",
" print(\"langue \" + quest_processing[3])\n",
"\n",
" tts.tts_to_file(text= str(quest_processing[0]),\n",
" file_path=\"output.wav\",\n",
" speaker_wav=f'Audio_Files/{voice}.wav',\n",
" language=quest_processing[3],\n",
" emotion = \"angry\")\n",
"\n",
" audio_path = \"output.wav\"\n",
" return audio_path, state['context'], state"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def definePlace(place):\n",
" if(place == 'Luxembourg Gare, Luxembourg'):\n",
" return('Luxembourg Gare', '49.5999681', '6.1342493' )\n",
" elif (place =='Kirchberg Campus, Kirchberg'):\n",
" return('Kirchberg Campus, Luxembourg', '49.62571206478235', '6.160082636815114')\n",
" elif (place =='Belval Campus, Belval'):\n",
" return('Belval-Université, Esch-sur-Alzette', '49.499531', '5.9462903')\n",
" elif (place =='Eiffel Tower, Paris'):\n",
" return('Eiffel Tower, Paris', '48.8582599', '2.2945006')\n",
" elif (place=='Thionville, France'):\n",
" return('Thionville, France', '49.357927', '6.167587')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Interfaces (text and audio)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"#INTERFACE WITH ONLY TEXT\n",
"\n",
"# Generate options for hours (00-23) \n",
"hour_options = [f\"{i:02d}:00:00\" for i in range(24)]\n",
"\n",
"model_answer= ''\n",
"general_context= ''\n",
"# Define the initial state with some initial context.\n",
"print(general_context)\n",
"initial_state = {'context': general_context}\n",
"initial_context= initial_state['context']\n",
"# Create the Gradio interface.\n",
"iface = gr.Interface(\n",
" fn=FnAnswer,\n",
" inputs=[\n",
" gr.Textbox(value=initial_context, visible=False),\n",
" gr.Textbox(lines=2, placeholder=\"Type your message here...\"),\n",
" gr.Radio(choices=['Luxembourg Gare, Luxembourg', 'Kirchberg Campus, Kirchberg', 'Belval Campus, Belval', 'Eiffel Tower, Paris', 'Thionville, France'], label='Choose a location for your car', value= 'Kirchberg Campus, Kirchberg', show_label=True),\n",
" gr.Dropdown(choices=hour_options, label=\"What time is it?\", value = \"08:00:00\"),\n",
" gr.Radio([\"Yes\", \"No\"], label=\"Delete the conversation history?\", value = 'No'),\n",
" gr.State() # This will keep track of the context state across interactions.\n",
" ],\n",
" outputs=[\n",
" gr.Textbox(),\n",
" gr.Textbox(visible=False),\n",
" gr.State()\n",
" ]\n",
")\n",
"gr.close_all()\n",
"# Launch the interface.\n",
"iface.launch(debug=True, share=True, server_name=\"0.0.0.0\", server_port=7860)\n",
"#contextual=gr.Textbox(value=general_context, visible=False)\n",
"#demo = gr.Interface(fn=FnAnswer, inputs=[contextual,\"text\"], outputs=[\"text\", contextual])\n",
"\n",
"#demo.launch()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"id": "mZTt3y3_KOOF"
},
"outputs": [],
"source": [
"#INTERFACE WITH AUDIO TO AUDIO\n",
"\n",
"#to be able to use the microphone on chrome, you will have to go to chrome://flags/#unsafely-treat-insecure-origin-as-secure and enter http://10.186.115.21:7860/ \n",
"#in \"Insecure origins treated as secure\", enable it and relaunch chrome\n",
"\n",
"#example question: \n",
"# what's the weather like outside?\n",
"# What's the closest restaurant from here?\n",
"\n",
"\n",
"\n",
"# Generate options for hours (00-23) \n",
"hour_options = [f\"{i:02d}:00:00\" for i in range(24)]\n",
"\n",
"model_answer= ''\n",
"general_context= ''\n",
"# Define the initial state with some initial context.\n",
"print(general_context)\n",
"initial_state = {'context': general_context}\n",
"initial_context= initial_state['context']\n",
"# Create the Gradio interface.\n",
"iface = gr.Interface(\n",
" fn=transcript,\n",
" inputs=[\n",
" gr.Textbox(value=initial_context, visible=False),\n",
" gr.Audio( type='filepath', label = 'input audio'),\n",
" gr.Radio(choices=['Donald Trump', 'Eddie Murphy'], label='Choose a voice', value= 'Donald Trump', show_label=True), # Radio button for voice selection\n",
" gr.Radio(choices=['Luxembourg Gare, Luxembourg', 'Kirchberg Campus, Kirchberg', 'Belval Campus, Belval', 'Eiffel Tower, Paris', 'Thionville, France'], label='Choose a location for your car', value= 'Kirchberg Campus, Kirchberg', show_label=True),\n",
" gr.Dropdown(choices=hour_options, label=\"What time is it?\", value = \"08:00:00\"),\n",
" gr.Radio([\"Yes\", \"No\"], label=\"Delete the conversation history?\", value = 'No'),\n",
" gr.State() # This will keep track of the context state across interactions.\n",
" ],\n",
" outputs=[\n",
" gr.Audio(label = 'output audio'),\n",
" gr.Textbox(visible=False),\n",
" gr.State()\n",
" ]\n",
")\n",
"#close all interfaces open to make the port available\n",
"gr.close_all()\n",
"# Launch the interface.\n",
"iface.launch(debug=True, share=True, server_name=\"0.0.0.0\", server_port=7860, ssl_verify=False)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Other possible APIs to use"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"def search_nearby(lat, lon, city, key):\n",
" \"\"\"\n",
" :param lat: latitude\n",
" :param lon: longitude\n",
" :param key: api key\n",
" :param type: type of poi\n",
" :return: [5] results ['poi']['name']/['freeformAddress'] || ['position']['lat']/['lon']\n",
" \"\"\"\n",
" results = []\n",
"\n",
" r = requests.get('https://api.tomtom.com/search/2/nearbySearch/.json?key={0}&lat={1}&lon={2}&radius=10000&limit=50'.format(\n",
" key,\n",
" lat,\n",
" lon\n",
" ))\n",
"\n",
" for result in r.json()['results']:\n",
" results.append(f\"The {' '.join(result['poi']['categories'])} {result['poi']['name']} is {int(result['dist'])} meters far from {city}\")\n",
" if len(results) == 7:\n",
" break\n",
"\n",
" return \". \".join(results)\n",
"\n",
"\n",
"print(search_nearby('49.625892805337514', '6.160417066963513', 'your location', TOMTOM_KEY))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"gpuType": "T4",
"provenance": []
},
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 1
}
|