Spaces:
Sleeping
Sleeping
File size: 3,770 Bytes
6574024 b023f8e 6574024 58c8e18 6574024 b023f8e 6574024 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 |
import os
import requests
import time
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from dotenv import load_dotenv
import subprocess
load_dotenv()
# Settings
VM_IP = os.getenv("MACHINE")
GRADIO_UI_URL = "http://paffenroth-23.dyn.wpi.edu:8005/"
CHECK_INTERVAL = 60 # 60 seconds
RECOVERY_SCRIPT_PATH = 'recovery-deployment.sh'
# Email settings (if EMAIL_ALERT is True)
EMAIL_SENDER = os.getenv("EMAIL_SENDER")
EMAIL_RECEIVER = os.getenv("EMAIL_RECEIVER")
EMAIL_PASSWORD = os.getenv("EMAIL_PASSWORD")
def send_email_alert(subject, body):
msg = MIMEMultipart()
msg['From'] = EMAIL_SENDER
msg['To'] = EMAIL_RECEIVER
msg['Subject'] = subject
msg.attach(MIMEText(body, 'plain'))
try:
with smtplib.SMTP('smtp.gmail.com', 587) as server:
server.starttls()
server.login(EMAIL_SENDER, EMAIL_PASSWORD)
text = msg.as_string()
server.sendmail(EMAIL_SENDER, EMAIL_RECEIVER, text)
print("Email alert sent successfully!")
except Exception as e:
print(f"Failed to send email alert: {e}")
def check_vm_status():
response = os.system(f"ping -c 1 {VM_IP}") # send one ICMP echo request
return response == 0 # successful ping
def check_ui_status():
try:
response = requests.get(GRADIO_UI_URL)
if response.status_code == 200:
return True
else:
print(f"UI returned status code {response.status_code}")
return False
except requests.ConnectionError:
print(f"{time.ctime()} ---- Failed to connect to UI.")
return False
def monitor_vm(EMAIL_ALERT = False):
while True:
ui_status = check_ui_status()
# vm_status = check_vm_status()
if ui_status: # and vm_status:
print(f"{time.ctime()} ---- Meowthematical chatbot is up and running")
else:
if not ui_status:
print(f"{time.ctime()} ---- PRODUCT is DOWN!!!")
if check_vm_status(): # check if VM is up and running
print(f"{time.ctime()} ---- VM {VM_IP} is up and running, starting recovery process...")
try:
result = subprocess.run(['bash', RECOVERY_SCRIPT_PATH],
check=True,
capture_output=True,
text=True)
# Output from the script
# print("Output of the script:")
# print(result.stdout) # Standard output of the script
# print("Errors (if any):")
# print(result.stderr) # Standard error output of the script
except subprocess.CalledProcessError as e:
print(f"An error occurred: {e}")
print(f"Return code: {e.returncode}")
print(f"Output: {e.output}")
print(f"Error: {e.stderr}")
if EMAIL_ALERT:
send_email_alert(
subject="MEOWTHMATICAL Down Alert",
body=f"MEOWTHEMATICAL with IP {VM_IP} is down as of {time.ctime()}"
)
else: # both product and VM is gone
if EMAIL_ALERT:
send_email_alert(
subject="VM Down Alert",
body=f"Vitural Machine at IP {VM_IP} is down as of {time.ctime()}"
)
time.sleep(CHECK_INTERVAL)
if __name__ == "__main__":
monitor_vm(EMAIL_ALERT=False)
|