Update functions.py

functions.py

@@ -1,123 +1,122 @@
+Hugging Face's logo
+Hugging Face
+Search models, datasets, users...
+Models
+Datasets
+Spaces
+Docs
+Solutions
+Pricing
+
+
+
+Hugging Face is way more fun with friends and colleagues! 🤗 Join an organization
+Spaces:
+
+Mei000
+/
+air_quality_gb Copied
+like
+0
+App
+Files and versions
+Community
+air_quality_gb
+/
+functions.py
+Mei000's picture
+Mei000
+Upload 3 files
+557ecc6
+about 6 hours ago
+raw
+history
+blame
+contribute
+delete
+5.54 kB
 import requests
 import os
+import joblib
 import pandas as pd
 import datetime
 import numpy as np
+import time
 from sklearn.preprocessing import OrdinalEncoder
 from dotenv import load_dotenv
-load_dotenv()
-
-
-## TODO: write function to display the color coding of the categoies both in the df and as a guide.
-#sg like:
-def color_aq(val):
-    color = 'green' if val else 'red'
-    return f'background-color: {color}'
-# but better
-
-
-def get_air_quality_data(station_name):
-    AIR_QUALITY_API_KEY = os.getenv('AIR_QUALITY_API_KEY')
-    request_value = f'https://api.waqi.info/feed/{station_name}/?token={AIR_QUALITY_API_KEY}'
-    answer = requests.get(request_value).json()["data"]
-    forecast = answer['forecast']['daily']
-    return [
-        answer["time"]["s"][:10],      # Date
-        int(forecast['pm25'][0]['avg']),  # avg predicted pm25
-        int(forecast['pm10'][0]['avg']),  # avg predicted pm10
-        max(int(forecast['pm25'][0]['avg']), int(forecast['pm10'][0]['avg'])) # avg predicted aqi
-    ]
-
-def get_air_quality_df(data):
-    col_names = [
-        'date',
-        'pm25',
-        'pm10',
-        'aqi'
-    ]
-
-    new_data = pd.DataFrame(
-        data
-    ).T
-    new_data.columns = col_names
-    new_data['pm25'] = pd.to_numeric(new_data['pm25'])
-    new_data['pm10'] = pd.to_numeric(new_data['pm10'])
-    new_data['aqi'] = pd.to_numeric(new_data['aqi'])
-
-    return new_data
-
-
-def get_weather_data_daily(city):
-    WEATHER_API_KEY = os.getenv('WEATHER_API_KEY')
-    answer = requests.get(f'https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/{city}/today?unitGroup=metric&include=days&key={WEATHER_API_KEY}&contentType=json').json()
-    data = answer['days'][0]
-    return [
-        answer['address'].lower(),
-        data['datetime'],
-        data['tempmax'],
-        data['tempmin'],
-        data['temp'],
-        data['feelslikemax'],
-        data['feelslikemin'],
-        data['feelslike'],
-        data['dew'],
-        data['humidity'],
-        data['precip'],
-        data['precipprob'],
-        data['precipcover'],
-        data['snow'],
-        data['snowdepth'],
-        data['windgust'],
-        data['windspeed'],
-        data['winddir'],
-        data['pressure'],
-        data['cloudcover'],
-        data['visibility'],
-        data['solarradiation'],
-        data['solarenergy'],
-        data['uvindex'],
-        data['conditions']
-    ]
-
+load_dotenv(override=True)
+
+
+def decode_features(df, feature_view):
+    """Decodes features in the input DataFrame using corresponding Hopsworks Feature Store transformation functions"""
+    df_res = df.copy()
+
+    import inspect
+
+
+    td_transformation_functions = feature_view._batch_scoring_server._transformation_functions
+
+    res = {}
+    for feature_name in td_transformation_functions:
+        if feature_name in df_res.columns:
+            td_transformation_function = td_transformation_functions[feature_name]
+            sig, foobar_locals = inspect.signature(td_transformation_function.transformation_fn), locals()
+            param_dict = dict([(param.name, param.default) for param in sig.parameters.values() if param.default != inspect._empty])
+            if td_transformation_function.name == "min_max_scaler":
+                df_res[feature_name] = df_res[feature_name].map(
+                    lambda x: x * (param_dict["max_value"] - param_dict["min_value"]) + param_dict["min_value"])
+
+            elif td_transformation_function.name == "standard_scaler":
+                df_res[feature_name] = df_res[feature_name].map(
+                    lambda x: x * param_dict['std_dev'] + param_dict["mean"])
+            elif td_transformation_function.name == "label_encoder":
+                dictionary = param_dict['value_to_index']
+                dictionary_ = {v: k for k, v in dictionary.items()}
+                df_res[feature_name] = df_res[feature_name].map(
+                    lambda x: dictionary_[x])
+    return df_res
+
+
+def get_model(project, model_name, evaluation_metric, sort_metrics_by):
+    """Retrieve desired model or download it from the Hopsworks Model Registry.
+    In second case, it will be physically downloaded to this directory"""
+    TARGET_FILE = "model.pkl"
+    list_of_files = [os.path.join(dirpath,filename) for dirpath, _, filenames \
+                     in os.walk('.') for filename in filenames if filename == TARGET_FILE]
+
+    if list_of_files:
+        model_path = list_of_files[0]
+        model = joblib.load(model_path)
+    else:
+        if not os.path.exists(TARGET_FILE):
+            mr = project.get_model_registry()
+            # get best model based on custom metrics
+            model = mr.get_best_model(model_name,
+                                      evaluation_metric,
+                                      sort_metrics_by)
+            model_dir = model.download()
+            model = joblib.load(model_dir + "/model.pkl")
+
+    return model
+
+    
 def get_weather_data_weekly(city: str, start_date: datetime) -> pd.DataFrame:
-    WEATHER_API_KEY = os.getenv('WEATHER_API_KEY')
-    end_date = f"{start_date + datetime.timedelta(days=6):%Y-%m-%d}"
-    answer = requests.get(f'https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/{city}/{start_date}/{end_date}?unitGroup=metric&include=days&key={WEATHER_API_KEY}&contentType=json').json()
-    weather_data = answer['days']
-    final_df = pd.DataFrame()
-
-    for i in range(7):
-        data = weather_data[i]
-        list_of_data = [
-        answer['address'].lower(),
-        data['datetime'],
-        data['tempmax'],
-        data['tempmin'],
-        data['temp'],
-        data['feelslikemax'],
-        data['feelslikemin'],
-        data['feelslike'],
-        data['dew'],
-        data['humidity'],
-        data['precip'],
-        data['precipprob'],
-        data['precipcover'],
-        data['snow'],
-        data['snowdepth'],
-        data['windgust'],
-        data['windspeed'],
-        data['winddir'],
-        data['pressure'],
-        data['cloudcover'],
-        data['visibility'],
-        data['solarradiation'],
-        data['solarenergy'],
-        data['uvindex'],
-        data['conditions']
-    ]
-        weather_df = get_weather_df(list_of_data)
-        final_df = pd.concat([final_df, weather_df])
-    return final_df
+    #WEATHER_API_KEY = os.getenv('WEATHER_API_KEY')
+    ##end_date = f"{start_date + datetime.timedelta(days=6):%Y-%m-%d}"
+    next7days_weather=pd.read_csv('https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/Beijing/next7days?unitGroup=metric&include=days&key=5WNL2M94KKQ4R4F32LFV8DPE4&contentType=csv')
+    #answer = requests.get(f'https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/{city}/{start_date}/{end_date}?unitGroup=metric&include=days&key={WEATHER_API_KEY}&contentType=json').json()
+    
+    
+    df_weather = pd.DataFrame(next7days_weather)
+    df_weather.rename(columns = {"datetime": "date"}, 
+              inplace = True)
+    df_weather.rename(columns = {"name": "city"}, 
+              inplace = True)
+    df_weather.rename(columns = {"sealevelpressure": "pressure"}, 
+              inplace = True)
+    df_weather = df_weather.drop(labels=['city','dew','precip','tempmax','pressure','tempmin','temp','feelslikemax','feelslikemin','feelslike','precipprob','precipcover','snow','snowdepth','cloudcover','severerisk','moonphase','preciptype','sunrise','sunset','conditions','description','icon','stations'], axis=1) #删除不用的列
+
+    return df_weather
 
 def get_weather_df(data):
     col_names = [
@@ -158,23 +157,19 @@ def get_weather_df(data):
 
     return new_data
 
-def data_encoder(X):
-    X.drop(columns=['date', 'name'], inplace=True)
-    X['conditions'] = OrdinalEncoder().fit_transform(X[['conditions']])
-    return X
 
-def get_aplevel(temps:np.ndarray, table:list):
+
+def get_aplevel(temps:np.ndarray) -> list:
     boundary_list = np.array([0, 50, 100, 150, 200, 300]) # assert temps.shape == [x, 1]
     redf = np.logical_not(temps<=boundary_list) # temps.shape[0] x boundary_list.shape[0] ndarray
     hift = np.concatenate((np.roll(redf, -1)[:, :-1], np.full((temps.shape[0], 1), False)), axis = 1)
     cat = np.nonzero(np.not_equal(redf,hift))
 
-    level = [table[el] for el in cat[1]]
-    return level
-
-def get_color(level:list):
     air_pollution_level = ['Good', 'Moderate', 'Unhealthy for sensitive Groups','Unhealthy' ,'Very Unhealthy', 'Hazardous']
-    color_list = ["Green", "Yellow", "DarkOrange", "Red", "Purple", "DarkRed"]
-    ind = [air_pollution_level.index(lel) for lel in level]
-    text = [f"color:{color_list[idex]};" for idex in ind]
-    return text
+    level = [air_pollution_level[el] for el in cat[1]]
+    return level
+    
+def timestamp_2_time(x):
+    dt_obj = datetime.datetime.strptime(str(x), '%Y-%m-%d')
+    dt_obj = dt_obj.timestamp() * 1000
+    return int(dt_obj)