trying to fix cuda error

dvats_xai/.ipynb_checkpoints/__init__-checkpoint.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

dvats_xai/.ipynb_checkpoints/_modidx-checkpoint.py

@@ -0,0 +1,14 @@
+# Autogenerated by nbdev
+
+d = { 'settings': { 'branch': 'master',
+                'doc_baseurl': '/dvats/',
+                'doc_host': 'https://vrodriguezf.github.io',
+                'git_url': 'https://github.com/vrodriguezf/deepvats',
+                'lib_path': 'dvats'},
+  'syms': { 'dvats.all': {},
+            'dvats.dr': {},
+            'dvats.encoder': {},
+            'dvats.imports': {},
+            'dvats.load': {},
+            'dvats.utils': {},
+            'dvats.visualization': {}}}

dvats_xai/.ipynb_checkpoints/_nbdev-checkpoint.py

@@ -0,0 +1,39 @@
+# AUTOGENERATED BY NBDEV! DO NOT EDIT!
+
+__all__ = ["index", "modules", "custom_doc_links", "git_url"]
+
+index = {"check_compatibility": "dr.ipynb",
+         "get_UMAP_prjs": "dr.ipynb",
+         "get_PCA_prjs": "dr.ipynb",
+         "get_TSNE_prjs": "dr.ipynb",
+         "DCAE_torch": "encoder.ipynb",
+         "ENCODER_EMBS_MODULE_NAME": "encoder.ipynb",
+         "get_enc_embs": "encoder.ipynb",
+         "TSArtifact": "load.ipynb",
+         "wandb.apis.public.Artifact.to_df": "load.ipynb",
+         "wandb.apis.public.Artifact.to_tsartifact": "load.ipynb",
+         "infer_or_inject_freq": "load.ipynb",
+         "generate_TS_df": "utils.ipynb",
+         "normalize_columns": "utils.ipynb",
+         "remove_constant_columns": "utils.ipynb",
+         "ReferenceArtifact": "utils.ipynb",
+         "wandb.apis.public.Artifact.to_obj": "utils.ipynb",
+         "PrintLayer": "utils.ipynb",
+         "Learner.export_and_get": "utils.ipynb",
+         "get_wandb_artifacts": "utils.ipynb",
+         "get_pickle_artifact": "utils.ipynb",
+         "plot_TS": "visualization.ipynb",
+         "plot_validation_ts_ae": "visualization.ipynb",
+         "plot_mask": "visualization.ipynb"}
+
+modules = ["dr.py",
+           "encoder.py",
+           "load.py",
+           "utils.py",
+           "visualization.py"]
+
+doc_url = "https://vrodriguezf.github.io/tchub/"
+
+git_url = "https://gitlab.geist.re/pml/x_timecluster_extension/tree/master/"
+
+def custom_doc_links(name): return None

dvats_xai/.ipynb_checkpoints/all-checkpoint.py

@@ -0,0 +1,8 @@
+import dvats
+from .imports import *
+from .load import *
+from .utils import *
+from .dr import *
+from .encoder import *
+from .visualization import *
+from .xai import *

dvats_xai/.ipynb_checkpoints/dr-checkpoint.py

@@ -0,0 +1,166 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/dr.ipynb.
+
+# %% auto 0
+__all__ = ['get_gpu_memory', 'color_for_percentage', 'create_bar', 'gpu_memory_status', 'check_compatibility', 'get_UMAP_prjs',
+           'get_PCA_prjs', 'get_TSNE_prjs', 'cluster_score']
+
+# %% ../nbs/dr.ipynb 2
+import subprocess
+def get_gpu_memory(device = 0):
+    total_memory = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits", "--id=" + str(device)])
+    total_memory = int(total_memory.decode().split('\n')[0])
+    used_memory = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits",  "--id=" + str(device)])
+    used_memory = int(used_memory.decode().split('\n')[0])
+
+    percentage = round((used_memory / total_memory) * 100)
+    return used_memory, total_memory, percentage
+
+def color_for_percentage(percentage):
+    if percentage < 20:
+        return "\033[90m"  # Gray
+    elif percentage < 40:
+        return "\033[94m"  # Blue
+    elif percentage < 60:
+        return "\033[92m"  # Green
+    elif percentage < 80:
+        return "\033[93m"  # Orange
+    else:
+        return "\033[91m"  # Red
+        
+def create_bar(percentage, color_code, length=20):
+    filled_length = int(length * percentage // 100)
+    bar = "█" * filled_length + "-" * (length - filled_length)
+    return color_code + bar + "\033[0m"  # Apply color and reset after bar
+
+def gpu_memory_status(device=0):
+    used, total, percentage = get_gpu_memory(device)
+    color_code = color_for_percentage(percentage)
+    bar = create_bar(percentage, color_code)
+    print(f"Used mem: {used}")
+    print(f"Used mem: {total}")
+    print(f"Memory Usage: [{bar}] {color_code}{percentage}%\033[0m")
+
+# %% ../nbs/dr.ipynb 4
+import umap
+import cudf
+import cuml
+import pandas as pd
+import numpy as np
+from fastcore.all import *
+from .imports import *
+from .load import TSArtifact
+
+# %% ../nbs/dr.ipynb 5
+def check_compatibility(dr_ar:TSArtifact, enc_ar:TSArtifact):
+    "Function to check that the artifact used by the encoder model and the artifact that is \
+    going to be passed through the DR are compatible"
+    try:
+        # Check that both artifacts have the same variables
+        chk_vars = dr_ar.metadata['TS']['vars'] == enc_ar.metadata['TS']['vars']
+        # Check that both artifacts have the same freq
+        chk_freq = dr_ar.metadata['TS']['freq'] == enc_ar.metadata['TS']['freq']
+        # Check that the dr artifact is not normalized (not normalized data has not the key normalization)
+        chk_norm = dr_ar.metadata['TS'].get('normalization') is None
+        # Check that the dr artifact has not missing values
+        chk_miss = dr_ar.metadata['TS']['has_missing_values'] == "False"
+        # Check all logical vars.
+        if chk_vars and chk_freq and chk_norm and chk_miss:
+            print("Artifacts are compatible.")
+        else:
+            raise Exception
+    except Exception as e:
+        print("Artifacts are not compatible.")
+        raise e
+    return None
+
+# %% ../nbs/dr.ipynb 7
+#Comment this part after 4_seconds debugged
+import hashlib
+
+# %% ../nbs/dr.ipynb 8
+import warnings
+import sys
+from numba.core.errors import NumbaPerformanceWarning
+@delegates(cuml.UMAP)
+def get_UMAP_prjs(
+    input_data, 
+    cpu=True, 
+    print_flag = False, 
+    check_memory_usage = True,
+    **kwargs
+):
+    "Compute the projections of `input_data` using UMAP, with a configuration contained in `**kwargs`."
+    if print_flag: 
+        print("--> get_UMAP_prjs")
+        print("kwargs: ", kwargs)
+        sys.stdout.flush()
+        ####
+        checksum = hashlib.md5(input_data.tobytes()).hexdigest()
+        print(checksum)
+        ####
+        
+    if check_memory_usage: gpu_memory_status()
+    
+    warnings.filterwarnings("ignore", category=NumbaPerformanceWarning) # silence NumbaPerformanceWarning
+    
+    #reducer = umap.UMAP(**kwargs) if cpu else cuml.UMAP(**kwargs)
+    if cpu:
+        print("-- umap.UMAP --", cpu)
+        sys.stdout.flush()
+        reducer = umap.UMAP(**kwargs)
+    else:
+        print("-- cuml.UMAP --", cpu)
+        sys.stdout.flush()
+        if 'random_state' in kwargs:
+            kwargs['random_state'] = np.uint64(kwargs['random_state'])
+        reducer = cuml.UMAP(**kwargs)
+    
+    if print_flag: 
+        print("------- reducer --------")
+        print(reducer)
+        print(reducer.get_params())
+        print("------- reducer --------")
+        sys.stdout.flush()
+    
+    projections = reducer.fit_transform(input_data)
+    
+    if check_memory_usage: gpu_memory_status()
+    if print_flag: 
+        checksum = hashlib.md5(projections.tobytes()).hexdigest()
+        print("prjs checksum ", checksum)
+        print("get_UMAP_prjs -->")
+        sys.stdout.flush()
+    return projections
+
+# %% ../nbs/dr.ipynb 13
+@delegates(cuml.PCA)
+def get_PCA_prjs(X, cpu=False, **kwargs):
+    r"""
+    Computes PCA projections of X
+    """
+    if cpu:
+        raise NotImplementedError
+    else:
+        reducer = cuml.PCA(**kwargs)
+    projections = reducer.fit_transform(X)
+    return projections
+
+# %% ../nbs/dr.ipynb 15
+@delegates(cuml.TSNE)
+def get_TSNE_prjs(X, cpu=False, **kwargs):
+    r"""
+    Computes TSNE projections of X
+    """
+    if cpu:
+        raise NotImplementedError
+    else:
+        reducer = cuml.TSNE(**kwargs)
+    projections = reducer.fit_transform(X)
+    return projections
+
+# %% ../nbs/dr.ipynb 18
+from sklearn.metrics import silhouette_score
+def cluster_score(prjs, clusters_labels, print_flag):
+    score = silhouette_score(prjs, clusters_labels)
+    if print_flag: print("Silhouette_score:", score)
+    return score

dvats_xai/.ipynb_checkpoints/encoder-checkpoint.py

@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+"""encoder.ipynb
+
+Automatically generated.
+
+Original file is located at:
+    /home/macu/work/nbs/encoder.ipynb
+"""
+
+#default_exp encoder
+
+#hide
+%load_ext autoreload
+%autoreload 2
+
+#export
+import pandas as pd
+import numpy as np
+from fastcore.all import *
+from tsai.callback.MVP import *
+from tsai.imports import *
+from tsai.models.InceptionTimePlus import InceptionTimePlus
+from tsai.models.explainability import get_acts_and_grads
+from tsai.models.layers import *
+from tsai.data.validation import combine_split_data
+
+#hide
+from tsai.all import *
+
+#export 
+class DCAE_torch(Module):
+    def __init__(self, c_in, seq_len, delta, nfs=[64, 32, 12], kss=[10, 5, 5],
+                 pool_szs=[2,2,3], output_fsz=10):
+        """
+        Create a Deep Convolutional Autoencoder for multivariate time series of `d` dimensions,
+        sliced with a window size of `w`. The parameter `delta` sets the number of latent features that will be
+        contained in the Dense layer of the network. The the number of features
+        maps (filters), the filter size and the pool size can also be adjusted."
+        """
+        assert all_equal([len(x) for x in [nfs, kss, pool_szs]], np.repeat(len(nfs), 3)), \
+            'nfs, kss, and pool_szs must have the same length'
+        assert np.prod(pool_szs) == nfs[-1], \
+            'The number of filters in the last conv layer must be equal to the product of pool sizes'
+        assert seq_len % np.prod(pool_szs) == 0, \
+            'The product of pool sizes must be a divisor of the window size'
+        layers = []
+        for i in range_of(kss):
+            layers += [Conv1d(ni=nfs[i-1] if i>0 else c_in, nf=nfs[i], ks=kss[i]),
+                       nn.MaxPool1d(kernel_size=pool_szs[i])]
+        self.downsample = nn.Sequential(*layers)
+        self.bottleneck = nn.Sequential(OrderedDict([
+            ('flatten', nn.Flatten()),
+            ('latent_in', nn.Linear(seq_len, delta)),
+            ('latent_out', nn.Linear(delta, seq_len)),
+            ('reshape', Reshape(nfs[-1], seq_len // np.prod(pool_szs)))
+        ]))
+        layers = []
+        for i in reversed(range_of(kss)):
+            layers += [Conv1d(ni=nfs[i+1] if i != (len(nfs)-1) else nfs[-1],
+                              nf=nfs[i], ks=kss[i]),
+                       nn.Upsample(scale_factor=pool_szs[i])]
+        layers += [Conv1d(ni=nfs[0], nf=c_in, kernel_size=output_fsz)]
+        self.upsample = nn.Sequential(*layers)
+
+    def forward(self, x):
+        x = self.downsample(x)
+        x = self.bottleneck(x)
+        x = self.upsample(x)
+        return x
+
+#hide
+foo = torch.rand(3, 1, 48)
+m = DCAE_torch(c_in=foo.shape[1], seq_len=foo.shape[2], delta=12)
+m(foo).shape
+
+#export
+ENCODER_EMBS_MODULE_NAME = {
+    InceptionTimePlus: 'backbone', # for mvp based models
+    DCAE_torch: 'bottleneck.latent_in'
+}
+
+#export
+def get_enc_embs(X, enc_learn, module=None, cpu=False, average_seq_dim=True, to_numpy=True):
+    """
+        Get the embeddings of X from an encoder, passed in `enc_learn as a fastai
+        learner. By default, the embeddings are obtained from the last layer
+        before the model head, although any layer can be passed to `model`.
+        Input
+        - `cpu`: Whether to do the model inference in cpu of gpu (GPU recommended)
+        - `average_seq_dim`: Whether to aggregate the embeddings in the sequence dimensions
+        - `to_numpy`: Whether to return the result as a numpy array (if false returns a tensor)
+    """
+    if cpu:
+        print("--> Get enc embs CPU")
+        enc_learn.dls.cpu()
+        enc_learn.cpu()
+    else:
+        print("--> Use CUDA |Get enc embs GPU")
+        enc_learn.dls.cuda()
+        enc_learn.cuda()
+        print("devices: ", enc_learn.dls.device, enc_learn.model.device)
+        print("Use CUDA -->")
+    if enc_learn.dls.bs == 0: enc_learn.dls.bs = 64
+    print("--> Get enc embs bs: ", enc_learn.dls.bs)
+    aux_dl = enc_learn.dls.valid.new_dl(X=X)
+    aux_dl.bs = enc_learn.dls.bs if enc_learn.dls.bs>0 else 64
+    module = nested_attr(enc_learn.model,
+                         ENCODER_EMBS_MODULE_NAME[type(enc_learn.model)]) \
+                if module is None else module
+    embs = [get_acts_and_grads(model=enc_learn.model,
+                               modules=module,
+                               x=xb[0], cpu=cpu)[0] for xb in aux_dl]
+    embs = to_concat(embs)
+    if embs.ndim == 3 and average_seq_dim: embs = embs.mean(axis=2)
+    if to_numpy: embs = embs.numpy() if cpu else embs.cpu().numpy()
+    return embs
+
+#hide
+import wandb
+from dvats.utils import *
+wandb_api = wandb.Api()
+enc_artifact = wandb_api.artifact('deepvats/mvp:latest')
+enc_learner = enc_artifact.to_obj()
+X = torch.rand(9, 1, 48)
+
+#hide
+#slow
+#%%time
+embs = get_enc_embs(X, enc_learner, cpu=True)
+test_eq(embs.shape[0], X.shape[0])
+embs.shape, embs.__class__
+
+#hide
+%%time
+embs = get_enc_embs(X, enc_learner, cpu=False, to_numpy=False)
+test_eq(embs.shape[0], X.shape[0])
+embs.shape, embs.__class__, embs.device
+
+#hide 
+%%time
+embs = get_enc_embs(X, enc_learner, cpu=False, to_numpy=True)
+test_eq(embs.shape[0], X.shape[0])
+embs.shape, embs.__class__
+
+#hide
+
+#from nbdev.export import notebook2script
+
+#notebook2script()
+
+#from tsai import nb2py
+#nb2py
+#beep(1)

dvats_xai/.ipynb_checkpoints/imports-checkpoint.py

@@ -0,0 +1,24 @@
+from IPython.display import Audio, display, HTML, Javascript, clear_output # from tsai
+import importlib
+import numpy as np
+import time
+import sys
+
+##
+# Constants
+##
+WANDB_ARTIFACTS_DIR = 'data/wandb_artifacts'
+
+# General purpose functions
+def beep(inp=1, duration=.1, n=1):
+    rate = 10000
+    mult = 1.6 * inp if inp else .08
+    wave = np.sin(mult*np.arange(rate*duration))
+    for i in range(n): 
+        display(Audio(wave, rate=10000, autoplay=True))
+        time.sleep(duration / .1)
+        
+def m_reload(package_name):
+    for k,v in sys.modules.items():
+        if k.startswith(package_name):
+            importlib.reload(v)

dvats_xai/.ipynb_checkpoints/load-checkpoint.py

@@ -0,0 +1,166 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/load.ipynb.
+
+# %% auto 0
+__all__ = ['TSArtifact', 'infer_or_inject_freq']
+
+# %% ../nbs/load.ipynb 2
+import pandas as pd
+import numpy as np
+from fastcore.all import *
+import wandb
+from datetime import datetime, timedelta
+from .imports import *
+from .utils import *
+import pickle
+import pyarrow.feather as ft
+
+# %% ../nbs/load.ipynb 7
+class TSArtifact(wandb.Artifact):
+
+    default_storage_path = Path(Path.home()/'data/wandb_artifacts/')
+    date_format = '%Y-%m-%d %H:%M:%S' # TODO add milliseconds
+    handle_missing_values_techniques = {
+        'linear_interpolation': lambda df : df.interpolate(method='linear', limit_direction='both'),
+        'overall_mean': lambda df : df.fillna(df.mean()),
+        'overall_median': lambda df : df.fillna(df.median()),
+        'backward_fill' : lambda df : df.fillna(method='bfill'),
+        'forward_fill' : lambda df : df.fillna(method='ffill')
+    }
+
+    "Class that represents a wandb artifact containing time series data. sd stands for start_date \
+    and ed for end_date. Both should be pd.Timestamps"
+
+    @delegates(wandb.Artifact.__init__)
+    def __init__(self, name, sd:pd.Timestamp, ed:pd.Timestamp, **kwargs):
+        super().__init__(type='dataset', name=name, **kwargs)
+        self.sd = sd
+        self.ed = ed
+        if self.metadata is None:
+            self.metadata = dict()
+        self.metadata['TS'] = dict(sd = self.sd.strftime(self.date_format),
+                                   ed = self.ed.strftime(self.date_format))
+
+
+    @classmethod
+    def from_daily_csv_files(cls, root_path, fread=pd.read_csv, start_date=None, end_date=None, metadata=None, **kwargs):
+
+        "Create a wandb artifact of type `dataset`, containing the CSV files from `start_date` \
+        to `end_date`. Dates must be pased as `datetime.datetime` objects. If a `wandb_run` is \
+        defined, the created artifact will be logged to that run, using the longwall name as \
+        artifact name, and the date range as version."
+
+        return None
+
+
+    @classmethod
+    @delegates(__init__)
+    def from_df(cls, df:pd.DataFrame, name:str, path:str=None, sd:pd.Timestamp=None, ed:pd.Timestamp=None,
+                normalize:bool=False, missing_values_technique:str=None, resampling_freq:str=None, **kwargs):
+
+        """
+        Create a TSArtifact of type `dataset`, using the DataFrame `df` samples from \
+        `sd` (start date) to `ed` (end date). Dates must be passed as `datetime.datetime` \
+        objects. The transformed DataFrame is stored as a pickle file in the path `path` \
+        and its reference is added to the artifact entries. Additionally, the dataset can \
+        be normalized (see `normalize` argument) or transformed using missing values \
+        handling techniques (see `missing_values_technique` argument) or resampling (see \
+        `resampling_freq` argument).
+
+        Arguments:
+            df: (DataFrame) The dataframe you want to convert into an artifact.
+            name: (str) The artifact name.
+            path: (str, optional) The path where the file, containing the new transformed \
+                dataframe, is saved. Default None.
+            sd: (sd, optional) Start date. By default, the first index of `df` is taken.
+            ed: (ed, optional) End date. By default, the last index of `df` is taken.
+            normalize: (bool, optional) If the dataset values should be normalized. Default\
+                False.
+            missing_values_technique: (str, optional) The technique used to handle missing \
+                values. Options: "linear_iterpolation", "overall_mean", "overall_median" or \
+                None. Default None.
+            resampling_freq: (str, optional) The offset string or object representing \
+                frequency conversion for time series resampling. Default None.
+
+        Returns:
+            TSArtifact object.
+        """
+        sd = df.index[0] if sd is None else sd
+        ed = df.index[-1] if ed is None else ed
+        obj = cls(name, sd=sd, ed=ed, **kwargs)
+        df = df.query('@obj.sd <= index <= @obj.ed')
+        obj.metadata['TS']['created'] = 'from-df'
+        obj.metadata['TS']['n_vars'] = df.columns.__len__()
+
+        # Handle Missing Values
+        df = obj.handle_missing_values_techniques[missing_values_technique](df) if missing_values_technique is not None else df
+        obj.metadata['TS']['handle_missing_values_technique'] = missing_values_technique.__str__()
+        obj.metadata['TS']['has_missing_values'] = np.any(df.isna().values).__str__()
+
+        # Indexing and Resampling
+        if resampling_freq: df = df.resample(resampling_freq).mean()
+        obj.metadata['TS']['n_samples'] = len(df)
+        obj.metadata['TS']['freq'] = str(df.index.freq)
+
+        # Time Series Variables
+        obj.metadata['TS']['vars'] = list(df.columns)
+
+        # Normalization - Save the previous means and stds
+        if normalize:
+            obj.metadata['TS']['normalization'] = dict(means = df.describe().loc['mean'].to_dict(),
+                                                       stds = df.describe().loc['std'].to_dict())
+            df = normalize_columns(df)
+
+        # Hash and save
+        hash_code = str(pd.util.hash_pandas_object(df).sum()) # str(hash(df.values.tobytes()))
+        path = obj.default_storage_path/f'{hash_code}' if path is None else Path(path)/f'{hash_code}.feather'
+        ft.write_feather(df, path)
+        obj.metadata['TS']['hash'] = hash_code
+        obj.add_file(str(path))
+
+        return obj
+
+# %% ../nbs/load.ipynb 11
+@patch
+def to_df(self:wandb.apis.public.Artifact):
+    "Download the files of a saved wandb artifact and process them as a single dataframe. The artifact must \
+    come from a call to `run.use_artifact` with a proper wandb run."
+    # The way we have to ensure that the argument comes from a TS arfitact is the metadata
+    if self.metadata.get('TS') is None:
+        print(f'ERROR:{self} does not come from a logged TSArtifact')
+        return None
+    dir = Path(self.download())
+    if self.metadata['TS']['created'] == 'from-df':
+        # Call read_pickle with the single file from dir
+        #return pd.read_pickle(dir.ls()[0])
+        return ft.read_feather(dir.ls()[0])
+    else:
+        print("ERROR: Only from_df method is allowed yet")
+
+# %% ../nbs/load.ipynb 13
+@patch
+def to_tsartifact(self:wandb.apis.public.Artifact):
+    "Cast an artifact as a TS artifact. The artifact must have been created from one of the \
+    class creation methods of the class `TSArtifact`. This is useful to go back to a TSArtifact \
+    after downloading an artifact through the wand API"
+    return TSArtifact(name=self.digest, #TODO change this
+                      sd=pd.to_datetime(self.metadata['TS']['sd'], format=TSArtifact.date_format),
+                      ed=pd.to_datetime(self.metadata['TS']['sd'], format=TSArtifact.date_format),
+                      description=self.description,
+                      metadata=self.metadata)
+
+# %% ../nbs/load.ipynb 15
+@delegates(pd.to_datetime)
+def infer_or_inject_freq(df, injected_freq='1s', start_date=None, **kwargs):
+    """
+        Infer index frequency. If there's not a proper time index, create fake timestamps,
+        keeping the desired `injected_freq`. If that is None, set a default one of 1 second.
+        start_date: the first date of the index (int or string).
+    """
+    inferred_freq = pd.infer_freq(df.index)
+    if inferred_freq == 'N':
+        timedelta = pd.to_timedelta(injected_freq)
+        df.index = pd.to_datetime(ifnone(start_date, 0), **kwargs) + timedelta*df.index
+        df.index.freq = pd.infer_freq(df.index)
+    else:
+        df.index.freq = inferred_freq
+    return df

dvats_xai/.ipynb_checkpoints/utils-checkpoint.py

@@ -0,0 +1,134 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/utils.ipynb (unless otherwise specified).
+
+__all__ = ['generate_TS_df', 'normalize_columns', 'remove_constant_columns', 'ReferenceArtifact', 'PrintLayer',
+           'get_wandb_artifacts', 'get_pickle_artifact']
+
+# Cell
+from .imports import *
+from fastcore.all import *
+import wandb
+import pickle
+import pandas as pd
+import numpy as np
+#import tensorflow as tf
+import torch.nn as nn
+from fastai.basics import *
+
+# Cell
+def generate_TS_df(rows, cols):
+    "Generates a dataframe containing a multivariate time series, where each column \
+    represents a variable and each row a time point (sample). The timestamp is in the \
+    index of the dataframe, and it is created with a even space of 1 second between samples"
+    index = np.arange(pd.Timestamp.now(),
+                      pd.Timestamp.now() + pd.Timedelta(rows-1, 'seconds'),
+                      pd.Timedelta(1, 'seconds'))
+    data = np.random.randn(len(index), cols)
+    return pd.DataFrame(data, index=index)
+
+# Cell
+def normalize_columns(df:pd.DataFrame):
+    "Normalize columns from `df` to have 0 mean and 1 standard deviation"
+    mean = df.mean()
+    std = df.std() + 1e-7
+    return (df-mean)/std
+
+# Cell
+def remove_constant_columns(df:pd.DataFrame):
+    return df.loc[:, (df != df.iloc[0]).any()]
+
+# Cell
+class ReferenceArtifact(wandb.Artifact):
+    default_storage_path = Path('data/wandb_artifacts/') # * this path is relative to Path.home()
+    "This class is meant to create an artifact with a single reference to an object \
+    passed as argument in the contructor. The object will be pickled, hashed and stored \
+    in a specified folder."
+    @delegates(wandb.Artifact.__init__)
+    def __init__(self, obj, name, type='object', folder=None, **kwargs):
+        super().__init__(type=type, name=name, **kwargs)
+        # pickle dumps the object and then hash it
+        hash_code = str(hash(pickle.dumps(obj)))
+        folder = Path(ifnone(folder, Path.home()/self.default_storage_path))
+        with open(f'{folder}/{hash_code}', 'wb') as f:
+            pickle.dump(obj, f)
+        self.add_reference(f'file://{folder}/{hash_code}')
+        if self.metadata is None:
+            self.metadata = dict()
+        self.metadata['ref'] = dict()
+        self.metadata['ref']['hash'] = hash_code
+        self.metadata['ref']['type'] = str(obj.__class__)
+
+# Cell
+@patch
+def to_obj(self:wandb.apis.public.Artifact):
+    """Download the files of a saved ReferenceArtifact and get the referenced object. The artifact must \
+    come from a call to `run.use_artifact` with a proper wandb run."""
+    if self.metadata.get('ref') is None:
+        print(f'ERROR:{self} does not come from a saved ReferenceArtifact')
+        return None
+    original_path = ReferenceArtifact.default_storage_path/self.metadata['ref']['hash']
+    path = original_path if original_path.exists() else Path(self.download()).ls()[0]
+    with open(path, 'rb') as f:
+        obj = pickle.load(f)
+    return obj
+
+# Cell
+import torch.nn as nn
+class PrintLayer(nn.Module):
+    def __init__(self):
+        super(PrintLayer, self).__init__()
+
+    def forward(self, x):
+        # Do your print / debug stuff here
+        print(x.shape)
+        return x
+
+# Cell
+@patch
+def export_and_get(self:Learner, keep_exported_file=False):
+    """
+        Export the learner into an auxiliary file, load it and return it back.
+    """
+    aux_path = Path('aux.pkl')
+    self.export(fname='aux.pkl')
+    aux_learn = load_learner('aux.pkl')
+    if not keep_exported_file: aux_path.unlink()
+    return aux_learn
+
+# Cell
+def get_wandb_artifacts(project_path, type=None, name=None, last_version=True):
+    """
+        Get the artifacts logged in a wandb project.
+        Input:
+        - `project_path` (str): entity/project_name
+        - `type` (str): whether to return only one type of artifacts
+        - `name` (str): Leave none to have all artifact names
+        - `last_version`: whether to return only the last version of each artifact or not
+
+        Output: List of artifacts
+    """
+    public_api = wandb.Api()
+    if type is not None:
+        types = [public_api.artifact_type(type, project_path)]
+    else:
+        types = public_api.artifact_types(project_path)
+
+    res = L()
+    for kind in types:
+        for collection in kind.collections():
+            if name is None or name == collection.name:
+                versions = public_api.artifact_versions(
+                    kind.type,
+                    "/".join([kind.entity, kind.project, collection.name]),
+                    per_page=1,
+                )
+                if last_version: res += next(versions)
+                else: res += L(versions)
+    return list(res)
+
+# Cell
+def get_pickle_artifact(filename):
+
+    with open(filename, "rb") as f:
+        df = pickle.load(f)
+
+    return df

dvats_xai/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

dvats_xai/_modidx.py

@@ -0,0 +1,105 @@
+# Autogenerated by nbdev
+
+d = { 'settings': { 'branch': 'master',
+                'doc_baseurl': '/dvats/',
+                'doc_host': 'https://vrodriguezf.github.io',
+                'git_url': 'https://github.com/vrodriguezf/deepvats',
+                'lib_path': 'dvats'},
+  'syms': { 'dvats.all': {},
+            'dvats.dr': { 'dvats.dr.check_compatibility': ('dr.html#check_compatibility', 'dvats/dr.py'),
+                          'dvats.dr.cluster_score': ('dr.html#cluster_score', 'dvats/dr.py'),
+                          'dvats.dr.color_for_percentage': ('dr.html#color_for_percentage', 'dvats/dr.py'),
+                          'dvats.dr.create_bar': ('dr.html#create_bar', 'dvats/dr.py'),
+                          'dvats.dr.get_PCA_prjs': ('dr.html#get_pca_prjs', 'dvats/dr.py'),
+                          'dvats.dr.get_TSNE_prjs': ('dr.html#get_tsne_prjs', 'dvats/dr.py'),
+                          'dvats.dr.get_UMAP_prjs': ('dr.html#get_umap_prjs', 'dvats/dr.py'),
+                          'dvats.dr.get_gpu_memory': ('dr.html#get_gpu_memory', 'dvats/dr.py'),
+                          'dvats.dr.gpu_memory_status': ('dr.html#gpu_memory_status', 'dvats/dr.py')},
+            'dvats.encoder': { 'dvats.encoder.DCAE_torch': ('encoder.html#dcae_torch', 'dvats/encoder.py'),
+                               'dvats.encoder.DCAE_torch.__init__': ('encoder.html#__init__', 'dvats/encoder.py'),
+                               'dvats.encoder.DCAE_torch.forward': ('encoder.html#forward', 'dvats/encoder.py'),
+                               'dvats.encoder.color_for_percentage': ('encoder.html#color_for_percentage', 'dvats/encoder.py'),
+                               'dvats.encoder.create_bar': ('encoder.html#create_bar', 'dvats/encoder.py'),
+                               'dvats.encoder.get_enc_embs': ('encoder.html#get_enc_embs', 'dvats/encoder.py'),
+                               'dvats.encoder.get_enc_embs_set_stride_set_batch_size': ( 'encoder.html#get_enc_embs_set_stride_set_batch_size',
+                                                                                         'dvats/encoder.py'),
+                               'dvats.encoder.get_gpu_memory_': ('encoder.html#get_gpu_memory_', 'dvats/encoder.py'),
+                               'dvats.encoder.gpu_memory_status_': ('encoder.html#gpu_memory_status_', 'dvats/encoder.py')},
+            'dvats.imports': {},
+            'dvats.load': { 'dvats.load.TSArtifact': ('load.html#tsartifact', 'dvats/load.py'),
+                            'dvats.load.TSArtifact.__init__': ('load.html#__init__', 'dvats/load.py'),
+                            'dvats.load.TSArtifact.from_daily_csv_files': ('load.html#from_daily_csv_files', 'dvats/load.py'),
+                            'dvats.load.TSArtifact.from_df': ('load.html#from_df', 'dvats/load.py'),
+                            'dvats.load.infer_or_inject_freq': ('load.html#infer_or_inject_freq', 'dvats/load.py'),
+                            'dvats.load.wandb.apis.public.Artifact.to_df': ('load.html#wandb.apis.public.artifact.to_df', 'dvats/load.py'),
+                            'dvats.load.wandb.apis.public.Artifact.to_tsartifact': ( 'load.html#wandb.apis.public.artifact.to_tsartifact',
+                                                                                     'dvats/load.py')},
+            'dvats.utils': { 'dvats.utils.Learner.export_and_get': ('utils.html#learner.export_and_get', 'dvats/utils.py'),
+                             'dvats.utils.PrintLayer': ('utils.html#printlayer', 'dvats/utils.py'),
+                             'dvats.utils.PrintLayer.__init__': ('utils.html#__init__', 'dvats/utils.py'),
+                             'dvats.utils.PrintLayer.forward': ('utils.html#forward', 'dvats/utils.py'),
+                             'dvats.utils.ReferenceArtifact': ('utils.html#referenceartifact', 'dvats/utils.py'),
+                             'dvats.utils.ReferenceArtifact.__init__': ('utils.html#__init__', 'dvats/utils.py'),
+                             'dvats.utils.exec_with_and_feather_k_output': ('utils.html#exec_with_and_feather_k_output', 'dvats/utils.py'),
+                             'dvats.utils.exec_with_feather': ('utils.html#exec_with_feather', 'dvats/utils.py'),
+                             'dvats.utils.exec_with_feather_k_output': ('utils.html#exec_with_feather_k_output', 'dvats/utils.py'),
+                             'dvats.utils.generate_TS_df': ('utils.html#generate_ts_df', 'dvats/utils.py'),
+                             'dvats.utils.get_pickle_artifact': ('utils.html#get_pickle_artifact', 'dvats/utils.py'),
+                             'dvats.utils.get_wandb_artifacts': ('utils.html#get_wandb_artifacts', 'dvats/utils.py'),
+                             'dvats.utils.learner_module_leaves': ('utils.html#learner_module_leaves', 'dvats/utils.py'),
+                             'dvats.utils.learner_module_leaves_subtables': ( 'utils.html#learner_module_leaves_subtables',
+                                                                              'dvats/utils.py'),
+                             'dvats.utils.normalize_columns': ('utils.html#normalize_columns', 'dvats/utils.py'),
+                             'dvats.utils.py_function': ('utils.html#py_function', 'dvats/utils.py'),
+                             'dvats.utils.remove_constant_columns': ('utils.html#remove_constant_columns', 'dvats/utils.py'),
+                             'dvats.utils.wandb.apis.public.Artifact.to_obj': ( 'utils.html#wandb.apis.public.artifact.to_obj',
+                                                                                'dvats/utils.py')},
+            'dvats.visualization': { 'dvats.visualization.plot_TS': ('visualization.html#plot_ts', 'dvats/visualization.py'),
+                                     'dvats.visualization.plot_mask': ('visualization.html#plot_mask', 'dvats/visualization.py'),
+                                     'dvats.visualization.plot_validation_ts_ae': ( 'visualization.html#plot_validation_ts_ae',
+                                                                                    'dvats/visualization.py')},
+            'dvats.xai': { 'dvats.xai.InteractiveAnomalyPlot': ('xai.html#interactiveanomalyplot', 'dvats/xai.py'),
+                           'dvats.xai.InteractiveAnomalyPlot.__init__': ('xai.html#__init__', 'dvats/xai.py'),
+                           'dvats.xai.InteractiveAnomalyPlot.plot_projections_clusters_interactive': ( 'xai.html#plot_projections_clusters_interactive',
+                                                                                                       'dvats/xai.py'),
+                           'dvats.xai.InteractiveTSPlot': ('xai.html#interactivetsplot', 'dvats/xai.py'),
+                           'dvats.xai.InteractiveTSPlot.__init__': ('xai.html#__init__', 'dvats/xai.py'),
+                           'dvats.xai.add_movement_buttons': ('xai.html#add_movement_buttons', 'dvats/xai.py'),
+                           'dvats.xai.add_selected_features': ('xai.html#add_selected_features', 'dvats/xai.py'),
+                           'dvats.xai.add_windows': ('xai.html#add_windows', 'dvats/xai.py'),
+                           'dvats.xai.anomaly_score': ('xai.html#anomaly_score', 'dvats/xai.py'),
+                           'dvats.xai.calculate_cluster_stats': ('xai.html#calculate_cluster_stats', 'dvats/xai.py'),
+                           'dvats.xai.delta_x_bigger': ('xai.html#delta_x_bigger', 'dvats/xai.py'),
+                           'dvats.xai.delta_x_lower': ('xai.html#delta_x_lower', 'dvats/xai.py'),
+                           'dvats.xai.delta_y_bigger': ('xai.html#delta_y_bigger', 'dvats/xai.py'),
+                           'dvats.xai.delta_y_lower': ('xai.html#delta_y_lower', 'dvats/xai.py'),
+                           'dvats.xai.detector': ('xai.html#detector', 'dvats/xai.py'),
+                           'dvats.xai.get_anomalies': ('xai.html#get_anomalies', 'dvats/xai.py'),
+                           'dvats.xai.get_anomaly_styles': ('xai.html#get_anomaly_styles', 'dvats/xai.py'),
+                           'dvats.xai.get_dataset': ('xai.html#get_dataset', 'dvats/xai.py'),
+                           'dvats.xai.get_dateformat': ('xai.html#get_dateformat', 'dvats/xai.py'),
+                           'dvats.xai.get_df_selected': ('xai.html#get_df_selected', 'dvats/xai.py'),
+                           'dvats.xai.get_embeddings': ('xai.html#get_embeddings', 'dvats/xai.py'),
+                           'dvats.xai.get_prjs': ('xai.html#get_prjs', 'dvats/xai.py'),
+                           'dvats.xai.initial_plot': ('xai.html#initial_plot', 'dvats/xai.py'),
+                           'dvats.xai.merge_overlapping_windows': ('xai.html#merge_overlapping_windows', 'dvats/xai.py'),
+                           'dvats.xai.move_down': ('xai.html#move_down', 'dvats/xai.py'),
+                           'dvats.xai.move_left': ('xai.html#move_left', 'dvats/xai.py'),
+                           'dvats.xai.move_right': ('xai.html#move_right', 'dvats/xai.py'),
+                           'dvats.xai.move_up': ('xai.html#move_up', 'dvats/xai.py'),
+                           'dvats.xai.plot_anomaly_scores_distribution': ('xai.html#plot_anomaly_scores_distribution', 'dvats/xai.py'),
+                           'dvats.xai.plot_clusters_with_anomalies': ('xai.html#plot_clusters_with_anomalies', 'dvats/xai.py'),
+                           'dvats.xai.plot_clusters_with_anomalies_interactive_plot': ( 'xai.html#plot_clusters_with_anomalies_interactive_plot',
+                                                                                        'dvats/xai.py'),
+                           'dvats.xai.plot_initial_config': ('xai.html#plot_initial_config', 'dvats/xai.py'),
+                           'dvats.xai.plot_projections': ('xai.html#plot_projections', 'dvats/xai.py'),
+                           'dvats.xai.plot_projections_clusters': ('xai.html#plot_projections_clusters', 'dvats/xai.py'),
+                           'dvats.xai.plot_save': ('xai.html#plot_save', 'dvats/xai.py'),
+                           'dvats.xai.set_features_buttons': ('xai.html#set_features_buttons', 'dvats/xai.py'),
+                           'dvats.xai.setup_boxes': ('xai.html#setup_boxes', 'dvats/xai.py'),
+                           'dvats.xai.setup_style': ('xai.html#setup_style', 'dvats/xai.py'),
+                           'dvats.xai.shift_datetime': ('xai.html#shift_datetime', 'dvats/xai.py'),
+                           'dvats.xai.show': ('xai.html#show', 'dvats/xai.py'),
+                           'dvats.xai.toggle_trace': ('xai.html#toggle_trace', 'dvats/xai.py'),
+                           'dvats.xai.umap_parameters': ('xai.html#umap_parameters', 'dvats/xai.py'),
+                           'dvats.xai.update_plot': ('xai.html#update_plot', 'dvats/xai.py')}}}

dvats_xai/_nbdev.py

@@ -0,0 +1,39 @@
+# AUTOGENERATED BY NBDEV! DO NOT EDIT!
+
+__all__ = ["index", "modules", "custom_doc_links", "git_url"]
+
+index = {"check_compatibility": "dr.ipynb",
+         "get_UMAP_prjs": "dr.ipynb",
+         "get_PCA_prjs": "dr.ipynb",
+         "get_TSNE_prjs": "dr.ipynb",
+         "DCAE_torch": "encoder.ipynb",
+         "ENCODER_EMBS_MODULE_NAME": "encoder.ipynb",
+         "get_enc_embs": "encoder.ipynb",
+         "TSArtifact": "load.ipynb",
+         "wandb.apis.public.Artifact.to_df": "load.ipynb",
+         "wandb.apis.public.Artifact.to_tsartifact": "load.ipynb",
+         "infer_or_inject_freq": "load.ipynb",
+         "generate_TS_df": "utils.ipynb",
+         "normalize_columns": "utils.ipynb",
+         "remove_constant_columns": "utils.ipynb",
+         "ReferenceArtifact": "utils.ipynb",
+         "wandb.apis.public.Artifact.to_obj": "utils.ipynb",
+         "PrintLayer": "utils.ipynb",
+         "Learner.export_and_get": "utils.ipynb",
+         "get_wandb_artifacts": "utils.ipynb",
+         "get_pickle_artifact": "utils.ipynb",
+         "plot_TS": "visualization.ipynb",
+         "plot_validation_ts_ae": "visualization.ipynb",
+         "plot_mask": "visualization.ipynb"}
+
+modules = ["dr.py",
+           "encoder.py",
+           "load.py",
+           "utils.py",
+           "visualization.py"]
+
+doc_url = "https://vrodriguezf.github.io/tchub/"
+
+git_url = "https://gitlab.geist.re/pml/x_timecluster_extension/tree/master/"
+
+def custom_doc_links(name): return None

dvats_xai/all.py

@@ -0,0 +1,8 @@
+import dvats
+from .imports import *
+from .load import *
+from .utils import *
+from .dr import *
+from .encoder import *
+from .visualization import *
+from .xai import *

dvats_xai/dr.py

@@ -0,0 +1,166 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/dr.ipynb.
+
+# %% auto 0
+__all__ = ['get_gpu_memory', 'color_for_percentage', 'create_bar', 'gpu_memory_status', 'check_compatibility', 'get_UMAP_prjs',
+           'get_PCA_prjs', 'get_TSNE_prjs', 'cluster_score']
+
+# %% ../nbs/dr.ipynb 2
+import subprocess
+def get_gpu_memory(device = 0):
+    total_memory = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits", "--id=" + str(device)])
+    total_memory = int(total_memory.decode().split('\n')[0])
+    used_memory = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits",  "--id=" + str(device)])
+    used_memory = int(used_memory.decode().split('\n')[0])
+
+    percentage = round((used_memory / total_memory) * 100)
+    return used_memory, total_memory, percentage
+
+def color_for_percentage(percentage):
+    if percentage < 20:
+        return "\033[90m"  # Gray
+    elif percentage < 40:
+        return "\033[94m"  # Blue
+    elif percentage < 60:
+        return "\033[92m"  # Green
+    elif percentage < 80:
+        return "\033[93m"  # Orange
+    else:
+        return "\033[91m"  # Red
+        
+def create_bar(percentage, color_code, length=20):
+    filled_length = int(length * percentage // 100)
+    bar = "█" * filled_length + "-" * (length - filled_length)
+    return color_code + bar + "\033[0m"  # Apply color and reset after bar
+
+def gpu_memory_status(device=0):
+    used, total, percentage = get_gpu_memory(device)
+    color_code = color_for_percentage(percentage)
+    bar = create_bar(percentage, color_code)
+    print(f"GPU | Used mem: {used}")
+    print(f"GPU | Used mem: {total}")
+    print(f"GPU | Memory Usage: [{bar}] {color_code}{percentage}%\033[0m")
+
+# %% ../nbs/dr.ipynb 4
+import umap
+import cudf
+import cuml
+import pandas as pd
+import numpy as np
+from fastcore.all import *
+from .imports import *
+from .load import TSArtifact
+
+# %% ../nbs/dr.ipynb 5
+def check_compatibility(dr_ar:TSArtifact, enc_ar:TSArtifact):
+    "Function to check that the artifact used by the encoder model and the artifact that is \
+    going to be passed through the DR are compatible"
+    try:
+        # Check that both artifacts have the same variables
+        chk_vars = dr_ar.metadata['TS']['vars'] == enc_ar.metadata['TS']['vars']
+        # Check that both artifacts have the same freq
+        chk_freq = dr_ar.metadata['TS']['freq'] == enc_ar.metadata['TS']['freq']
+        # Check that the dr artifact is not normalized (not normalized data has not the key normalization)
+        chk_norm = dr_ar.metadata['TS'].get('normalization') is None
+        # Check that the dr artifact has not missing values
+        chk_miss = dr_ar.metadata['TS']['has_missing_values'] == "False"
+        # Check all logical vars.
+        if chk_vars and chk_freq and chk_norm and chk_miss:
+            print("Artifacts are compatible.")
+        else:
+            raise Exception
+    except Exception as e:
+        print("Artifacts are not compatible.")
+        raise e
+    return None
+
+# %% ../nbs/dr.ipynb 7
+#Comment this part after 4_seconds debugged
+import hashlib
+
+# %% ../nbs/dr.ipynb 8
+import warnings
+import sys
+from numba.core.errors import NumbaPerformanceWarning
+@delegates(cuml.UMAP)
+def get_UMAP_prjs(
+    input_data, 
+    cpu=True, 
+    print_flag = False, 
+    check_memory_usage = True,
+    **kwargs
+):
+    "Compute the projections of `input_data` using UMAP, with a configuration contained in `**kwargs`."
+    if print_flag: 
+        print("--> get_UMAP_prjs")
+        print("kwargs: ", kwargs)
+        sys.stdout.flush()
+        ####
+        checksum = hashlib.md5(input_data.tobytes()).hexdigest()
+        print(checksum)
+        ####
+        
+    if check_memory_usage: gpu_memory_status()
+    
+    warnings.filterwarnings("ignore", category=NumbaPerformanceWarning) # silence NumbaPerformanceWarning
+    
+    #reducer = umap.UMAP(**kwargs) if cpu else cuml.UMAP(**kwargs)
+    if cpu:
+        print("-- umap.UMAP --", cpu)
+        sys.stdout.flush()
+        reducer = umap.UMAP(**kwargs)
+    else:
+        print("-- cuml.UMAP --", cpu)
+        sys.stdout.flush()
+        if 'random_state' in kwargs:
+            kwargs['random_state'] = np.uint64(kwargs['random_state'])
+        reducer = cuml.UMAP(**kwargs)
+    
+    if print_flag: 
+        print("------- reducer --------")
+        print(reducer)
+        print(reducer.get_params())
+        print("------- reducer --------")
+        sys.stdout.flush()
+    
+    projections = reducer.fit_transform(input_data)
+    
+    if check_memory_usage: gpu_memory_status()
+    if print_flag: 
+        checksum = hashlib.md5(projections.tobytes()).hexdigest()
+        print("prjs checksum ", checksum)
+        print("get_UMAP_prjs -->")
+        sys.stdout.flush()
+    return projections
+
+# %% ../nbs/dr.ipynb 13
+@delegates(cuml.PCA)
+def get_PCA_prjs(X, cpu=False, **kwargs):
+    r"""
+    Computes PCA projections of X
+    """
+    if cpu:
+        raise NotImplementedError
+    else:
+        reducer = cuml.PCA(**kwargs)
+    projections = reducer.fit_transform(X)
+    return projections
+
+# %% ../nbs/dr.ipynb 15
+@delegates(cuml.TSNE)
+def get_TSNE_prjs(X, cpu=False, **kwargs):
+    r"""
+    Computes TSNE projections of X
+    """
+    if cpu:
+        raise NotImplementedError
+    else:
+        reducer = cuml.TSNE(**kwargs)
+    projections = reducer.fit_transform(X)
+    return projections
+
+# %% ../nbs/dr.ipynb 18
+from sklearn.metrics import silhouette_score
+def cluster_score(prjs, clusters_labels, print_flag):
+    score = silhouette_score(prjs, clusters_labels)
+    if print_flag: print("Silhouette_score:", score)
+    return score

dvats_xai/encoder.py

@@ -0,0 +1,301 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/encoder.ipynb.
+
+# %% auto 0
+__all__ = ['ENCODER_EMBS_MODULE_NAME', 'get_gpu_memory_', 'color_for_percentage', 'create_bar', 'gpu_memory_status_',
+           'DCAE_torch', 'get_enc_embs', 'get_enc_embs_set_stride_set_batch_size']
+
+# %% ../nbs/encoder.ipynb 2
+import subprocess
+def get_gpu_memory_(device = 0):
+    total_memory = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits", "--id=" + str(device)])
+    total_memory = int(total_memory.decode().split('\n')[0])
+    used_memory = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits",  "--id=" + str(device)])
+    used_memory = int(used_memory.decode().split('\n')[0])
+
+    percentage = round((used_memory / total_memory) * 100)
+    return used_memory, total_memory, percentage
+
+def color_for_percentage(percentage):
+    if percentage < 20:
+        return "\033[90m"  # Gray
+    elif percentage < 40:
+        return "\033[94m"  # Blue
+    elif percentage < 60:
+        return "\033[92m"  # Green
+    elif percentage < 80:
+        return "\033[93m"  # Orange
+    else:
+        return "\033[91m"  # Red
+        
+def create_bar(percentage, color_code, length=20):
+    filled_length = int(length * percentage // 100)
+    bar = "█" * filled_length + "-" * (length - filled_length)
+    return color_code + bar + "\033[0m"  # Apply color and reset after bar
+
+def gpu_memory_status_(device=0):
+    used, total, percentage = get_gpu_memory_(device)
+    color_code = color_for_percentage(percentage)
+    bar = create_bar(percentage, color_code)
+    print(f"GPU | Used mem: {used}")
+    print(f"GPU | Used mem: {total}")
+    print(f"GPU | Memory Usage: [{bar}] {color_code}{percentage}%\033[0m")
+
+
+# %% ../nbs/encoder.ipynb 4
+import pandas as pd
+import numpy as np
+from fastcore.all import *
+from tsai.callback.MVP import *
+from tsai.imports import *
+from tsai.models.InceptionTimePlus import InceptionTimePlus
+from tsai.models.explainability import get_acts_and_grads
+from tsai.models.layers import *
+from tsai.data.validation import combine_split_data
+import time
+
+# %% ../nbs/encoder.ipynb 7
+class DCAE_torch(Module):
+    def __init__(self, c_in, seq_len, delta, nfs=[64, 32, 12], kss=[10, 5, 5],
+                 pool_szs=[2,2,3], output_fsz=10):
+        """
+        Create a Deep Convolutional Autoencoder for multivariate time series of `d` dimensions,
+        sliced with a window size of `w`. The parameter `delta` sets the number of latent features that will be
+        contained in the Dense layer of the network. The the number of features
+        maps (filters), the filter size and the pool size can also be adjusted."
+        """
+        assert all_equal([len(x) for x in [nfs, kss, pool_szs]], np.repeat(len(nfs), 3)), \
+            'nfs, kss, and pool_szs must have the same length'
+        assert np.prod(pool_szs) == nfs[-1], \
+            'The number of filters in the last conv layer must be equal to the product of pool sizes'
+        assert seq_len % np.prod(pool_szs) == 0, \
+            'The product of pool sizes must be a divisor of the window size'
+        layers = []
+        for i in range_of(kss):
+            layers += [Conv1d(ni=nfs[i-1] if i>0 else c_in, nf=nfs[i], ks=kss[i]),
+                       nn.MaxPool1d(kernel_size=pool_szs[i])]
+        self.downsample = nn.Sequential(*layers)
+        self.bottleneck = nn.Sequential(OrderedDict([
+            ('flatten', nn.Flatten()),
+            ('latent_in', nn.Linear(seq_len, delta)),
+            ('latent_out', nn.Linear(delta, seq_len)),
+            ('reshape', Reshape(nfs[-1], seq_len // np.prod(pool_szs)))
+        ]))
+        layers = []
+        for i in reversed(range_of(kss)):
+            layers += [Conv1d(ni=nfs[i+1] if i != (len(nfs)-1) else nfs[-1],
+                              nf=nfs[i], ks=kss[i]),
+                       nn.Upsample(scale_factor=pool_szs[i])]
+        layers += [Conv1d(ni=nfs[0], nf=c_in, kernel_size=output_fsz)]
+        self.upsample = nn.Sequential(*layers)
+
+
+    def forward(self, x):
+        x = self.downsample(x)
+        x = self.bottleneck(x)
+        x = self.upsample(x)
+        return x
+
+# %% ../nbs/encoder.ipynb 10
+ENCODER_EMBS_MODULE_NAME = {
+    InceptionTimePlus: 'backbone', # for mvp based models
+    DCAE_torch: 'bottleneck.latent_in'
+}
+
+# %% ../nbs/encoder.ipynb 12
+def get_enc_embs(X, enc_learn, module=None, cpu=False, average_seq_dim=True, to_numpy=True):
+    """
+        Get the embeddings of X from an encoder, passed in `enc_learn as a fastai
+        learner. By default, the embeddings are obtained from the last layer
+        before the model head, although any layer can be passed to `model`.
+        Input
+        - `cpu`: Whether to do the model inference in cpu of gpu (GPU recommended)
+        - `average_seq_dim`: Whether to aggregate the embeddings in the sequence dimensions
+        - `to_numpy`: Whether to return the result as a numpy array (if false returns a tensor)
+    """
+    print("--> Check CUDA")
+    if cpu:
+        print("--> Get enc embs CPU")
+        enc_learn.dls.cpu()
+        enc_learn.cpu()
+    else:
+        print("--> Ensure empty cache")
+        torch.cuda.empty_cache()
+        print("--> Use CUDA |Get enc embs GPU ")
+        enc_learn.dls.cuda()
+        enc_learn.cuda()
+        if torch.cuda.is_available():
+            print("CUDA está disponible")
+            print("Dispositivo CUDA actual: ", torch.cuda.current_device())
+            print("Nombre del dispositivo CUDA actual: ", torch.cuda.get_device_name(torch.cuda.current_device()))
+            
+        else:
+            print("CUDA no está disponible ")
+            print("Use CUDA -->")
+    if enc_learn.dls.bs == 0: enc_learn.dls.bs = 64
+    
+    print("--> Set dataset from X (enc_learn does not contain dls)")
+    aux_dl = enc_learn.dls.valid.new_dl(X=X)
+    aux_dl.bs = enc_learn.dls.bs if enc_learn.dls.bs>0 else 64
+    print("--> Get module")
+    module = nested_attr(enc_learn.model,ENCODER_EMBS_MODULE_NAME[type(enc_learn.model)]) if module is None else module
+    
+    print("--> Get enc embs bs: ", aux_dl.bs)
+    embs = [
+        get_acts_and_grads(
+            model=enc_learn.model,
+            modules=module,
+            x=xb[0], 
+            cpu=cpu
+        )[0] 
+        for xb in aux_dl
+    ]
+    print("--> Concat")
+    if not cpu:
+        total_emb_size = sum([emb.element_size() * emb.nelement() for emb in embs])
+        free_memory = torch.cuda.get_device_properties(0).total_memory - torch.cuda.memory_allocated()
+        if (total_emb_size < free_memory):
+            print("Fit in GPU")
+            embs=[emb.cuda() for emb in embs]
+        else:
+            print("Dont fit in GPU --> Go to CPU")
+            embs=[emb.cpu() for emb in embs]
+    embs = to_concat(embs)
+    print("--> reduce")
+    if embs.ndim == 3 and average_seq_dim: embs = embs.mean(axis=2)
+    print("--> 2 numpy")
+    if to_numpy: embs = embs.numpy() if cpu else embs.cpu().numpy()
+    return embs
+
+# %% ../nbs/encoder.ipynb 13
+def get_enc_embs_set_stride_set_batch_size(
+    X, enc_learn, stride, batch_size, module=None, cpu=False, average_seq_dim=True, to_numpy=True, 
+    print_flag = False, time_flag=False, chunk_size = 0, check_memory_usage = False
+):
+    """
+        Get the embeddings of X from an encoder, passed in `enc_learn as a fastai
+        learner. By default, the embeddings are obtained from the last layer
+        before the model head, although any layer can be passed to `model`.
+        Input
+        - `cpu`: Whether to do the model inference in cpu of gpu (GPU recommended)
+        - `average_seq_dim`: Whether to aggregate the embeddings in the sequence dimensions
+        - `to_numpy`: Whether to return the result as a numpy array (if false returns a tensor)
+    """
+    if time_flag:
+        t_start = time.time()
+    if print_flag:
+        print("--> get_enc_embs_set_stride_set_batch_size")
+    if check_memory_usage: gpu_memory_status_()
+        #print("get_enc_embs_set_stride_set_batch_size | Check versions")
+        #import sys
+        #print("get_enc_embs_set_stride_set_batch_size | Check versions | Python version", sys.version)
+        #print("get_enc_embs_set_stride_set_batch_size | Check versions | PyTorch version", torch.__version__)
+        #print("get_enc_embs_set_stride_set_batch_size | Check versions | CUDA version", torch.version.cuda)
+        #print("get_enc_embs_set_stride_set_batch_size | Apply stride & batch size")
+    
+    X = X[::stride]
+    enc_learn.dls.bs = batch_size 
+    
+    if (print_flag): print("get_enc_embs_set_stride_set_batch_size | Check CUDA | X ~ ", X.shape[0])
+    if cpu:
+        if (print_flag): print("get_enc_embs_set_stride_set_batch_size | Get enc embs CPU")
+        enc_learn.dls.cpu()
+        enc_learn.cpu()
+    else:
+        if torch.cuda.is_available():
+            if (print_flag): 
+                print("get_enc_embs_set_stride_set_batch_size | CUDA device id:", torch.cuda.current_device())
+                print("get_enc_embs_set_stride_set_batch_size | CUDA device name: ", torch.cuda.get_device_name(torch.cuda.current_device()))
+                print("get_enc_embs_set_stride_set_batch_size | Ensure empty cache & move 2 GPU")
+            torch.cuda.empty_cache()
+            enc_learn.dls.cuda()
+            enc_learn.cuda()
+        else:
+            if (print_flag): print("get_enc_embs_set_stride_set_batch_size | No cuda available. Set CPU = true")
+            cpu = True
+    
+    if enc_learn.dls.bs is None or enc_learn.dls.bs == 0: enc_learn.dls.bs = 64
+
+    if (print_flag): print("get_enc_embs_set_stride_set_batch_size | Set dataset from X (enc_learn does not contain dls)")
+    aux_dl = enc_learn.dls.valid.new_dl(X=X)
+    aux_dl.bs = enc_learn.dls.bs if enc_learn.dls.bs>0 else 64
+    if (print_flag): print("get_enc_embs_set_stride_set_batch_size | Get module")
+    module = nested_attr(enc_learn.model,ENCODER_EMBS_MODULE_NAME[type(enc_learn.model)]) if module is None else module
+    
+    if (print_flag): 
+        #print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | module ", module)
+        print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | aux_dl len", len(aux_dl))
+        print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | aux_dl.batch_len ", len(next(iter(aux_dl))))
+        print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | aux_dl.bs ", aux_dl.bs)
+        if (not cpu):
+            total = torch.cuda.get_device_properties(device).total_memory
+            used = torch.cuda.memory_allocated(torch.cuda.current_device())
+            reserved = torch.cuda.memory_reserved(torch.cuda.current_device())
+            print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | total_mem ", total)
+            print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | used_mem ", used)
+            print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | reserved_mem ", reserved)
+            print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | available_mem ", total-reserved)
+            sys.stdout.flush()
+                                              
+    if (cpu or ( chunk_size == 0 )):
+        embs = [
+            get_acts_and_grads(
+                model=enc_learn.model,
+                modules=module, 
+                x=xb[0], 
+                cpu=cpu
+            )[0] 
+            for xb in aux_dl
+        ]
+        if not cpu: embs=[emb.cpu() for emb in embs]
+    else:
+        embs = []
+        total_chunks=max(1,round(len(X)/chunk_size))
+        if print_flag: print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | aux_dl len | " + str(len(X)) + " chunk size: " + str(chunk_size) + " => " + str(total_chunks) + " chunks")
+        for i in range(0, total_chunks):
+            if print_flag: 
+                print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | Chunk [ " + str(i) + "/"+str(total_chunks)+"] => " + str(round(i*100/total_chunks)) + "%")
+                sys.stdout.flush()
+            chunk = [batch for (n, batch) in enumerate(aux_dl) if (chunk_size*i <= n  and chunk_size*(i+1) > n) ]
+            chunk_embs = [
+                get_acts_and_grads(
+                    model=enc_learn.model,
+                    modules=module,
+                    x=xb[0], 
+                    cpu=cpu
+                )[0]
+                for xb in chunk
+            ]
+            # Mueve los embeddings del bloque a la CPU
+            chunk_embs = [emb.cpu() for emb in chunk_embs]
+            embs.extend(chunk_embs)
+            torch.cuda.empty_cache()
+        if print_flag: 
+            print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | 100%")
+            sys.stdout.flush()
+    
+    if print_flag: print("get_enc_embs_set_stride_set_batch_size | concat embeddings")
+    
+    embs = to_concat(embs)
+    
+    if print_flag: print("get_enc_embs_set_stride_set_batch_size | Reduce")
+    
+    if embs.ndim == 3 and average_seq_dim: embs = embs.mean(axis=2)
+    
+    if print_flag: print("get_enc_embs_set_stride_set_batch_size | Convert to numpy")
+    
+    if to_numpy: 
+        if cpu or chunk_size > 0:
+            embs = embs.numpy() 
+        else: 
+            embs = embs.cpu().numpy()
+            torch.cuda.empty_cache()
+    if time_flag:
+        t = time.time()-t_start
+        if print_flag:
+            print("get_enc_embs_set_stride_set_batch_size " + str(t) + " seconds -->")
+        else:
+            print("get_enc_embs_set_stride_set_batch_size " + str(t) + " seconds")
+    if check_memory_usage: gpu_memory_status_()
+    if print_flag: 
+        print("get_enc_embs_set_stride_set_batch_size -->")
+    return embs

dvats_xai/imports.py

@@ -0,0 +1,24 @@
+from IPython.display import Audio, display, HTML, Javascript, clear_output # from tsai
+import importlib
+import numpy as np
+import time
+import sys
+
+##
+# Constants
+##
+WANDB_ARTIFACTS_DIR = 'data/wandb_artifacts'
+
+# General purpose functions
+def beep(inp=1, duration=.1, n=1):
+    rate = 10000
+    mult = 1.6 * inp if inp else .08
+    wave = np.sin(mult*np.arange(rate*duration))
+    for i in range(n): 
+        display(Audio(wave, rate=10000, autoplay=True))
+        time.sleep(duration / .1)
+        
+def m_reload(package_name):
+    for k,v in sys.modules.items():
+        if k.startswith(package_name):
+            importlib.reload(v)

dvats_xai/load.py

@@ -0,0 +1,168 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/load.ipynb.
+
+# %% auto 0
+__all__ = ['TSArtifact', 'infer_or_inject_freq']
+
+# %% ../nbs/load.ipynb 2
+import pandas as pd
+import numpy as np
+from fastcore.all import *
+import wandb
+from datetime import datetime, timedelta
+from .imports import *
+from .utils import *
+import pickle
+import pyarrow.feather as ft
+
+# %% ../nbs/load.ipynb 7
+class TSArtifact(wandb.Artifact):
+
+    default_storage_path = Path(Path.home()/'data/wandb_artifacts/')
+    date_format = '%Y-%m-%d %H:%M:%S' # TODO add milliseconds
+    handle_missing_values_techniques = {
+        'linear_interpolation': lambda df : df.interpolate(method='linear', limit_direction='both'),
+        'overall_mean': lambda df : df.fillna(df.mean()),
+        'overall_median': lambda df : df.fillna(df.median()),
+        'backward_fill' : lambda df : df.fillna(method='bfill'),
+        'forward_fill' : lambda df : df.fillna(method='ffill')
+    }
+
+    "Class that represents a wandb artifact containing time series data. sd stands for start_date \
+    and ed for end_date. Both should be pd.Timestamps"
+
+    @delegates(wandb.Artifact.__init__)
+    def __init__(self, name, sd:pd.Timestamp, ed:pd.Timestamp, **kwargs):
+        super().__init__(type='dataset', name=name, **kwargs)
+        self.sd = sd
+        self.ed = ed
+        if self.metadata is None:
+            self.metadata = dict()
+        self.metadata['TS'] = dict(sd = self.sd.strftime(self.date_format),
+                                   ed = self.ed.strftime(self.date_format))
+
+
+    @classmethod
+    def from_daily_csv_files(cls, root_path, fread=pd.read_csv, start_date=None, end_date=None, metadata=None, **kwargs):
+
+        "Create a wandb artifact of type `dataset`, containing the CSV files from `start_date` \
+        to `end_date`. Dates must be pased as `datetime.datetime` objects. If a `wandb_run` is \
+        defined, the created artifact will be logged to that run, using the longwall name as \
+        artifact name, and the date range as version."
+
+        return None
+
+
+    @classmethod
+    @delegates(__init__)
+    def from_df(cls, df:pd.DataFrame, name:str, path:str=None, sd:pd.Timestamp=None, ed:pd.Timestamp=None,
+                normalize:bool=False, missing_values_technique:str=None, resampling_freq:str=None, **kwargs):
+
+        """
+        Create a TSArtifact of type `dataset`, using the DataFrame `df` samples from \
+        `sd` (start date) to `ed` (end date). Dates must be passed as `datetime.datetime` \
+        objects. The transformed DataFrame is stored as a pickle file in the path `path` \
+        and its reference is added to the artifact entries. Additionally, the dataset can \
+        be normalized (see `normalize` argument) or transformed using missing values \
+        handling techniques (see `missing_values_technique` argument) or resampling (see \
+        `resampling_freq` argument).
+
+        Arguments:
+            df: (DataFrame) The dataframe you want to convert into an artifact.
+            name: (str) The artifact name.
+            path: (str, optional) The path where the file, containing the new transformed \
+                dataframe, is saved. Default None.
+            sd: (sd, optional) Start date. By default, the first index of `df` is taken.
+            ed: (ed, optional) End date. By default, the last index of `df` is taken.
+            normalize: (bool, optional) If the dataset values should be normalized. Default\
+                False.
+            missing_values_technique: (str, optional) The technique used to handle missing \
+                values. Options: "linear_iterpolation", "overall_mean", "overall_median" or \
+                None. Default None.
+            resampling_freq: (str, optional) The offset string or object representing \
+                frequency conversion for time series resampling. Default None.
+
+        Returns:
+            TSArtifact object.
+        """
+        sd = df.index[0] if sd is None else sd
+        ed = df.index[-1] if ed is None else ed
+        obj = cls(name, sd=sd, ed=ed, **kwargs)
+        df = df.query('@obj.sd <= index <= @obj.ed')
+        obj.metadata['TS']['created'] = 'from-df'
+        obj.metadata['TS']['n_vars'] = df.columns.__len__()
+
+        # Handle Missing Values
+        df = obj.handle_missing_values_techniques[missing_values_technique](df) if missing_values_technique is not None else df
+        obj.metadata['TS']['handle_missing_values_technique'] = missing_values_technique.__str__()
+        obj.metadata['TS']['has_missing_values'] = np.any(df.isna().values).__str__()
+
+        # Indexing and Resampling
+        if resampling_freq: df = df.resample(resampling_freq).mean()
+        obj.metadata['TS']['n_samples'] = len(df)
+        obj.metadata['TS']['freq'] = str(df.index.freq)
+
+        # Time Series Variables
+        obj.metadata['TS']['vars'] = list(df.columns)
+
+        # Normalization - Save the previous means and stds
+        if normalize:
+            obj.metadata['TS']['normalization'] = dict(means = df.describe().loc['mean'].to_dict(),
+                                                       stds = df.describe().loc['std'].to_dict())
+            df = normalize_columns(df)
+
+        # Hash and save
+        hash_code = str(pd.util.hash_pandas_object(df).sum()) # str(hash(df.values.tobytes()))
+        path = obj.default_storage_path/f'{hash_code}' if path is None else Path(path)/f'{hash_code}'
+        print("About to write df to ", path)
+        ft.write_feather(df, path, compression = 'lz4')
+        #feather.write_dataframe
+        obj.metadata['TS']['hash'] = hash_code
+        obj.add_file(str(path))
+
+        return obj
+
+# %% ../nbs/load.ipynb 14
+@patch
+def to_df(self:wandb.apis.public.Artifact):
+    "Download the files of a saved wandb artifact and process them as a single dataframe. The artifact must \
+    come from a call to `run.use_artifact` with a proper wandb run."
+    # The way we have to ensure that the argument comes from a TS arfitact is the metadata
+    if self.metadata.get('TS') is None:
+        print(f'ERROR:{self} does not come from a logged TSArtifact')
+        return None
+    dir = Path(self.download())
+    if self.metadata['TS']['created'] == 'from-df':
+        # Call read_pickle with the single file from dir
+        #return pd.read_pickle(dir.ls()[0])
+        return ft.read_feather(dir.ls()[0])
+    else:
+        print("ERROR: Only from_df method is allowed yet")
+
+# %% ../nbs/load.ipynb 16
+@patch
+def to_tsartifact(self:wandb.apis.public.Artifact):
+    "Cast an artifact as a TS artifact. The artifact must have been created from one of the \
+    class creation methods of the class `TSArtifact`. This is useful to go back to a TSArtifact \
+    after downloading an artifact through the wand API"
+    return TSArtifact(name=self.digest, #TODO change this
+                      sd=pd.to_datetime(self.metadata['TS']['sd'], format=TSArtifact.date_format),
+                      ed=pd.to_datetime(self.metadata['TS']['sd'], format=TSArtifact.date_format),
+                      description=self.description,
+                      metadata=self.metadata)
+
+# %% ../nbs/load.ipynb 18
+@delegates(pd.to_datetime)
+def infer_or_inject_freq(df, injected_freq='1s', start_date=None, **kwargs):
+    """
+        Infer index frequency. If there's not a proper time index, create fake timestamps,
+        keeping the desired `injected_freq`. If that is None, set a default one of 1 second.
+        start_date: the first date of the index (int or string).
+    """
+    inferred_freq = pd.infer_freq(df.index)
+    if inferred_freq == 'N':
+        timedelta = pd.to_timedelta(injected_freq)
+        df.index = pd.to_datetime(ifnone(start_date, 0), **kwargs) + timedelta*df.index
+        df.index.freq = pd.infer_freq(df.index)
+    else:
+        df.index.freq = inferred_freq
+    return df

dvats_xai/utils.py

@@ -0,0 +1,245 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/utils.ipynb.
+
+# %% auto 0
+__all__ = ['generate_TS_df', 'normalize_columns', 'remove_constant_columns', 'ReferenceArtifact', 'PrintLayer',
+           'get_wandb_artifacts', 'get_pickle_artifact', 'exec_with_feather', 'py_function',
+           'exec_with_feather_k_output', 'exec_with_and_feather_k_output', 'learner_module_leaves',
+           'learner_module_leaves_subtables']
+
+# %% ../nbs/utils.ipynb 3
+from .imports import *
+from fastcore.all import *
+import wandb
+import pickle
+import pandas as pd
+import numpy as np
+#import tensorflow as tf
+import torch.nn as nn
+from fastai.basics import *
+
+# %% ../nbs/utils.ipynb 5
+def generate_TS_df(rows, cols):
+    "Generates a dataframe containing a multivariate time series, where each column \
+    represents a variable and each row a time point (sample). The timestamp is in the \
+    index of the dataframe, and it is created with a even space of 1 second between samples"
+    index = np.arange(pd.Timestamp.now(),
+                      pd.Timestamp.now() + pd.Timedelta(rows-1, 'seconds'),
+                      pd.Timedelta(1, 'seconds'))
+    data = np.random.randn(len(index), cols)
+    return pd.DataFrame(data, index=index)
+
+# %% ../nbs/utils.ipynb 10
+def normalize_columns(df:pd.DataFrame):
+    "Normalize columns from `df` to have 0 mean and 1 standard deviation"
+    mean = df.mean()
+    std = df.std() + 1e-7
+    return (df-mean)/std
+
+# %% ../nbs/utils.ipynb 16
+def remove_constant_columns(df:pd.DataFrame):
+    return df.loc[:, (df != df.iloc[0]).any()]
+
+# %% ../nbs/utils.ipynb 21
+class ReferenceArtifact(wandb.Artifact):
+    default_storage_path = Path('data/wandb_artifacts/') # * this path is relative to Path.home()
+    "This class is meant to create an artifact with a single reference to an object \
+    passed as argument in the contructor. The object will be pickled, hashed and stored \
+    in a specified folder."
+    @delegates(wandb.Artifact.__init__)
+    def __init__(self, obj, name, type='object', folder=None, **kwargs):
+        super().__init__(type=type, name=name, **kwargs)
+        # pickle dumps the object and then hash it
+        hash_code = str(hash(pickle.dumps(obj)))
+        folder = Path(ifnone(folder, Path.home()/self.default_storage_path))
+        with open(f'{folder}/{hash_code}', 'wb') as f:
+            pickle.dump(obj, f)
+        self.add_reference(f'file://{folder}/{hash_code}')
+        if self.metadata is None:
+            self.metadata = dict()
+        self.metadata['ref'] = dict()
+        self.metadata['ref']['hash'] = hash_code
+        self.metadata['ref']['type'] = str(obj.__class__)
+
+# %% ../nbs/utils.ipynb 24
+@patch
+def to_obj(self:wandb.apis.public.Artifact):
+    """Download the files of a saved ReferenceArtifact and get the referenced object. The artifact must \
+    come from a call to `run.use_artifact` with a proper wandb run."""
+    if self.metadata.get('ref') is None:
+        print(f'ERROR:{self} does not come from a saved ReferenceArtifact')
+        return None
+    original_path = ReferenceArtifact.default_storage_path/self.metadata['ref']['hash']
+    path = original_path if original_path.exists() else Path(self.download()).ls()[0]
+    with open(path, 'rb') as f:
+        obj = pickle.load(f)
+    return obj
+
+# %% ../nbs/utils.ipynb 33
+import torch.nn as nn
+class PrintLayer(nn.Module):
+    def __init__(self):
+        super(PrintLayer, self).__init__()
+
+    def forward(self, x):
+        # Do your print / debug stuff here
+        print(x.shape)
+        return x
+
+# %% ../nbs/utils.ipynb 34
+@patch
+def export_and_get(self:Learner, keep_exported_file=False):
+    """
+        Export the learner into an auxiliary file, load it and return it back.
+    """
+    aux_path = Path('aux.pkl')
+    self.export(fname='aux.pkl')
+    aux_learn = load_learner('aux.pkl')
+    if not keep_exported_file: aux_path.unlink()
+    return aux_learn
+
+# %% ../nbs/utils.ipynb 35
+def get_wandb_artifacts(project_path, type=None, name=None, last_version=True):
+    """
+        Get the artifacts logged in a wandb project.
+        Input:
+        - `project_path` (str): entity/project_name
+        - `type` (str): whether to return only one type of artifacts
+        - `name` (str): Leave none to have all artifact names
+        - `last_version`: whether to return only the last version of each artifact or not
+
+        Output: List of artifacts
+    """
+    public_api = wandb.Api()
+    if type is not None:
+        types = [public_api.artifact_type(type, project_path)]
+    else:
+        types = public_api.artifact_types(project_path)
+
+    res = L()
+    for kind in types:
+        for collection in kind.collections():
+            if name is None or name == collection.name:
+                versions = public_api.artifact_versions(
+                    kind.type,
+                    "/".join([kind.entity, kind.project, collection.name]),
+                    per_page=1,
+                )
+                if last_version: res += next(versions)
+                else: res += L(versions)
+    return list(res)
+
+# %% ../nbs/utils.ipynb 39
+def get_pickle_artifact(filename):
+
+    with open(filename, "rb") as f:
+        df = pickle.load(f)
+    
+    return df
+
+# %% ../nbs/utils.ipynb 41
+import pyarrow.feather as ft
+import pickle
+
+# %% ../nbs/utils.ipynb 42
+def exec_with_feather(function, path = None, print_flag = False, *args, **kwargs):
+    result = None
+    if not (path is none):
+        if print_flag: print("--> Exec with feather | reading input from ", path)
+        input = ft.read_feather(path)
+        if print_flag: print("--> Exec with feather | Apply function ", path)
+        result = function(input, *args, **kwargs)
+        if print_flag: print("Exec with feather --> ", path)
+    return result
+
+# %% ../nbs/utils.ipynb 43
+def py_function(module_name, function_name, print_flag = False):
+    try:
+        function = getattr(__import__('__main__'), function_name)
+    except:
+        module = __import__(module_name, fromlist=[''])
+        function = getattr(module, function_name)
+    print("py function: ", function_name, ": ", function)
+    return function
+
+# %% ../nbs/utils.ipynb 46
+import time
+def exec_with_feather_k_output(function_name, module_name = "main", path = None, k_output = 0, print_flag = False, time_flag = False, *args, **kwargs):
+    result = None
+    function = py_function(module_name, function_name, print_flag)
+    if time_flag: t_start = time.time()
+    if not (path is None):
+        if print_flag: print("--> Exec with feather | reading input from ", path)
+        input = ft.read_feather(path)
+        if print_flag: print("--> Exec with feather | Apply function ", path)
+        result = function(input, *args, **kwargs)[k_output]
+    if time_flag:
+        t_end = time.time()
+        print("Exec with feather | time: ", t_end-t_start)
+    if print_flag: print("Exec with feather --> ", path)
+    return result
+
+# %% ../nbs/utils.ipynb 48
+def exec_with_and_feather_k_output(function_name, module_name = "main", path_input = None, path_output = None, k_output = 0, print_flag = False, time_flag = False, *args, **kwargs):
+    result = None
+    function = py_function(module_name, function_name, print_flag)
+    if time_flag: t_start = time.time()
+    if not (path_input is None):
+        if print_flag: print("--> Exec with feather | reading input from ", path_input)
+        input = ft.read_feather(path_input)
+        if print_flag: 
+            print("--> Exec with feather | Apply function ", function_name, "input type: ", type(input))
+        
+        result = function(input, *args, **kwargs)[k_output]
+        ft.write_feather(df, path, compression = 'lz4')
+    if time_flag:
+        t_end = time.time()
+        print("Exec with feather | time: ", t_end-t_start)
+    if print_flag: print("Exec with feather --> ", path_output)
+    return path_output
+
+# %% ../nbs/utils.ipynb 52
+def learner_module_leaves(learner):
+    modules = list(learner.modules())[0]  # Obtener el módulo raíz
+    rows = []
+
+    def find_leave_modules(module, path=[]):
+        for name, sub_module in module.named_children():
+            current_path = path + [f"{type(sub_module).__name__}"]
+            if not list(sub_module.children()):
+                leave_name = ' -> '.join(current_path)
+                leave_params = str(sub_module).strip()  
+                rows.append([
+                    leave_name,
+                    f"{type(sub_module).__name__}",
+                    name,
+                    leave_params
+                ]
+                )
+
+            find_leave_modules(sub_module, current_path)
+
+    find_leave_modules(modules)
+    
+    df = pd.DataFrame(rows, columns=['Path', 'Module_type', 'Module_name', 'Module'])
+    return df
+
+# %% ../nbs/utils.ipynb 56
+def learner_module_leaves_subtables(learner, print_flag = False):
+    df = pd.DataFrame(columns=['Path', 'Module_type', 'Module_name', 'Module'])
+    md = learner_module_leaves(learner).drop(
+            'Path', axis = 1
+        ).sort_values(
+            by = 'Module_type'
+        )
+    if print_flag: print("The layers are of this types:")
+
+    md_types = pd.DataFrame(md['Module_type'].drop_duplicates())
+    if print_flag: 
+        display(md_types)
+        print("And they are called with this parameters:")
+    
+    md_modules = pd.DataFrame(md['Module'].drop_duplicates())
+    
+    if print_flag: display(md_modules)
+    
+    return md_types, md_modules

dvats_xai/visualization.py

@@ -0,0 +1,63 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/visualization.ipynb.
+
+# %% auto 0
+__all__ = ['plot_TS', 'plot_validation_ts_ae', 'plot_mask']
+
+# %% ../nbs/visualization.ipynb 3
+from fastcore.all import *
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import torch
+
+# %% ../nbs/visualization.ipynb 6
+@delegates(pd.DataFrame.plot)
+def plot_TS(df:pd.core.frame.DataFrame, **kwargs):
+    df.plot(subplots=True, **kwargs)
+    plt.show()
+
+# %% ../nbs/visualization.ipynb 8
+def plot_validation_ts_ae(prediction:np.array, original:np.array, title_str = "Validation plot", fig_size = (15,15), anchor = (-0.01, 0.89), window_num = 0, return_fig=True, title_pos = 0.9):
+    # Create the figure
+    fig = plt.figure(figsize=(fig_size[0],fig_size[1]))
+    # Create the subplot axes
+    axes = fig.subplots(nrows=original.shape[2], ncols=1)
+    # We iterate over the sensor data and plot both the original and the prediction
+    for i,ax in zip(range(original.shape[2]),fig.axes):
+        ax.plot(original[window_num,:,i], label='Original Data')
+        ax.plot(prediction[window_num,:,i], label='Prediction')
+    # Handle the legend configuration and position
+    lines, labels = fig.axes[-1].get_legend_handles_labels()
+    fig.legend(lines, labels,loc='upper left', ncol=2)
+    # Write the plot title (and position it closer to the top of the graph)
+    fig.suptitle(title_str, y = title_pos)
+    # Tight results:
+    fig.tight_layout()
+    # Returns
+    if return_fig:
+        return fig
+    fig
+    return None
+
+# %% ../nbs/visualization.ipynb 12
+def plot_mask(mask, i=0, fig_size=(10,10), title_str="Mask", return_fig=False):
+    """
+    Plot the mask passed as argument. The mask is a 3D boolean tensor. The first 
+    dimension is the window number (or item index), the second is the variable, and the third is the time step.
+    Input:
+        mask: 3D boolean tensor
+        i: index of the window to plot
+        fig_size: size of the figure
+        title_str: title of the plot
+        return_fig: if True, returns the figure
+    Output:
+        if return_fig is True, returns the figure, otherwise, it does not return anything
+    """
+    plt.figure(figsize=fig_size)
+    plt.pcolormesh(mask[i], cmap='cool')
+    plt.title(f'{title_str} {i}, mean: {mask[0].float().mean().item():.3f}')
+    if return_fig:
+        return plt.gcf()
+    else:
+        plt.show()
+        return None

r_shiny_app/global.R

@@ -32,7 +32,10 @@ torch <- reticulate::import("torch")
 #options(shiny.trace = TRUE)
 if(torch$cuda$is_available()){
   print(paste0("CUDA AVAILABLE. Num devices: ", torch$cuda$device_count()))
-  torch$cuda$set_device(as.integer(0))
+  device=as.integer(0)
+  print(paste0("Set up device ordinal "), device)
+  torch$cuda$set_device(device)
+  print(paste0("Device setted up"))
   #torch$cuda$set_device(as.integer(1))
   #torch$cuda$set_device(as.integer(2))
   #print(torch$cuda$memory_summary())