optimized version working at low z

insight/.ipynb_checkpoints/archive-checkpoint.py

@@ -13,7 +13,7 @@ rcParams["font.family"] = "STIXGeneral"
 
 
 class archive():
-    def __init__(self, path, aperture=2, drop_stars=True, clean_photometry=True, convert_colors=True, extinction_corr=True, only_zspec=True, reliable_zspec=True):
+    def __init__(self, path, aperture=2, drop_stars=True, clean_photometry=True, convert_colors=True, extinction_corr=True, only_zspec=True, Qz_cut=1):
         
         self.aperture = aperture
         
@@ -39,30 +39,28 @@ class archive():
         hdu_list = fits.open(os.path.join(path,filename_valid))
         cat_test = Table(hdu_list[1].data).to_pandas()
         
+        self._get_loss_weights(cat)
+        self._get_loss_weights(cat_test)
+        
         gold_sample = pd.read_csv(os.path.join(path,filename_gold))
         
         #cat_test = self._match_gold_sample(cat_test,gold_sample)        
         
         if drop_stars==True:
             cat = cat[cat.mu_class_L07==1]
+            cat_test = cat_test[cat_test.mu_class_L07==1]
 
         if clean_photometry==True:
             cat = self._clean_photometry(cat)
             cat_test = self._clean_photometry(cat_test)
             
-        self._get_loss_weights(cat)
         
         cat = cat[cat.w_Q_f_S15>0]
-            
-        self._set_training_data(cat, only_zspec=only_zspec, reliable_zspec=reliable_zspec, extinction_corr=extinction_corr, convert_colors=convert_colors)
-        
-        
-        self._set_testing_data(cat_test, only_zspec=only_zspec, reliable_zspec='Total', extinction_corr=extinction_corr, convert_colors=convert_colors)
+                    
+        self._set_training_data(cat, only_zspec=only_zspec, extinction_corr=extinction_corr, convert_colors=convert_colors,Qz_cut=Qz_cut)
+        self._set_testing_data(cat_test, only_zspec=only_zspec, extinction_corr=extinction_corr, convert_colors=convert_colors)
         
         self._get_loss_weights(cat)
-        
-        #self.cat_test=cat_test
-        #self.cat_train=cat
             
     def _extract_fluxes(self,catalogue):
         columns_f = [f'FLUX_{x}_{self.aperture}' for x in ['G','R','I','Z','Y','J','H']]
@@ -100,13 +98,9 @@ class archive():
             catalogue = catalogue[catalogue.z_spec_S15>0]
         return catalogue
 
-    def _clean_zspec_sample(self,catalogue ,kind=None):
-        if kind==None:
-            return catalogue
-        elif kind=='Total':
-            return catalogue[catalogue['reliable_S15']>0]
-        elif kind=='Partial':
-            return catalogue[(catalogue['w_Q_f_S15']>0.5)]
+    def _clean_zspec_sample(self,catalogue ,Qz_cut):
+        catalogue = catalogue[catalogue.w_Q_f_S15>=Qz_cut]
+        return catalogue
         
     def _map_weight(self,Qz):
         for key, value in self.weight_dict.items():
@@ -134,11 +128,11 @@ class archive():
         return catalogue_valid
 
     
-    def _set_training_data(self,catalogue, only_zspec=True, reliable_zspec=True, extinction_corr=True, convert_colors=True):
+    def _set_training_data(self,catalogue, only_zspec=True, extinction_corr=True, convert_colors=True,Qz_cut=1):
         
         if only_zspec:
             catalogue = self._take_only_zspec(catalogue, cat_flag='Calib')
-            catalogue = self._clean_zspec_sample(catalogue, kind=reliable_zspec)
+            catalogue = self._clean_zspec_sample(catalogue, Qz_cut=Qz_cut)
             
         self.cat_train=catalogue
         f, ferr = self._extract_fluxes(catalogue)
@@ -159,11 +153,11 @@ class archive():
         self.target_z_train = catalogue['z_spec_S15'].values
         self.target_qz_train = catalogue['w_Q_f_S15'].values
         
-    def _set_testing_data(self,catalogue, only_zspec=True, reliable_zspec=True, extinction_corr=True, convert_colors=True):
+    def _set_testing_data(self,catalogue, only_zspec=True, extinction_corr=True, convert_colors=True):
  
         if only_zspec:
             catalogue = self._take_only_zspec(catalogue, cat_flag='Valid')
-            catalogue = self._clean_zspec_sample(catalogue, kind=reliable_zspec)
+            catalogue = self._clean_zspec_sample(catalogue, Qz_cut=1)
             
         self.cat_test=catalogue
             

insight/.ipynb_checkpoints/insight-checkpoint.py

@@ -8,34 +8,37 @@ from astropy.io import fits
 import os
 from astropy.table import Table
 from scipy.spatial import KDTree
+from scipy.special import erf
 
 class Insight_module():
     """ Define class"""
     
-    def __init__(self, model):
+    def __init__(self, model, batch_size):
         self.model=model
         self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        self.batch_size=batch_size
         
-    def _get_dataloaders(self, input_data, target_data, target_weights, val_fraction=0.1):
+    def _get_dataloaders(self, input_data, target_data, val_fraction=0.1):
         input_data = torch.Tensor(input_data)
         target_data = torch.Tensor(target_data)
-        target_weights = torch.Tensor(target_weights)
 
-        dataset = TensorDataset(input_data, target_data, target_weights)
+        dataset = TensorDataset(input_data, target_data)
 
         trainig_dataset, val_dataset = torch.utils.data.random_split(dataset, [int(len(dataset)*(1-val_fraction)), int(len(dataset)*val_fraction)+1])
-        loader_train = DataLoader(trainig_dataset, batch_size=64, shuffle = True)
+        loader_train = DataLoader(trainig_dataset, batch_size=self.batch_size, shuffle = True)
         loader_val = DataLoader(val_dataset, batch_size=64, shuffle = True)
 
         return loader_train, loader_val
 
+                
 
-    def _loss_function(self,mean, std, logmix, true, target_weights):
-        log_prob =   logmix - 0.5*(mean - true[:,None]).pow(2) / std.pow(2) - torch.log(std)
 
-        log_prob = torch.logsumexp(log_prob, 1)
+    def _loss_function(self,mean, std, logmix, true):
+                
+        logerf = torch.log(erf(true.cpu()[:,None]/(np.sqrt(2)*std.detach().cpu())+1))
         
-        #log_prob = log_prob * target_weights
+        log_prob =   logmix - 0.5*(mean - true[:,None]).pow(2) / std.pow(2) - torch.log(std) #- logerf.to(self.device)
+        log_prob = torch.logsumexp(log_prob, 1)
         loss = -log_prob.mean()
 
         return loss     
@@ -43,21 +46,25 @@ class Insight_module():
     def _to_numpy(self,x):
         return x.detach().cpu().numpy()
         
-    def train(self,input_data, target_data, target_weights,  nepochs=10, val_fraction=0.1, lr=1e-3 ):
+    def train(self,input_data, target_data,  nepochs=10, step_size = 100, val_fraction=0.1, lr=1e-3 ):
         self.model = self.model.train()
-        loader_train, loader_val = self._get_dataloaders(input_data, target_data, target_weights, val_fraction=0.1)
+        loader_train, loader_val = self._get_dataloaders(input_data, target_data, val_fraction=0.1)
         optimizer = optim.Adam(self.model.parameters(), lr=lr, weight_decay=1e-4)
+        scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=step_size, gamma =0.1)
+    
         
         self.model = self.model.to(self.device)
         
-        loss_train, loss_validation = [],[]
+        self.loss_train, self.loss_validation = [],[]
+        
+        
 
         for epoch in range(nepochs):
-            for input_data, target_data, target_weights in loader_train:
+            for input_data, target_data in loader_train:
+                _loss_train, _loss_validation = [],[]
 
                 input_data = input_data.to(self.device)
                 target_data = target_data.to(self.device)
-                target_weights = target_weights.to(self.device)
 
 
                 optimizer.zero_grad()
@@ -69,32 +76,33 @@ class Insight_module():
 
                 #print(mu,sig,target_data,torch.exp(logmix_coeff))
 
-                loss = self._loss_function(mu, sig, logmix_coeff, target_data,target_weights)
+                loss = self._loss_function(mu, sig, logmix_coeff, target_data)
+                _loss_train.append(loss.item())
                 
                 loss.backward()
-                optimizer.step()  
+                optimizer.step()
+            
+            scheduler.step()   
                                 
-            loss_train.append(loss.item())
+            self.loss_train.append(np.mean(_loss_train))
 
-            for input_data, target_data, target_weights in loader_val:
+            for input_data, target_data in loader_val:
 
 
                 input_data = input_data.to(self.device)
                 target_data = target_data.to(self.device)
-                target_weights = target_weights.to(self.device)
 
 
                 mu, logsig, logmix_coeff = self.model(input_data)
                 logsig = torch.clamp(logsig,-6,2)
                 sig = torch.exp(logsig)
 
-                loss_val = self._loss_function(mu, sig, logmix_coeff, target_data, target_weights)
-            loss_validation.append(loss_val.item())
+                loss_val = self._loss_function(mu, sig, logmix_coeff, target_data)
+                _loss_validation.append(loss_val.item())
 
-            print(f'training_loss:{loss}',f'testing_loss:{loss_val}')
-            
-        self.loss_train=loss_train
-        self.loss_validation=loss_validation
+            self.loss_validation.append(np.mean(_loss_validation))
+
+            #print(f'training_loss:{loss}',f'testing_loss:{loss_val}')
            
             
     def get_photoz(self,input_data, target_data):

insight/.ipynb_checkpoints/insight_arch-checkpoint.py

@@ -8,64 +8,37 @@ class Photoz_network(nn.Module):
             nn.Linear(6, 10),
             nn.Dropout(dropout_prob),
             nn.ReLU(),
-            nn.Linear(10, 30),
+            nn.Linear(10, 20),
             nn.Dropout(dropout_prob),
             nn.ReLU(),
-            nn.Linear(30, 50),
+            nn.Linear(20, 50),
             nn.Dropout(dropout_prob),
             nn.ReLU(),
-            nn.Linear(50, 70),
+            nn.Linear(50, 20),
             nn.Dropout(dropout_prob),
             nn.ReLU(),
-            nn.Linear(70, 100)
+            nn.Linear(20, 10)
         )
         
         self.measure_mu = nn.Sequential(
-            nn.Linear(100, 80),
+            nn.Linear(10, 20),
             nn.Dropout(dropout_prob),
             nn.ReLU(),
-            nn.Linear(80, 70),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),        
-            nn.Linear(70, 60),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),
-            nn.Linear(60, 50),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),
-            nn.Linear(50, num_gauss)
+            nn.Linear(20, num_gauss)
         )
 
         self.measure_coeffs = nn.Sequential(
-            nn.Linear(100, 80),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),
-            nn.Linear(80, 70),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),        
-            nn.Linear(70, 60),
+            nn.Linear(10, 20),
             nn.Dropout(dropout_prob),
             nn.ReLU(),
-            nn.Linear(60, 50),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),
-            nn.Linear(50, num_gauss)
+            nn.Linear(20, num_gauss)
         )
 
         self.measure_sigma = nn.Sequential(
-            nn.Linear(100, 80),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),
-            nn.Linear(80, 70),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),        
-            nn.Linear(70, 60),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),
-            nn.Linear(60, 50),
+            nn.Linear(10, 20),
             nn.Dropout(dropout_prob),
             nn.ReLU(),
-            nn.Linear(50, num_gauss)
+            nn.Linear(20, num_gauss)
         )
         
     def forward(self, x):

insight/.ipynb_checkpoints/utils-checkpoint.py

@@ -9,9 +9,9 @@ def nmad(data):
 def sigma68(data): return 0.5*(pd.Series(data).quantile(q = 0.84) - pd.Series(data).quantile(q = 0.16))
 
 
-def plot_photoz(df, nbins,xvariable,metric, type_bin='bin'):
+def plot_photoz_estimates(df, nbins,xvariable,metric, type_bin='bin'):
     bin_edges = stats.mstats.mquantiles(df[xvariable].values, np.linspace(0.1,1,nbins))
-    ydata,xlab = [],[]
+    ydata,xdata = [],[]
     
     
     for k in range(len(bin_edges)-1):
@@ -21,26 +21,37 @@ def plot_photoz(df, nbins,xvariable,metric, type_bin='bin'):
         mean_mag =  (edge_max + edge_min) / 2
         
         if type_bin=='bin':
-            df_plot = df_test[(df_test.imag > edge_min) & (df_test.imag < edge_max)]
+            df_plot = df_test[(df_test[xvariable] > edge_min) & (df_test[xvariable] < edge_max)]
         elif type_bin=='cum':
-            df_plot = df_test[(df_test.imag < edge_max)]
+            df_plot = df_test[(df_test[xvariable] < edge_max)]
         else:
             raise ValueError("Only type_bin=='bin' for binned and 'cum' for cumulative are supported")
 
 
-        xlab.append(mean_mag)
+        xdata.append(mean_mag)
         if metric=='sig68':
             ydata.append(sigma68(df_plot.zwerr))
+            ylab=r'$\sigma_{\rm NMAD} [\Delta z]$'
         elif metric=='bias':
-            ydata.append(np.mean(df_plot.zwerr))
+            ydata.append(np.median(df_plot.zwerr))
+            ylab=r'Median $[\Delta z]$'
         elif metric=='nmad':
             ydata.append(nmad(df_plot.zwerr))
+            ylab=r'$\sigma_{\rm NMAD} [\Delta z]$'
         elif metric=='outliers':
-            ydata.append(len(df_plot[np.abs(df_plot.zwerr)>0.15])/len(df_plot))
-
-    plt.plot(xlab,ydata, ls = '-', marker = '.', color = 'navy',lw = 1, label = '')
-    plt.ylabel(f'{metric}$[\Delta z]$', fontsize = 18)
-    plt.xlabel(f'{xvariable}', fontsize = 16)
+            ydata.append(len(df_plot[np.abs(df_plot.zwerr)>0.15])/len(df_plot) *100)
+            ylab=r'$\eta$ [%]'
+            
+    if xvariable=='VISmag':
+        xlab='VIS'
+    elif xvariable=='zs':
+        xlab=r'$z_{\rm spec}$'
+    elif xvariable=='z':
+        xlab=r'$z$'
+
+    plt.plot(xdata,ydata, ls = '-', marker = '.', color = 'navy',lw = 1, label = '')
+    plt.ylabel(f'{ylab}', fontsize = 18)
+    plt.xlabel(f'{xlab}', fontsize = 16)
 
     plt.xticks(fontsize = 14)
     plt.yticks(fontsize = 14)

insight/archive.py

@@ -13,7 +13,7 @@ rcParams["font.family"] = "STIXGeneral"
 
 
 class archive():
-    def __init__(self, path, aperture=2, drop_stars=True, clean_photometry=True, convert_colors=True, extinction_corr=True, only_zspec=True, reliable_zspec=True):
+    def __init__(self, path, aperture=2, drop_stars=True, clean_photometry=True, convert_colors=True, extinction_corr=True, only_zspec=True, Qz_cut=1):
         
         self.aperture = aperture
         
@@ -39,30 +39,28 @@ class archive():
         hdu_list = fits.open(os.path.join(path,filename_valid))
         cat_test = Table(hdu_list[1].data).to_pandas()
         
+        self._get_loss_weights(cat)
+        self._get_loss_weights(cat_test)
+        
         gold_sample = pd.read_csv(os.path.join(path,filename_gold))
         
         #cat_test = self._match_gold_sample(cat_test,gold_sample)        
         
         if drop_stars==True:
             cat = cat[cat.mu_class_L07==1]
+            cat_test = cat_test[cat_test.mu_class_L07==1]
 
         if clean_photometry==True:
             cat = self._clean_photometry(cat)
             cat_test = self._clean_photometry(cat_test)
             
-        self._get_loss_weights(cat)
         
         cat = cat[cat.w_Q_f_S15>0]
-            
-        self._set_training_data(cat, only_zspec=only_zspec, reliable_zspec=reliable_zspec, extinction_corr=extinction_corr, convert_colors=convert_colors)
-        
-        
-        self._set_testing_data(cat_test, only_zspec=only_zspec, reliable_zspec='Total', extinction_corr=extinction_corr, convert_colors=convert_colors)
+                    
+        self._set_training_data(cat, only_zspec=only_zspec, extinction_corr=extinction_corr, convert_colors=convert_colors,Qz_cut=Qz_cut)
+        self._set_testing_data(cat_test, only_zspec=only_zspec, extinction_corr=extinction_corr, convert_colors=convert_colors)
         
         self._get_loss_weights(cat)
-        
-        #self.cat_test=cat_test
-        #self.cat_train=cat
             
     def _extract_fluxes(self,catalogue):
         columns_f = [f'FLUX_{x}_{self.aperture}' for x in ['G','R','I','Z','Y','J','H']]
@@ -100,13 +98,9 @@ class archive():
             catalogue = catalogue[catalogue.z_spec_S15>0]
         return catalogue
 
-    def _clean_zspec_sample(self,catalogue ,kind=None):
-        if kind==None:
-            return catalogue
-        elif kind=='Total':
-            return catalogue[catalogue['reliable_S15']>0]
-        elif kind=='Partial':
-            return catalogue[(catalogue['w_Q_f_S15']>0.5)]
+    def _clean_zspec_sample(self,catalogue ,Qz_cut):
+        catalogue = catalogue[catalogue.w_Q_f_S15>=Qz_cut]
+        return catalogue
         
     def _map_weight(self,Qz):
         for key, value in self.weight_dict.items():
@@ -134,11 +128,11 @@ class archive():
         return catalogue_valid
 
     
-    def _set_training_data(self,catalogue, only_zspec=True, reliable_zspec=True, extinction_corr=True, convert_colors=True):
+    def _set_training_data(self,catalogue, only_zspec=True, extinction_corr=True, convert_colors=True,Qz_cut=1):
         
         if only_zspec:
             catalogue = self._take_only_zspec(catalogue, cat_flag='Calib')
-            catalogue = self._clean_zspec_sample(catalogue, kind=reliable_zspec)
+            catalogue = self._clean_zspec_sample(catalogue, Qz_cut=Qz_cut)
             
         self.cat_train=catalogue
         f, ferr = self._extract_fluxes(catalogue)
@@ -159,11 +153,11 @@ class archive():
         self.target_z_train = catalogue['z_spec_S15'].values
         self.target_qz_train = catalogue['w_Q_f_S15'].values
         
-    def _set_testing_data(self,catalogue, only_zspec=True, reliable_zspec=True, extinction_corr=True, convert_colors=True):
+    def _set_testing_data(self,catalogue, only_zspec=True, extinction_corr=True, convert_colors=True):
  
         if only_zspec:
             catalogue = self._take_only_zspec(catalogue, cat_flag='Valid')
-            catalogue = self._clean_zspec_sample(catalogue, kind=reliable_zspec)
+            catalogue = self._clean_zspec_sample(catalogue, Qz_cut=1)
             
         self.cat_test=catalogue
             

insight/insight.py

@@ -8,34 +8,37 @@ from astropy.io import fits
 import os
 from astropy.table import Table
 from scipy.spatial import KDTree
+from scipy.special import erf
 
 class Insight_module():
     """ Define class"""
     
-    def __init__(self, model):
+    def __init__(self, model, batch_size):
         self.model=model
         self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        self.batch_size=batch_size
         
-    def _get_dataloaders(self, input_data, target_data, target_weights, val_fraction=0.1):
+    def _get_dataloaders(self, input_data, target_data, val_fraction=0.1):
         input_data = torch.Tensor(input_data)
         target_data = torch.Tensor(target_data)
-        target_weights = torch.Tensor(target_weights)
 
-        dataset = TensorDataset(input_data, target_data, target_weights)
+        dataset = TensorDataset(input_data, target_data)
 
         trainig_dataset, val_dataset = torch.utils.data.random_split(dataset, [int(len(dataset)*(1-val_fraction)), int(len(dataset)*val_fraction)+1])
-        loader_train = DataLoader(trainig_dataset, batch_size=64, shuffle = True)
+        loader_train = DataLoader(trainig_dataset, batch_size=self.batch_size, shuffle = True)
         loader_val = DataLoader(val_dataset, batch_size=64, shuffle = True)
 
         return loader_train, loader_val
 
+                
 
-    def _loss_function(self,mean, std, logmix, true, target_weights):
-        log_prob =   logmix - 0.5*(mean - true[:,None]).pow(2) / std.pow(2) - torch.log(std)
 
-        log_prob = torch.logsumexp(log_prob, 1)
+    def _loss_function(self,mean, std, logmix, true):
+                
+        logerf = torch.log(erf(true.cpu()[:,None]/(np.sqrt(2)*std.detach().cpu())+1))
         
-        #log_prob = log_prob * target_weights
+        log_prob =   logmix - 0.5*(mean - true[:,None]).pow(2) / std.pow(2) - torch.log(std) #- logerf.to(self.device)
+        log_prob = torch.logsumexp(log_prob, 1)
         loss = -log_prob.mean()
 
         return loss     
@@ -43,21 +46,25 @@ class Insight_module():
     def _to_numpy(self,x):
         return x.detach().cpu().numpy()
         
-    def train(self,input_data, target_data, target_weights,  nepochs=10, val_fraction=0.1, lr=1e-3 ):
+    def train(self,input_data, target_data,  nepochs=10, step_size = 100, val_fraction=0.1, lr=1e-3 ):
         self.model = self.model.train()
-        loader_train, loader_val = self._get_dataloaders(input_data, target_data, target_weights, val_fraction=0.1)
+        loader_train, loader_val = self._get_dataloaders(input_data, target_data, val_fraction=0.1)
         optimizer = optim.Adam(self.model.parameters(), lr=lr, weight_decay=1e-4)
+        scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=step_size, gamma =0.1)
+    
         
         self.model = self.model.to(self.device)
         
-        loss_train, loss_validation = [],[]
+        self.loss_train, self.loss_validation = [],[]
+        
+        
 
         for epoch in range(nepochs):
-            for input_data, target_data, target_weights in loader_train:
+            for input_data, target_data in loader_train:
+                _loss_train, _loss_validation = [],[]
 
                 input_data = input_data.to(self.device)
                 target_data = target_data.to(self.device)
-                target_weights = target_weights.to(self.device)
 
 
                 optimizer.zero_grad()
@@ -69,32 +76,33 @@ class Insight_module():
 
                 #print(mu,sig,target_data,torch.exp(logmix_coeff))
 
-                loss = self._loss_function(mu, sig, logmix_coeff, target_data,target_weights)
+                loss = self._loss_function(mu, sig, logmix_coeff, target_data)
+                _loss_train.append(loss.item())
                 
                 loss.backward()
-                optimizer.step()  
+                optimizer.step()
+            
+            scheduler.step()   
                                 
-            loss_train.append(loss.item())
+            self.loss_train.append(np.mean(_loss_train))
 
-            for input_data, target_data, target_weights in loader_val:
+            for input_data, target_data in loader_val:
 
 
                 input_data = input_data.to(self.device)
                 target_data = target_data.to(self.device)
-                target_weights = target_weights.to(self.device)
 
 
                 mu, logsig, logmix_coeff = self.model(input_data)
                 logsig = torch.clamp(logsig,-6,2)
                 sig = torch.exp(logsig)
 
-                loss_val = self._loss_function(mu, sig, logmix_coeff, target_data, target_weights)
-            loss_validation.append(loss_val.item())
+                loss_val = self._loss_function(mu, sig, logmix_coeff, target_data)
+                _loss_validation.append(loss_val.item())
 
-            print(f'training_loss:{loss}',f'testing_loss:{loss_val}')
-            
-        self.loss_train=loss_train
-        self.loss_validation=loss_validation
+            self.loss_validation.append(np.mean(_loss_validation))
+
+            #print(f'training_loss:{loss}',f'testing_loss:{loss_val}')
            
             
     def get_photoz(self,input_data, target_data):

insight/insight_arch.py

@@ -8,64 +8,37 @@ class Photoz_network(nn.Module):
             nn.Linear(6, 10),
             nn.Dropout(dropout_prob),
             nn.ReLU(),
-            nn.Linear(10, 30),
+            nn.Linear(10, 20),
             nn.Dropout(dropout_prob),
             nn.ReLU(),
-            nn.Linear(30, 50),
+            nn.Linear(20, 50),
             nn.Dropout(dropout_prob),
             nn.ReLU(),
-            nn.Linear(50, 70),
+            nn.Linear(50, 20),
             nn.Dropout(dropout_prob),
             nn.ReLU(),
-            nn.Linear(70, 100)
+            nn.Linear(20, 10)
         )
         
         self.measure_mu = nn.Sequential(
-            nn.Linear(100, 80),
+            nn.Linear(10, 20),
             nn.Dropout(dropout_prob),
             nn.ReLU(),
-            nn.Linear(80, 70),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),        
-            nn.Linear(70, 60),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),
-            nn.Linear(60, 50),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),
-            nn.Linear(50, num_gauss)
+            nn.Linear(20, num_gauss)
         )
 
         self.measure_coeffs = nn.Sequential(
-            nn.Linear(100, 80),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),
-            nn.Linear(80, 70),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),        
-            nn.Linear(70, 60),
+            nn.Linear(10, 20),
             nn.Dropout(dropout_prob),
             nn.ReLU(),
-            nn.Linear(60, 50),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),
-            nn.Linear(50, num_gauss)
+            nn.Linear(20, num_gauss)
         )
 
         self.measure_sigma = nn.Sequential(
-            nn.Linear(100, 80),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),
-            nn.Linear(80, 70),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),        
-            nn.Linear(70, 60),
-            nn.Dropout(dropout_prob),
-            nn.ReLU(),
-            nn.Linear(60, 50),
+            nn.Linear(10, 20),
             nn.Dropout(dropout_prob),
             nn.ReLU(),
-            nn.Linear(50, num_gauss)
+            nn.Linear(20, num_gauss)
         )
         
     def forward(self, x):

insight/utils.py

@@ -8,10 +8,9 @@ def nmad(data):
 
 def sigma68(data): return 0.5*(pd.Series(data).quantile(q = 0.84) - pd.Series(data).quantile(q = 0.16))
 
-
-def plot_photoz(df, nbins,xvariable,metric, type_bin='bin'):
+def plot_photoz_estimates(df, nbins,xvariable,metric, type_bin='bin'):
     bin_edges = stats.mstats.mquantiles(df[xvariable].values, np.linspace(0.1,1,nbins))
-    ydata,xlab = [],[]
+    ydata,xdata = [],[]
     
     
     for k in range(len(bin_edges)-1):
@@ -21,26 +20,37 @@ def plot_photoz(df, nbins,xvariable,metric, type_bin='bin'):
         mean_mag =  (edge_max + edge_min) / 2
         
         if type_bin=='bin':
-            df_plot = df_test[(df_test.imag > edge_min) & (df_test.imag < edge_max)]
+            df_plot = df[(df[xvariable] > edge_min) & (df[xvariable] < edge_max)]
         elif type_bin=='cum':
-            df_plot = df_test[(df_test.imag < edge_max)]
+            df_plot = df[(df[xvariable] < edge_max)]
         else:
             raise ValueError("Only type_bin=='bin' for binned and 'cum' for cumulative are supported")
 
 
-        xlab.append(mean_mag)
+        xdata.append(mean_mag)
         if metric=='sig68':
             ydata.append(sigma68(df_plot.zwerr))
+            ylab=r'$\sigma_{\rm NMAD} [\Delta z]$'
         elif metric=='bias':
-            ydata.append(np.mean(df_plot.zwerr))
+            ydata.append(np.median(df_plot.zwerr))
+            ylab=r'Median $[\Delta z]$'
         elif metric=='nmad':
             ydata.append(nmad(df_plot.zwerr))
+            ylab=r'$\sigma_{\rm NMAD} [\Delta z]$'
         elif metric=='outliers':
-            ydata.append(len(df_plot[np.abs(df_plot.zwerr)>0.15])/len(df_plot))
+            ydata.append(len(df_plot[np.abs(df_plot.zwerr)>0.15])/len(df_plot) *100)
+            ylab=r'$\eta$ [%]'
+            
+    if xvariable=='VISmag':
+        xlab='VIS'
+    elif xvariable=='zs':
+        xlab=r'$z_{\rm spec}$'
+    elif xvariable=='z':
+        xlab=r'$z$'
 
-    plt.plot(xlab,ydata, ls = '-', marker = '.', color = 'navy',lw = 1, label = '')
-    plt.ylabel(f'{metric}$[\Delta z]$', fontsize = 18)
-    plt.xlabel(f'{xvariable}', fontsize = 16)
+    plt.plot(xdata,ydata, ls = '-', marker = '.', color = 'navy',lw = 1, label = '')
+    plt.ylabel(f'{ylab}', fontsize = 18)
+    plt.xlabel(f'{xlab}', fontsize = 16)
 
     plt.xticks(fontsize = 14)
     plt.yticks(fontsize = 14)
@@ -48,4 +58,23 @@ def plot_photoz(df, nbins,xvariable,metric, type_bin='bin'):
     plt.grid(False)
     
     plt.show()
-   
+    
+    return
+   
+
+def plot_nz(df, bins=np.arange(0,5,0.2)):
+    kwargs=dict( bins=bins,alpha=0.5)
+    plt.hist(df.zs.values, color='grey', ls='-' ,**kwargs)
+    counts, _, =np.histogram(df.z.values, bins=bins)
+    
+    plt.plot((bins[:-1]+bins[1:])*0.5,counts, color ='purple')
+    
+    #plt.legend(fontsize=14)
+    plt.xlabel(r'Redshift', fontsize=14)
+    plt.ylabel(r'Counts', fontsize=14)
+    plt.yscale('log')
+    
+    plt.show()
+    
+    return
+