notebooks/Colourspace.py

# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.16.2
#   kernelspec:
#     display_name: temps
#     language: python
#     name: temps
# ---

# # FIGURE COLOURSPACE IN THE PAPER

# %load_ext autoreload
# %autoreload 2

import pandas as pd
import numpy as np
import os
from astropy.io import fits
from astropy.table import Table
import torch
from pathlib import Path

#matplotlib settings
from matplotlib import rcParams
import matplotlib.pyplot as plt
rcParams["mathtext.fontset"] = "stix"
rcParams["font.family"] = "STIXGeneral"

from temps.archive import Archive 
from temps.utils import nmad
from temps.temps_arch import EncoderPhotometry, MeasureZ
from temps.temps import TempsModule


def estimate_som_map(df, plot_arg='z', nx=40, ny=40):
    """
    Estimate a Self-Organizing Map (SOM) visualization from a DataFrame.

    Parameters:
    - df (pandas.DataFrame): Input DataFrame containing data for SOM estimation.
    - plot_arg (str, optional): Column name to be used for plotting. Default is 'z'.
    - nx (int, optional): Number of cells along the X-axis. Default is 40.
    - ny (int, optional): Number of cells along the Y-axis. Default is 40.

    Returns:
    - som_data (numpy.ndarray): Estimated SOM visualization data.
    """
    x_cells = np.arange(0, nx)
    y_cells = np.arange(0, ny)
    index_cell = np.arange(nx * ny)
    cells = np.array(np.meshgrid(x_cells, y_cells)).T.reshape(-1, 2)
    cells = pd.DataFrame(np.c_[cells[:, 0], cells[:, 1], index_cell], columns=['x_cell', 'y_cell', 'cell'])

    if plot_arg == 'count':
        som_vis = df.groupby('cell')['z'].count().reset_index().rename(columns={f'z': 'plot_som'})
    else:
        som_vis = df.groupby('cell')[f'{plot_arg}'].mean().reset_index().rename(columns={f'{plot_arg}': 'plot_som'})

    som_data = som_vis.merge(cells, on='cell')
    som_data = som_data.pivot(index='x_cell', columns='y_cell', values='plot_som')

    return som_data



def plot_som_map(som_data, plot_arg = 'z', vmin=0, vmax=1):
    """
    Plot the Self-Organizing Map (SOM) data.

    Parameters:
    - som_data (numpy.ndarray): The SOM data to be visualized.
    - plot_arg (str, optional): The column name to be plotted. Default is 'z'.
    - vmin (float, optional): Minimum value for color scaling. Default is 0.
    - vmax (float, optional): Maximum value for color scaling. Default is 1.

    Returns:
    None
    """
    plt.imshow(som_data, vmin=vmin, vmax=vmax, cmap='viridis')  # Choose an appropriate colormap
    plt.colorbar(label=f'{plot_arg}')  # Add a colorbar with a label
    plt.xlabel(r'$x$ [pixel]', fontsize=14)  # Add an appropriate X-axis label
    plt.ylabel(r'$y$ [pixel]', fontsize=14)  # Add an appropriate Y-axis label
    plt.show()



# ### LOAD DATA

#define here the directory containing the photometric catalogues
parent_dir = Path('/data/astro/scratch/lcabayol/insight/data/Euclid_EXT_MER_PHZ_DC2_v1.5')
modules_dir = Path('../data/models/')
filename_calib = 'euclid_cosmos_DC2_S1_v2.1_calib_clean.fits'
filename_valid = 'euclid_cosmos_DC2_S1_v2.1_valid_matched.fits'

# +
filename_valid='euclid_cosmos_DC2_S1_v2.1_valid_matched.fits'

hdu_list = fits.open(parent_dir/filename_valid)
cat = Table(hdu_list[1].data).to_pandas()
cat = cat[cat['FLAG_PHOT']==0]
cat = cat[cat['mu_class_L07']==1]
cat = cat[(cat['z_spec_S15'] > 0) | (cat['photo_z_L15'] > 0)]
cat = cat[cat['MAG_VIS']<25]

# -

ztarget = [cat['z_spec_S15'].values[ii] if cat['z_spec_S15'].values[ii]> 0 else cat['photo_z_L15'].values[ii] for ii in range(len(cat))]
specz_or_photo = [0 if cat['z_spec_S15'].values[ii]> 0 else 1 for ii in range(len(cat))]
ID = cat['ID']
VISmag = cat['MAG_VIS']
zsflag = cat['reliable_S15']

photoz_archive = Archive(path_calib = parent_dir/filename_calib, 
                         path_valid = parent_dir/filename_valid,
                         only_zspec=False)
f = photoz_archive._extract_fluxes(catalogue= cat)
col = photoz_archive._to_colors(f)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# +
dfs = {}

for il, lab in enumerate(['z','L15','DA']):

    nn_features = EncoderPhotometry()
    nn_features.load_state_dict(torch.load(modules_dir / f'modelF_{lab}.pt',map_location=torch.device('cpu')))
    nn_z = MeasureZ(num_gauss=6)
    nn_z.load_state_dict(torch.load(modules_dir / f'modelZ_{lab}.pt',map_location=torch.device('cpu')))
    
    temps_module = TempsModule(nn_features, nn_z)
    
    z, pz, odds = temps_module.get_pz(input_data=torch.Tensor(col), 
                                return_pz=True)
    # Create a DataFrame with the desired columns
    df = pd.DataFrame(np.c_[ID, VISmag,z,odds, ztarget,zsflag, specz_or_photo], 
                      columns=['ID','VISmag','z', 'odds','ztarget','zsflag','S15_L15_flag'])

    # Calculate additional columns or operations if needed
    df['zwerr'] = (df.z - df.ztarget) / (1 + df.ztarget)

    # Drop any rows with NaN values
    df = df.dropna()

    # Assign the DataFrame to a key in the dictionary
    dfs[lab] = df

# -

# ### LOAD TRAINED MODELS AND EVALUATE PDFs AND REDSHIFT

#define here the directory containing the photometric catalogues
parent_dir = Path('/data/astro/scratch/lcabayol/insight/data/Euclid_EXT_MER_PHZ_DC2_v1.5')
modules_dir = Path('../data/models/')

df_z = dfs['z']
df_z_DA = dfs['DA']

# ##### LOAD TRAIN SOM ON TRAINING DATA

df_som = pd.read_csv(parent_dir/'som_dataframe.csv', header = 0, sep =',')
df_z = df_z.merge(df_som, on = 'ID')
df_z_DA = df_z_DA.merge(df_som, on = 'ID')

# ##### APPLY CUTS FOR DIFFERENT SAMPLES

df_zspec = df_z[(df_z.S15_L15_flag==0) & (df_z.zsflag==1)]
df_l15 = df_z[(df_z.ztarget>0)]
df_l15_DA = df_z_DA[(df_z_DA.ztarget>0)]

df_l15_euclid = df_z[(df_z.VISmag <24.5) & (df_z.z > 0.2) & (df_z.z < 2.6)]
df_l15_euclid_cut= df_l15_euclid[df_l15_euclid.odds>df_l15_euclid['odds'].quantile(0.2)]

df_l15_euclid_da = df_z_DA[(df_z_DA.VISmag <24.5) & (df_z_DA.z > 0.2) & (df_z_DA.z < 2.6)]
df_l15_euclid_cut_da= df_l15_euclid_da[df_l15_euclid_da.odds>df_l15_euclid['odds'].quantile(0.2)]

# ## MAKE SOM PLOT

from mpl_toolkits.axes_grid1 import make_axes_locatable

# +
fig, axs = plt.subplots(6, 4, figsize=(13, 15), sharex=True, sharey=True, gridspec_kw={'hspace': 0.05, 'wspace': 0.06})

# Plot in the top row (axs[0, i])
#top row, spectroscopic sample
columns = ['ztarget','z','zwerr','count']
titles = [r'$z_{true}$ (A)',r'$z$ (B)',r'$z_{\rm error}$ (C)','Counts']
limits = [[0,4],[0,4],[-0.5,0.5],[0,50]]
for ii in range(4):
    som_data = estimate_som_map(df_zspec, plot_arg=columns[ii], nx=40, ny=40)
    im = axs[0,ii].imshow(som_data, vmin=limits[ii][0], vmax=limits[ii][1], cmap='viridis')  # Choose an appropriate colormap
    axs[0, ii].set_title(f'{titles[ii]}', fontsize=18)
    
    if ii==0:
        axs[0, 0].set_ylabel(r'$y$', fontsize=14)
    elif ii==1:
        cbar_ax = fig.add_axes([0.49, 0.11, 0.01, 0.77])
        fig.colorbar(im, cax=cbar_ax)
    elif ii==2:
        cbar_ax = fig.add_axes([0.685, 0.11, 0.01, 0.77])
        fig.colorbar(im, cax=cbar_ax)
    elif ii==3:
        cbar_ax = fig.add_axes([0.885, 0.11, 0.01, 0.77])
        fig.colorbar(im, cax=cbar_ax)

for jj in range(4):
    som_data = estimate_som_map(df_l15, plot_arg=columns[jj], nx=40, ny=40)
    im = axs[1,jj].imshow(som_data, vmin=limits[jj][0], vmax=limits[jj][1], cmap='viridis')  # Choose an appropriate colormap
    #axs[1, jj].set_title(f'{titles[jj]}', fontsize=14)
    #axs[1, jj].set_xlabel(r'$x$', fontsize=14)
    
    
for kk in range(4):
    som_data = estimate_som_map(df_l15_DA, plot_arg=columns[kk], nx=40, ny=40)
    im = axs[2,kk].imshow(som_data, vmin=limits[kk][0], vmax=limits[kk][1], cmap='viridis')  # Choose an appropriate colormap
    #axs[2, kk].set_title(f'{titles[kk]}', fontsize=14)
    #axs[2, kk].set_xlabel(r'$x$', fontsize=14)
    
for rr in range(4):
    som_data = estimate_som_map(df_l15_euclid_da, plot_arg=columns[rr], nx=40, ny=40)
    im = axs[3,rr].imshow(som_data, vmin=limits[rr][0], vmax=limits[rr][1], cmap='viridis')  # Choose an appropriate colormap
    #axs[3, rr].set_title(f'{titles[rr]}', fontsize=14)
    #axs[3, rr].set_xlabel(r'$x$', fontsize=14)
    
for ll in range(4):
    som_data = estimate_som_map(df_l15_euclid_cut, plot_arg=columns[ll], nx=40, ny=40)
    im = axs[4,ll].imshow(som_data, vmin=limits[ll][0], vmax=limits[ll][1], cmap='viridis')  # Choose an appropriate colormap
    #axs[4, ll].set_title(f'{titles[ll]}', fontsize=14)
    axs[4, ll].set_xlabel(r'$x$', fontsize=14)
    
for ll in range(4):
    som_data = estimate_som_map(df_l15_euclid_cut_da, plot_arg=columns[ll], nx=40, ny=40)
    im = axs[5,ll].imshow(som_data, vmin=limits[ll][0], vmax=limits[ll][1], cmap='viridis')  # Choose an appropriate colormap
    #axs[4, ll].set_title(f'{titles[ll]}', fontsize=14)
    axs[5, ll].set_xlabel(r'$x$', fontsize=14)

    
axs[0, 0].set_ylabel(r'$y$', fontsize=14)
axs[1, 0].set_ylabel(r'$y$', fontsize=14)
axs[2, 0].set_ylabel(r'$y$', fontsize=14)
axs[3, 0].set_ylabel(r'$y$', fontsize=14)
axs[4, 0].set_ylabel(r'$y$', fontsize=14)
axs[5, 0].set_ylabel(r'$y$', fontsize=14)


fig.text(0.09, 0.815, r'$z_{\rm s}$ samp. (1)', va='center', rotation='vertical', fontsize=16)
fig.text(0.09, 0.69, r'L15 samp. (2)', va='center', rotation='vertical', fontsize=16)
fig.text(0.09, 0.56, r'L15 samp. + DA (3)', va='center', rotation='vertical', fontsize=14)
fig.text(0.09, 0.44, r'$Euclid$ samp. + DA (4)', va='center', rotation='vertical', fontsize=14)
fig.text(0.09, 0.3, r'$Euclid$ samp. + QC (5)', va='center', rotation='vertical', fontsize=14)

fig.text(0.09, 0.17, r'(5) + DA ', va='center', rotation='vertical', fontsize=13)


plt.savefig('SOM_colourspace.pdf', format='pdf', bbox_inches='tight', dpi=300)

# -