2D Data preprocessing

Version 1

# -*- coding: utf-8 -*-
"""
Created on Wed Jun 12 10:02:27 2019

@author: mhill
"""

"""
preprocessing.py

gets prepared data und maps it to lower dimensions, so the clustering
and visualization gets easier
"""

""" importing stuff """

import time
import umap
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE

""" declarations """

# datafilename = location of your datafile
cell_datafile = 'H:/Internship/ML-Models/Data/2D/cell_data.csv'
nuclei_datafile = 'H:/Internship/ML-Models/Data/2D/nuclei_data.csv'

# save_location = where you want to store your results
save_location = 'H:/Internship/ML-Models/Results/2D/Preprocessing/'

# ICA_iter = max number of iterations for your ICA
ICA_iter = 600

# perplexity parameter for t-SNE
perplexity = range(5,23,3)

# parameters for UMAP
neighbors = [2,10,15,30,100,200]
distance = [0.0,0.1,0.3,0.5,0.9]

""" loading features, data and models """

# loading data
cell_data = pd.read_csv(cell_datafile)
cell_data.drop(['Unnamed: 0'], axis=1, inplace=True)
nuclei_data = pd.read_csv(nuclei_datafile)
nuclei_data.drop(['Unnamed: 0'], axis=1, inplace=True)

# create the dataframes, where the mapped data should be stored
mapped_cell_data = pd.DataFrame()
mapped_nuclei_data = pd.DataFrame()

""" using the models """

# approach 1:
# PCA -> 3 dimensions
pca_3_transformer = PCA(n_components=3)
cell_pca_3 = pca_3_transformer.fit_transform(cell_data.values)
nuclei_pca_3 = pca_3_transformer.fit_transform(nuclei_data.values)

mapped_cell_data['PCA_1'] = cell_pca_3[:,0]
mapped_cell_data['PCA_2'] = cell_pca_3[:,1]
mapped_cell_data['PCA_3'] = cell_pca_3[:,2]

mapped_nuclei_data['PCA_1'] = nuclei_pca_3[:,0]
mapped_nuclei_data['PCA_2'] = nuclei_pca_3[:,1]
mapped_nuclei_data['PCA_3'] = nuclei_pca_3[:,2]

print('Explained variation per principal component - 3dim PCA: {}'.format(pca_3_transformer.explained_variance_ratio_))

# plot the 3D PCA data
ax = plt.figure().gca(projection='3d')
ax.scatter(xs=mapped_cell_data.loc[:,:]["PCA_1"],
    ys=mapped_cell_data.loc[:,:]["PCA_2"],
    zs=mapped_cell_data.loc[:,:]["PCA_3"])
ax.set_xlabel('PCA_1')
ax.set_ylabel('PCA_2')
ax.set_zlabel('PCA_3')
plt.show()

ax = plt.figure().gca(projection='3d')
ax.scatter(xs=mapped_nuclei_data.loc[:,:]["PCA_1"],
    ys=mapped_nuclei_data.loc[:,:]["PCA_2"],
    zs=mapped_nuclei_data.loc[:,:]["PCA_3"])
ax.set_xlabel('PCA_1')
ax.set_ylabel('PCA_2')
ax.set_zlabel('PCA_3')
plt.show()

# approach 2:
# PCA -> 2 dimensions

# plot the 2D PCA data
plt.figure(figsize=(16,10))
sns.scatterplot(x="PCA_1", y="PCA_2", data=mapped_cell_data)
plt.title("Cell Data - PCA - 2 dimensions")
plt.savefig(save_location+'cell_pca_2.png')
plt.close()

plt.figure(figsize=(16,10))
sns.scatterplot(x="PCA_1", y="PCA_2", data=mapped_nuclei_data)
plt.title("Nuclei Data - PCA - 2 dimensions")
plt.savefig(save_location+'nuclei_pca_2.png')
plt.close()

# approach 3 and 4:
# t-sne -> 2 dimensions
# PCA -> 10 dimensions -> t-sne -> 2 dimensions

pca_10_transformer = PCA(n_components=10)
cell_pca_10 = pca_10_transformer.fit_transform(cell_data.values)
nuclei_pca_10 = pca_10_transformer.fit_transform(nuclei_data.values)

# loop over the perplexity parameter
for p in perplexity:

    # create the TSNE mapping
    tsne = TSNE(n_components=2, verbose=1, perplexity=p, n_iter=10000)

    # approach 3
    # t-SNE -> 2 dimensions

    # start a timer
    time_start = time.time()
    # transform the data
    cell_tsne_2 = tsne.fit_transform(cell_data.values)
    print('t-SNE for cells done! Reduced from 18 to 2 dimensions! Time elapsed: {} seconds'.format(time.time()-time_start))

    time_start = time.time()
    nuclei_tsne_2 = tsne.fit_transform(nuclei_data.values)
    print('t-SNE for nuclei done! Reduced from 19 to 2 dimensions! Time elapsed: {} seconds'.format(time.time()-time_start))

    mapped_cell_data['2D_TSNE_1_perplexity_'+str(p)] = cell_tsne_2[:,0]
    mapped_cell_data['2D_TSNE_2_perplexity_'+str(p)] = cell_tsne_2[:,1]

    mapped_nuclei_data['2D_TSNE_1_perplexity_'+str(p)] = nuclei_tsne_2[:,0]
    mapped_nuclei_data['2D_TSNE_2_perplexity_'+str(p)] = nuclei_tsne_2[:,1]

    # plot the 2D t-SNE data
    plt.figure()
    sns.scatterplot(x='2D_TSNE_1_perplexity_'+str(p), y='2D_TSNE_2_perplexity_'+str(p), data=mapped_cell_data)
    plt.xlabel("t-SNE dimension 1", fontsize='large')
    plt.ylabel("t-SNE dimension 2", fontsize='large')
    #plt.title("Cell Data - t-SNE - 2 dimensions - perplexity "+str(p))
    plt.savefig(save_location+'cell_tsne_2_perplexity_'+str(p)+'.png')
    plt.close()

    plt.figure()
    sns.scatterplot(x='2D_TSNE_1_perplexity_'+str(p), y='2D_TSNE_2_perplexity_'+str(p), data=mapped_nuclei_data)
    plt.xlabel("t-SNE dimension 1", fontsize='large')
    plt.ylabel("t-SNE dimension 2", fontsize='large')
    #plt.title("Nuclei Data - t-SNE - 2 dimensions - perplexity "+str(p))
    plt.savefig(save_location+'nuclei_tsne_2_perplexity_'+str(p)+'.png')
    plt.close()

    # approach 4
    # PCA -> 10 dimensions -> t-sne -> 2 dimensions

    # start a timer
    time_start = time.time()
    # transform the data
    cell_pca_10_tsne_2 = tsne.fit_transform(cell_pca_10)
    print('t-SNE for cells done! Reduced from 10 to 2 dimensions! Time elapsed: {} seconds'.format(time.time()-time_start))

    time_start = time.time()
    nuclei_pca_10_tsne_2 = tsne.fit_transform(nuclei_pca_10)
    print('t-SNE nuclei done! Reduced from 10 to 2 dimensions! Time elapsed: {} seconds'.format(time.time()-time_start))

    mapped_cell_data['10D_PCA_2D_TSNE_1_perplexity_'+str(p)] = cell_pca_10_tsne_2[:,0]
    mapped_cell_data['10D_PCA_2D_TSNE_2_perplexity_'+str(p)] = cell_pca_10_tsne_2[:,1]

    mapped_nuclei_data['10D_PCA_2D_TSNE_1_perplexity_'+str(p)] = nuclei_pca_10_tsne_2[:,0]
    mapped_nuclei_data['10D_PCA_2D_TSNE_2_perplexity_'+str(p)] = nuclei_pca_10_tsne_2[:,1]

    # plot the 2D PCA t-SNE data
    plt.figure()
    sns.scatterplot(x="10D_PCA_2D_TSNE_1_perplexity_"+str(p), y="10D_PCA_2D_TSNE_2_perplexity_"+str(p), data=mapped_cell_data)
    plt.xlabel("t-SNE dimension 1", fontsize='large')
    plt.ylabel("t-SNE dimension 2", fontsize='large')
    #plt.title("Cell Data - Combination of PCA and t-SNE - 10 and 2 dimensions - perplexity " +str(p))
    plt.savefig(save_location+'cell_pca_10_tsne_2_perplexity_'+str(p)+'.png')
    plt.close()

    plt.figure()
    sns.scatterplot(x="10D_PCA_2D_TSNE_1_perplexity_"+str(p), y="10D_PCA_2D_TSNE_2_perplexity_"+str(p), data=mapped_nuclei_data)
    plt.xlabel("t-SNE dimension 1", fontsize='large')
    plt.ylabel("t-SNE dimension 2", fontsize='large')
    #plt.title("Nuclei Data - Combination of PCA and t-SNE - 10 and 2 dimensions - perplexity " +str(p))
    plt.savefig(save_location+'nuclei_pca_10_tsne_2_perplexity_'+str(p)+'.png')
    plt.close()

# approach 5:
# UMAP -> 2 dimensions

# loop over the UMAP parameters
for n in neighbors:
    for dist in distance:

        # create the UMAP transformer
        umap_transformer = umap.UMAP(n_neighbors=n,min_dist=dist,n_components=2)

        # transform the data
        cell_umap_2 = umap_transformer.fit_transform(cell_data.values)
        nuclei_umap_2 = umap_transformer.fit_transform(nuclei_data.values)

        mapped_cell_data['2D_UMAP_1_neighbors_'+str(n)+'_dist_'+str(dist)] = cell_umap_2[:,0]
        mapped_cell_data['2D_UMAP_2_neighbors_'+str(n)+'_dist_'+str(dist)] = cell_umap_2[:,1]

        mapped_nuclei_data['2D_UMAP_1_neighbors_'+str(n)+'_dist_'+str(dist)] = nuclei_umap_2[:,0]
        mapped_nuclei_data['2D_UMAP_2_neighbors_'+str(n)+'_dist_'+str(dist)] = nuclei_umap_2[:,1]

        # plot the 2D UMAP data
        plt.figure()
        sns.scatterplot(x="2D_UMAP_1_neighbors_"+str(n)+"_dist_"+str(dist), y="2D_UMAP_2_neighbors_"+str(n)+"_dist_"+str(dist), data= mapped_cell_data)
        plt.xlabel("UMAP dimension 1", fontsize='large')
        plt.ylabel("UMAP dimension 2", fontsize='large')
        #plt.title("Cell Data - UMAP - 2 dimensions - n_neighbors "+str(n)+" - min_dist "+str(dist))
        plt.savefig(save_location+'cell_umap_2_neighbors_'+str(n)+'_dist_'+str(dist)+'.png')
        plt.close()

        plt.figure()
        sns.scatterplot(x="2D_UMAP_1_neighbors_"+str(n)+"_dist_"+str(dist), y="2D_UMAP_2_neighbors_"+str(n)+"_dist_"+str(dist), data= mapped_nuclei_data)
        plt.xlabel("UMAP dimension 1", fontsize='large')
        plt.ylabel("UMAP dimension 2", fontsize='large')
        #plt.title("Nuclei Data - UMAP - 2 dimensions - n_neighbors "+str(n)+" - min_dist "+str(dist))
        plt.savefig(save_location+'nuclei_umap_2_neighbors_'+str(n)+'_dist_'+str(dist)+'.png')
        plt.close()

''' Output '''

mapped_cell_data.to_csv(save_location+'mapped_cell_data.csv')
mapped_nuclei_data.to_csv(save_location+'mapped_nuclei_data.csv')