path: root/scripts/maintenance/DataAnalyzer.py

# Created by Luis Del Mar for GeneNetwork.org
# Email: ladelmar99@gmail.com
# LinkedIn: https://www.linkedin.com/in/luis-del-mar/
# GitHub: https://github.com/ladm99
# This script performs the following normalizations to a specific tab delimited dataset:
# 1. Average 2. Log2 normalize 3. ZScore Normaliza 4. Outlier detection 
#!/usr/bin/env python

from tkinter import Tk
from tkinter.filedialog import askopenfilename
import math
import traceback
import os

# method for file selection
def fileSelector():
	# Code from: https://stackoverflow.com/questions/3579568/choosing-a-file-in-python-with-simple-dialog
	Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing
	filename = askopenfilename() # show an "Open" dialog box and return the path to the selected file
	return filename

# method for getting skipRow and skipCol
def skips():
	while True:
		skipRow = 0
		skipCol = 0
		r = input('Enter the number of rows that you wish skip (default is 0): ').strip()
		c = input('Enter the number of columns that you wish skip (default is 0): ').strip()
		try:
			if r != '':
				skipRow = int(r)
			if c != '':
				skipCol = int(c)
			return skipRow, skipCol
		except Exception as e:
			input('Enter a valid value')

# gets the mean of an array
def getMean(values):
	mean = 0.0
	for i in values:
		mean+=i
	mean /= len(values)
	return mean

# gets the standard deviation of an array
def getSTD(values):
	mean = getMean(values)
	phi = 0.0

	for i in values:
		phi += abs((i - mean)) ** 2
	phi = math.sqrt((phi/(len(values) - 1)))
	return phi


def log2Normalize(inputFile):
	try:
		skipRow, skipCol = skips()
		print(':Log2 normalize processing...looking for the minimal expression value')
		f = open('%s' % inputFile, 'r')
		# set min to max value
		min = float('inf')

		for i in range(skipRow + 1):
			f.readline()
		while True:
			data =f.readline()
			if not data:
				break
			else:
				s = data.split('\t')
				for j in range(skipCol + 1, len(s)):
					value = 0.0
					try:
						value = float(s[j])
						if min > value:
							min = value
					except Exception as e:
						pass
		f.close()
		print('Log2 normalize processing...calculating')
		f = open('%s' % inputFile, 'r')
		outputFile = os.path.split(inputFile)[1].replace('.txt', '_log2.txt')
		out = open(outputFile,'w')

		offset = 1.0
		if min < 0.0:
			offset = -min + 1.0
		for i in range(skipRow + 1):
			out.write(f.readline())
		linenum = 0
		while True:
			data = f.readline()
			if not data:
				break
			else:
				s = data.split('\t')
				for j in range(skipCol + 1):
					out.write(s[j] + '\t')
				for j in range(skipCol + 1, len(s)):
					try:
						value = math.log(float(s[j]) + offset) / math.log(2.0)
						out.write(str(round(value,3)) + '\t')
					except Exception as e:
						out.write('x')
						out.write('\t')
				out.write('\n')
				linenum +=1
				if linenum % 2500 == 0:
					print('Log2 normalize processing...finished' + str(linenum) + ' lines')
		f.close()
		out.close()
		print('Log2 normalize finished')
	except Exception as e:
		print(traceback.format_exc())

def ZScoreNormalize(inputFile):
	try:
		skipRow, skipCol = skips()
		print('ZScore normalize processing...calculating means')
		f = open('%s' % inputFile, 'r')
		for n in range(skipRow):
			f.readline()
		# skip first row which is just headers
		s = f.readline().split('\t')	
		col = len(s) - skipCol - 1
		# mean and phi lists are filled with 0.00
		mean = [0.00] * col
		phi = [0.00] * col

		row = 0

		# read to the end of the file
		while True:
			data = 	f.readline()
			if not data:
				break
			else:
				# put values into a list
				s = data.split('\t')
				for m in range(skipCol + 1, len(s)):
					try:
						mean[m - skipCol - 1] = mean[m - skipCol - 1] + float(s[m])
					except Exception as e:
						pass
				row+=1

		f.close()

		for m in range(col):
			mean[m] = mean[m] / row
		print('ZScore normalize processing...calculating standard divisions')
		f = open('%s' % inputFile, 'r')
		for k in range(skipRow + 1):
			# skip headers
			f.readline()
		while True:
			data = f.readline()
			if not data:
				break
			else:
				s = data.split('\t')
				for i1 in range(skipCol + 1, len(s)):
					value = 0.0
					try:
						value = float(s[i1])
						phi[i1 - skipCol - 1] = phi[i1 - skipCol - 1] + (value - mean[i1 - skipCol - 1]) ** 2
					except Exception as e:
						pass
					
		f.close()
		for j in range(col):
			phi[j] = math.sqrt(phi[j] / (row - 1))
			# print(str(mean[j]) + '\t' + str(phi[j]))

		outputFile = os.path.split(inputFile)[1].replace('.txt', '_Z.txt')
		f = open('%s' % inputFile, 'r')
		out = open(outputFile,'w')

		for i in range(skipRow + 1):
			out.write(f.readline())

		row = 0
		while True:
			data = f.readline()
			if not data:
				break
			else:
				s = data.split('\t')
				for i1 in range(skipRow + 1):
					out.write(s[i1] + '\t')
				for i1 in range(skipCol + 1, len(s)):
					try:
						value = float(s[i1])
						value = 2.0 * (value - mean[i1 - skipCol - 1]) / phi[i1 - skipCol - 1] + 8.0
						out.write(str(round(value, 3)) + '\t')
					except Exception as e:
						out.write('x' + '\t')
					

				out.write('\n')
				row+=1
				if row % 2500 == 0:
					print('ZScore normalize processing...finished ' + str(row) + ' lines')

		f.close()
		out.close()
		print('ZScore normalize finished')
	except Exception as e:
		print(e)

# find the average of values with column names that are the same
def average(inputFile, needSE):
	try:
		skipRow, skipCol = skips()
		f = open('%s' % inputFile, 'r')
		outputFile = os.path.split(inputFile)[1].replace('.txt', '_Avg.txt')
		out = open(outputFile,'w')

		titleList = [] # will hold all of the titles, used for getting indices for values
		itemList = [] # will hold all the unique titles

		# skips the appropriate amount of rows
		for i in range(skipRow):
			s = f.readline()
			out.write(s + '\n')

		s = f.readline().split('\t')
		for i in range(len(s)):
			s[i] = s[i].strip()
			titleList.append(s[i])
			if s[i] not in itemList and i > skipCol:
				itemList.append(s[i])

		for i in range(skipCol + 1):
			out.write(titleList[i])
			out.write('\t')

		for i in range(len(itemList)):
			out.write(itemList[i])
			if i < len(itemList) - 1:
				out.write('\t')
		out.write('\n')

		line = 0
		while True:
			data = f.readline()
			if not data:
				break
			else:
				s = data.split('\t')
				for j in range(skipCol + 1):
					out.write(str(s[j]) + '\t')

				for j in range(len(itemList)):
					avgItemValue = 0.0
					n = 0
					at = titleList.index(itemList[j])
					while(at >= 0):
						try:
							avgItemValue += float(s[at])
							n+=1
							at = titleList.index(itemList[j], at + 1)
						except Exception as e:
							at = -1
					if n == 0:
						out.write('x\t')
					else:
						avgItemValue /= n
						out.write(str(round(avgItemValue, 4)))
					if j < len(itemList) - 1:
						out.write('\t')

				out.write('\n')
				line +=1

				if line % 1000 == 0:
					print('Processing average...' + str(line) + ' lines\n')

		f.close()
		out.close()
	except Exception as e:
		print(traceback.format_exc())

	if needSE:
		getSE(inputFile, skipRow, skipCol)

# find the SE of the average of the values
def getSE(inputFile, skipRow, skipCol):
	try:
		f = open('%s' % inputFile, 'r')
		outputFile = os.path.split(inputFile)[1].replace('.txt', '_Avg_SE.txt')
		out = open(outputFile,'w')

		titleList = [] # will hold all of the titles, used for getting indices for values
		itemList = [] # will hold all the unique titles

		# skips the appropriate amount of rows
		for i in range(skipRow):
			s = f.readline()
			out.write(s + '\n')

		s = f.readline().split('\t')
		for i in range(len(s)):
			s[i] = s[i].strip()
			titleList.append(s[i])
			if s[i] not in itemList and i > skipCol:
				itemList.append(s[i])

		for i in range(skipCol + 1):
			out.write(titleList[i])
			out.write('\t')

		for i in range(len(itemList)):
			out.write(itemList[i])
			if i < len(itemList) - 1:
				out.write('\t')
		out.write('\n')

		line = 0
		while True:
			data = f.readline()
			if not data:
				break
			else:
				s = data.split('\t')
				for j in range(skipCol + 1):
					out.write(str(s[j]) + '\t')

				for j in range(len(itemList)):
					avgItemValue = 0.0
					n = 0
					at = titleList.index(itemList[j])
					while(at >= 0):
						try:
							avgItemValue += float(s[at])
							n+=1
							at = titleList.index(itemList[j], at + 1)
						except Exception as e:
							at = -1
					if n == 0:
						out.write('x\t')
					else:
						avgItemValue /= n
						SE = 0.0
						n = 0
						at = titleList.index(itemList[j])
						while at >= 0:
							try:
								SE += (avgItemValue - float(s[at])) * (avgItemValue - float(s[at]))
								n +=1
								at = titleList.index(itemList[j], at + 1)
							except Exception as e:
								at = -1

						if n > 1:
							SE = math.sqrt(SE / (n - 1))
							SE /= math.sqrt((n-1))
							out.write(str(round(SE, 8)))
							out.write('\t')
						else:
							out.write('\t')

				out.write('\n')
				line +=1

				if line % 1000 == 0:
					print('Processing average... Standard Error... ' + str(line) + ' lines\n')

		f.close()
		out.close()
	except Exception as e:
		print(traceback.format_exc())

# find outliers
def outlier(inputFile):
	try:
		skipRow, skipCol = skips()
		print('Outlier running')
		f = open('%s' % inputFile, 'r')
		outputFile = os.path.split(inputFile)[1].replace('.txt', '_Outlier.txt')
		out = open(outputFile,'w')

		for i in range(skipRow):
			f.readline()
		data = f.readline()
		for i in range(skipCol + 1):
			if data.index('\t') >= 0:
				data = data[data.index('\t'):]
			data = data.strip()

		sampleTitle = data.split('\t')
		sampleNum = len(sampleTitle)
		values = [0.0] * sampleNum
		marks = [0] * sampleNum
		geneNum = 0

		while True:
			data = f.readline()
			if not data:
				break
			else:
				s = data.split('\t')
				for k in range(sampleNum):
					try:
						values[k] = round(float(s[k + skipCol + 1]), 3)

					except Exception as e:
						pass

				mean = getMean(values)
				phi = getSTD(values)
				for k in range(sampleNum):
					if values[k] < (mean - 2.0 * phi) or values[k] > (mean + 2.0 * phi):
						marks[k] += 1
				
				geneNum += 1	
				if geneNum % 1000 == 0:
					print('Outlier running, finished...' + str(geneNum) + ' lines')

		for j in range(sampleNum):
			out.write(sampleTitle[j] + '\t')
			out.write(str(marks[j]) + '\t')
			out.write(str(round(marks[j] * 1.0 / geneNum, 3)))
			out.write('\n')
		out.close
		print('Outlier finished')

	except Exception as e:
		print(traceback.format_exc())

# adds dashes based on how long the filename is, just here to make the selection a little bit nicer
def dashes(string):
	dash = '-------------------------'
	for i in range(len(string)):
		dash+='-'
	return '\n' + dash

def main():
	input('Press Enter to select file')
	filename = fileSelector()

	while True:
		file = os.path.split(filename)[1]
		print('\nEnter Selection for file ' +  file + dashes(file))
		print('1. Average\n2. Log2 Normalize\n3. ZScore Normalize\n4. Outlier\n5. Select a new file\n6. Exit')
		select = input('Enter: ')

		if select == '1':
			needSE = False
			SE = input('Do you want to compute the standard error [Y/N] (default is no)').lower()
			if SE == 'y':
				needSE = True
			average(filename, needSE)
		elif select == '2':
			log2Normalize(filename)
		elif select == '3':
			ZScoreNormalize(filename)
		elif select == '4':
			outlier(filename)
		elif select == '5':
			filename = fileSelector()
		elif select == '6':
			exit()




main()