Add DataAnalyzer.py script to maintenance scripts

1 files changed, 470 insertions, 0 deletions

diff --git a/scripts/maintenance/DataAnalyzer.py b/scripts/maintenance/DataAnalyzer.py
new file mode 100644
index 00000000..6d14c8da
--- /dev/null
+++ b/scripts/maintenance/DataAnalyzer.py
@@ -0,0 +1,470 @@
+# Created by Luis Del Mar for GeneNetwork.org

+# Email: ladelmar99@gmail.com

+# LinkedIn: https://www.linkedin.com/in/luis-del-mar/

+# GitHub: https://github.com/ladm99

+# This script performs the following normalizations to a specific tab delimited dataset:

+# 1. Average 2. Log2 normalize 3. ZScore Normaliza 4. Outlier detection 

+#!/usr/bin/env python

+

+from tkinter import Tk

+from tkinter.filedialog import askopenfilename

+import math

+import traceback

+import os

+

+# method for file selection

+def fileSelector():

+	# Code from: https://stackoverflow.com/questions/3579568/choosing-a-file-in-python-with-simple-dialog

+	Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing

+	filename = askopenfilename() # show an "Open" dialog box and return the path to the selected file

+	return filename

+

+# method for getting skipRow and skipCol

+def skips():

+	while True:

+		skipRow = 0

+		skipCol = 0

+		r = input('Enter the number of rows that you wish skip (default is 0): ').strip()

+		c = input('Enter the number of columns that you wish skip (default is 0): ').strip()

+		try:

+			if r != '':

+				skipRow = int(r)

+			if c != '':

+				skipCol = int(c)

+			return skipRow, skipCol

+		except Exception as e:

+			input('Enter a valid value')

+

+# gets the mean of an array

+def getMean(values):

+	mean = 0.0

+	for i in values:

+		mean+=i

+	mean /= len(values)

+	return mean

+

+# gets the standard deviation of an array

+def getSTD(values):

+	mean = getMean(values)

+	phi = 0.0

+

+	for i in values:

+		phi += abs((i - mean)) ** 2

+	phi = math.sqrt((phi/(len(values) - 1)))

+	return phi

+

+

+def log2Normalize(inputFile):

+	try:

+		skipRow, skipCol = skips()

+		print(':Log2 normalize processing...looking for the minimal expression value')

+		f = open('%s' % inputFile, 'r')

+		# set min to max value

+		min = float('inf')

+

+		for i in range(skipRow + 1):

+			f.readline()

+		while True:

+			data =f.readline()

+			if not data:

+				break

+			else:

+				s = data.split('\t')

+				for j in range(skipCol + 1, len(s)):

+					value = 0.0

+					try:

+						value = float(s[j])

+						if min > value:

+							min = value

+					except Exception as e:

+						pass

+		f.close()

+		print('Log2 normalize processing...calculating')

+		f = open('%s' % inputFile, 'r')

+		outputFile = os.path.split(inputFile)[1].replace('.txt', '_log2.txt')

+		out = open(outputFile,'w')

+

+		offset = 1.0

+		if min < 0.0:

+			offset = -min + 1.0

+		for i in range(skipRow + 1):

+			out.write(f.readline())

+		linenum = 0

+		while True:

+			data = f.readline()

+			if not data:

+				break

+			else:

+				s = data.split('\t')

+				for j in range(skipCol + 1):

+					out.write(s[j] + '\t')

+				for j in range(skipCol + 1, len(s)):

+					try:

+						value = math.log(float(s[j]) + offset) / math.log(2.0)

+						out.write(str(round(value,3)) + '\t')

+					except Exception as e:

+						out.write('x')

+						out.write('\t')

+				out.write('\n')

+				linenum +=1

+				if linenum % 2500 == 0:

+					print('Log2 normalize processing...finished' + str(linenum) + ' lines')

+		f.close()

+		out.close()

+		print('Log2 normalize finished')

+	except Exception as e:

+		print(traceback.format_exc())

+

+def ZScoreNormalize(inputFile):

+	try:

+		skipRow, skipCol = skips()

+		print('ZScore normalize processing...calculating means')

+		f = open('%s' % inputFile, 'r')

+		for n in range(skipRow):

+			f.readline()

+		# skip first row which is just headers

+		s = f.readline().split('\t')	

+		col = len(s) - skipCol - 1

+		# mean and phi lists are filled with 0.00

+		mean = [0.00] * col

+		phi = [0.00] * col

+

+		row = 0

+

+		# read to the end of the file

+		while True:

+			data = 	f.readline()

+			if not data:

+				break

+			else:

+				# put values into a list

+				s = data.split('\t')

+				for m in range(skipCol + 1, len(s)):

+					try:

+						mean[m - skipCol - 1] = mean[m - skipCol - 1] + float(s[m])

+					except Exception as e:

+						pass

+				row+=1

+

+		f.close()

+

+		for m in range(col):

+			mean[m] = mean[m] / row

+		print('ZScore normalize processing...calculating standard divisions')

+		f = open('%s' % inputFile, 'r')

+		for k in range(skipRow + 1):

+			# skip headers

+			f.readline()

+		while True:

+			data = f.readline()

+			if not data:

+				break

+			else:

+				s = data.split('\t')

+				for i1 in range(skipCol + 1, len(s)):

+					value = 0.0

+					try:

+						value = float(s[i1])

+						phi[i1 - skipCol - 1] = phi[i1 - skipCol - 1] + (value - mean[i1 - skipCol - 1]) ** 2

+					except Exception as e:

+						pass

+					

+		f.close()

+		for j in range(col):

+			phi[j] = math.sqrt(phi[j] / (row - 1))

+			# print(str(mean[j]) + '\t' + str(phi[j]))

+

+		outputFile = os.path.split(inputFile)[1].replace('.txt', '_Z.txt')

+		f = open('%s' % inputFile, 'r')

+		out = open(outputFile,'w')

+

+		for i in range(skipRow + 1):

+			out.write(f.readline())

+

+		row = 0

+		while True:

+			data = f.readline()

+			if not data:

+				break

+			else:

+				s = data.split('\t')

+				for i1 in range(skipRow + 1):

+					out.write(s[i1] + '\t')

+				for i1 in range(skipCol + 1, len(s)):

+					try:

+						value = float(s[i1])

+						value = 2.0 * (value - mean[i1 - skipCol - 1]) / phi[i1 - skipCol - 1] + 8.0

+						out.write(str(round(value, 3)) + '\t')

+					except Exception as e:

+						out.write('x' + '\t')

+					

+

+				out.write('\n')

+				row+=1

+				if row % 2500 == 0:

+					print('ZScore normalize processing...finished ' + str(row) + ' lines')

+

+		f.close()

+		out.close()

+		print('ZScore normalize finished')

+	except Exception as e:

+		print(e)

+

+# find the average of values with column names that are the same

+def average(inputFile, needSE):

+	try:

+		skipRow, skipCol = skips()

+		f = open('%s' % inputFile, 'r')

+		outputFile = os.path.split(inputFile)[1].replace('.txt', '_Avg.txt')

+		out = open(outputFile,'w')

+

+		titleList = [] # will hold all of the titles, used for getting indices for values

+		itemList = [] # will hold all the unique titles

+

+		# skips the appropriate amount of rows

+		for i in range(skipRow):

+			s = f.readline()

+			out.write(s + '\n')

+

+		s = f.readline().split('\t')

+		for i in range(len(s)):

+			s[i] = s[i].strip()

+			titleList.append(s[i])

+			if s[i] not in itemList and i > skipCol:

+				itemList.append(s[i])

+

+		for i in range(skipCol + 1):

+			out.write(titleList[i])

+			out.write('\t')

+

+		for i in range(len(itemList)):

+			out.write(itemList[i])

+			if i < len(itemList) - 1:

+				out.write('\t')

+		out.write('\n')

+

+		line = 0

+		while True:

+			data = f.readline()

+			if not data:

+				break

+			else:

+				s = data.split('\t')

+				for j in range(skipCol + 1):

+					out.write(str(s[j]) + '\t')

+

+				for j in range(len(itemList)):

+					avgItemValue = 0.0

+					n = 0

+					at = titleList.index(itemList[j])

+					while(at >= 0):

+						try:

+							avgItemValue += float(s[at])

+							n+=1

+							at = titleList.index(itemList[j], at + 1)

+						except Exception as e:

+							at = -1

+					if n == 0:

+						out.write('x\t')

+					else:

+						avgItemValue /= n

+						out.write(str(round(avgItemValue, 4)))

+					if j < len(itemList) - 1:

+						out.write('\t')

+

+				out.write('\n')

+				line +=1

+

+				if line % 1000 == 0:

+					print('Processing average...' + str(line) + ' lines\n')

+

+		f.close()

+		out.close()

+	except Exception as e:

+		print(traceback.format_exc())

+

+	if needSE:

+		getSE(inputFile, skipRow, skipCol)

+

+# find the SE of the average of the values

+def getSE(inputFile, skipRow, skipCol):

+	try:

+		f = open('%s' % inputFile, 'r')

+		outputFile = os.path.split(inputFile)[1].replace('.txt', '_Avg_SE.txt')

+		out = open(outputFile,'w')

+

+		titleList = [] # will hold all of the titles, used for getting indices for values

+		itemList = [] # will hold all the unique titles

+

+		# skips the appropriate amount of rows

+		for i in range(skipRow):

+			s = f.readline()

+			out.write(s + '\n')

+

+		s = f.readline().split('\t')

+		for i in range(len(s)):

+			s[i] = s[i].strip()

+			titleList.append(s[i])

+			if s[i] not in itemList and i > skipCol:

+				itemList.append(s[i])

+

+		for i in range(skipCol + 1):

+			out.write(titleList[i])

+			out.write('\t')

+

+		for i in range(len(itemList)):

+			out.write(itemList[i])

+			if i < len(itemList) - 1:

+				out.write('\t')

+		out.write('\n')

+

+		line = 0

+		while True:

+			data = f.readline()

+			if not data:

+				break

+			else:

+				s = data.split('\t')

+				for j in range(skipCol + 1):

+					out.write(str(s[j]) + '\t')

+

+				for j in range(len(itemList)):

+					avgItemValue = 0.0

+					n = 0

+					at = titleList.index(itemList[j])

+					while(at >= 0):

+						try:

+							avgItemValue += float(s[at])

+							n+=1

+							at = titleList.index(itemList[j], at + 1)

+						except Exception as e:

+							at = -1

+					if n == 0:

+						out.write('x\t')

+					else:

+						avgItemValue /= n

+						SE = 0.0

+						n = 0

+						at = titleList.index(itemList[j])

+						while at >= 0:

+							try:

+								SE += (avgItemValue - float(s[at])) * (avgItemValue - float(s[at]))

+								n +=1

+								at = titleList.index(itemList[j], at + 1)

+							except Exception as e:

+								at = -1

+

+						if n > 1:

+							SE = math.sqrt(SE / (n - 1))

+							SE /= math.sqrt((n-1))

+							out.write(str(round(SE, 8)))

+							out.write('\t')

+						else:

+							out.write('\t')

+

+				out.write('\n')

+				line +=1

+

+				if line % 1000 == 0:

+					print('Processing average... Standard Error... ' + str(line) + ' lines\n')

+

+		f.close()

+		out.close()

+	except Exception as e:

+		print(traceback.format_exc())

+

+# find outliers

+def outlier(inputFile):

+	try:

+		skipRow, skipCol = skips()

+		print('Outlier running')

+		f = open('%s' % inputFile, 'r')

+		outputFile = os.path.split(inputFile)[1].replace('.txt', '_Outlier.txt')

+		out = open(outputFile,'w')

+

+		for i in range(skipRow):

+			f.readline()

+		data = f.readline()

+		for i in range(skipCol + 1):

+			if data.index('\t') >= 0:

+				data = data[data.index('\t'):]

+			data = data.strip()

+

+		sampleTitle = data.split('\t')

+		sampleNum = len(sampleTitle)

+		values = [0.0] * sampleNum

+		marks = [0] * sampleNum

+		geneNum = 0

+

+		while True:

+			data = f.readline()

+			if not data:

+				break

+			else:

+				s = data.split('\t')

+				for k in range(sampleNum):

+					try:

+						values[k] = round(float(s[k + skipCol + 1]), 3)

+

+					except Exception as e:

+						pass

+

+				mean = getMean(values)

+				phi = getSTD(values)

+				for k in range(sampleNum):

+					if values[k] < (mean - 2.0 * phi) or values[k] > (mean + 2.0 * phi):

+						marks[k] += 1

+				

+				geneNum += 1	

+				if geneNum % 1000 == 0:

+					print('Outlier running, finished...' + str(geneNum) + ' lines')

+

+		for j in range(sampleNum):

+			out.write(sampleTitle[j] + '\t')

+			out.write(str(marks[j]) + '\t')

+			out.write(str(round(marks[j] * 1.0 / geneNum, 3)))

+			out.write('\n')

+		out.close

+		print('Outlier finished')

+

+	except Exception as e:

+		print(traceback.format_exc())

+

+# adds dashes based on how long the filename is, just here to make the selection a little bit nicer

+def dashes(string):

+	dash = '-------------------------'

+	for i in range(len(string)):

+		dash+='-'

+	return '\n' + dash

+

+def main():

+	input('Press Enter to select file')

+	filename = fileSelector()

+

+	while True:

+		file = os.path.split(filename)[1]

+		print('\nEnter Selection for file ' +  file + dashes(file))

+		print('1. Average\n2. Log2 Normalize\n3. ZScore Normalize\n4. Outlier\n5. Select a new file\n6. Exit')

+		select = input('Enter: ')

+

+		if select == '1':

+			needSE = False

+			SE = input('Do you want to compute the standard error [Y/N] (default is no)').lower()

+			if SE == 'y':

+				needSE = True

+			average(filename, needSE)

+		elif select == '2':

+			log2Normalize(filename)

+		elif select == '3':

+			ZScoreNormalize(filename)

+		elif select == '4':

+			outlier(filename)

+		elif select == '5':

+			filename = fileSelector()

+		elif select == '6':

+			exit()

+

+

+

+

+main()
\ No newline at end of file