From 1a5ac67c8ac0a47deeea6fe2d1c4f59f94793a6c Mon Sep 17 00:00:00 2001 From: Zachary Sloan Date: Thu, 30 Jan 2014 18:44:01 +0000 Subject: The calculations for the manhattan plot are now run from the command line as a background process, allowing multiple to be run at the same time without the time increasing. --- .../wqflask/marker_regression/marker_regression.py | 24 +++-- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 101 +++++++++++---------- 2 files changed, 67 insertions(+), 58 deletions(-) diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 1594d35d..3435d0db 100755 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -117,8 +117,8 @@ class MarkerRegression(object): Redis.expire(key, 60*60) print("before printing command") - command = 'python /home/zas1024/gene/wqflask/wqflask/my_pylmm/pyLMM/lmm.py {} {}'.format(key, - "non-human") + command = 'python /home/zas1024/gene/wqflask/wqflask/my_pylmm/pyLMM/lmm.py --key {} --species {}'.format(key, + "other") print("command is:", command) print("after printing command") @@ -128,11 +128,9 @@ class MarkerRegression(object): #lmm.run(key) json_results = Redis.blpop("pylmm:results:" + temp_uuid, 45*60) - results = json.load(json_results) - t_stats = results['t_stats'] + results = json.loads(json_results[1]) p_values = results['p_values'] - - print("p_values:", p_values) + t_stats = results['t_stats'] #t_stats, p_values = lmm.run( # pheno_vector, @@ -169,7 +167,7 @@ class MarkerRegression(object): temp_uuid = temp_uuid, # meta data - timestamp = datetime.datetime.isoformat(), + timestamp = datetime.datetime.now().isoformat(), ) print("After creating params") @@ -180,14 +178,20 @@ class MarkerRegression(object): print("Before creating the command") - command = 'python /home/zas1024/gene/wqflask/wqflask/my_pylmm/pyLMM/lmm.py {} {}'.format(key, - "non-human") + command = 'python /home/zas1024/gene/wqflask/wqflask/my_pylmm/pyLMM/lmm.py --key {} --species {}'.format(key, + "human") print("command is:", command) os.system(command) + + json_results = Redis.blpop("pylmm:results:" + temp_uuid, 45*60) + results = json.loads(json_results[1]) + t_stats = results['t_stats'] + p_values = results['p_values'] + - p_values, t_stats = lmm.run_human(key) + #p_values, t_stats = lmm.run_human(key) #p_values, t_stats = lmm.run_human( # pheno_vector, diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 2560aa9e..2345e132 100755 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -38,13 +38,18 @@ import simplejson as json from pprint import pformat as pf +from redis import Redis +Redis = Redis() + +import sys +sys.path.append("/home/zas1024/gene/wqflask/") +print("sys.path2:", sys.path) + from utility.benchmark import Bench from utility import temp_data from wqflask.my_pylmm.pyLMM import chunks -from redis import Redis -Redis = Redis() #np.seterr('raise') @@ -68,17 +73,17 @@ def run_human(pheno_vector, identifier = str(uuid.uuid4()) - print("pheno_vector: ", pf(pheno_vector)) - print("kinship_matrix: ", pf(kinship_matrix)) - print("kinship_matrix.shape: ", pf(kinship_matrix.shape)) - - lmm_vars = pickle.dumps(dict( - pheno_vector = pheno_vector, - covariate_matrix = covariate_matrix, - kinship_matrix = kinship_matrix - )) - Redis.hset(identifier, "lmm_vars", lmm_vars) - Redis.expire(identifier, 60*60) + #print("pheno_vector: ", pf(pheno_vector)) + #print("kinship_matrix: ", pf(kinship_matrix)) + #print("kinship_matrix.shape: ", pf(kinship_matrix.shape)) + + #lmm_vars = pickle.dumps(dict( + # pheno_vector = pheno_vector, + # covariate_matrix = covariate_matrix, + # kinship_matrix = kinship_matrix + #)) + #Redis.hset(identifier, "lmm_vars", lmm_vars) + #Redis.expire(identifier, 60*60) if v.sum(): pheno_vector = pheno_vector[keep] @@ -696,44 +701,44 @@ class LMM: pl.ylabel("Probability of data") pl.title(title) - def main(): - parser = argparse.ArgumentParser(description='Run pyLMM') - parser.add_argument('-k', '--key') - - opts = parser.parse_args() - - key = opts.key - - json_params = Redis.get(key) - - params = json.loads(json_params) - print("params:", params) +def main(): + parser = argparse.ArgumentParser(description='Run pyLMM') + parser.add_argument('-k', '--key') + parser.add_argument('-s', '--species') + + opts = parser.parse_args() + + key = opts.key + species = opts.species + + json_params = Redis.get(key) + + params = json.loads(json_params) + print("params:", params) - is_human = params['human'] + tempdata = temp_data.TempData(params['temp_uuid']) + if species == "human" : + ps, ts = run_human(pheno_vector = np.array(params['pheno_vector']), + covariate_matrix = np.array(params['covariate_matrix']), + plink_input_file = params['input_file_name'], + kinship_matrix = np.array(params['kinship_matrix']), + refit = params['refit'], + tempdata = tempdata) + else: + ps, ts = run_other(pheno_vector = np.array(params['pheno_vector']), + genotype_matrix = np.array(params['genotype_matrix']), + restricted_max_likelihood = params['restricted_max_likelihood'], + refit = params['refit'], + tempdata = tempdata) - tempdata = temp_data.TempData(params['temp_uuid']) - if is_human: - ps, ts = run_human(pheno_vector = np.array(params['pheno_vector']), - covariate_matrix = np.array(params['covariate_matrix']), - plink_input_file = params['input_file_name'], - kinship_matrix = np.array(params['kinship_matrix']), - refit = params['refit'], - tempdata = tempdata) - else: - ps, ts = run_other(pheno_vector = np.array(params['pheno_vector']), - genotype_matrix = np.array(params['genotype_matrix']), - restricted_max_likelihood = params['restricted_max_likelihood'], - refit = params['refit'], - tempdata = tempdata) - - results_key = "pylmm:results:" + params['temp_uuid'] + results_key = "pylmm:results:" + params['temp_uuid'] - json_results = json.dumps(dict(p_values = ps, - t_stats = ts)) - - #Pushing json_results into a list where it is the only item because blpop needs a list - Redis.rpush(results_key, json_results) - Redis.expire(results_key, 60*60) + json_results = json.dumps(dict(p_values = ps, + t_stats = ts)) + + #Pushing json_results into a list where it is the only item because blpop needs a list + Redis.rpush(results_key, json_results) + Redis.expire(results_key, 60*60) if __name__ == '__main__': main() -- cgit v1.2.3