1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
|
"""Endpoints for running the gemma cmd"""
import os
import redis
from flask import Blueprint
from flask import current_app
from flask import jsonify
from flask import request
from gn3.commands import compose_gemma_cmd
from gn3.commands import queue_cmd
from gn3.commands import run_cmd
from gn3.file_utils import get_hash_of_files
from gn3.file_utils import jsonfile_to_dict
from gn3.computations.gemma import do_paths_exist
from gn3.computations.gemma import generate_hash_of_string
from gn3.computations.gemma import generate_pheno_txt_file
from gn3.computations.gemma import generate_gemma_computation_cmd
gemma = Blueprint("gemma", __name__)
@gemma.route("/version")
def get_version():
"""Display the installed version of gemma-wrapper"""
gemma_cmd = current_app.config["GEMMA_WRAPPER_CMD"]
return jsonify(
run_cmd(f"{gemma_cmd} -v | head -n 1"))
# This is basically extracted from genenetwork2
# wqflask/wqflask/marker_regression/gemma_ampping.py
@gemma.route("/k-gwa-computation", methods=["POST"])
def run_gemma():
"""Generates a command for generating K-Values and then later, generate a GWA
command that contains markers. These commands are queued; and the expected
file output is returned.
"""
data = request.get_json()
app_defaults = current_app.config
__hash = generate_hash_of_string(
f"{data.get('genofile_name')}_"
''.join(data.get("values", "")))
gemma_kwargs = {
"geno_filename": os.path.join(app_defaults.get("GENODIR"), "bimbam",
f"{data.get('geno_filename')}"),
"trait_filename": generate_pheno_txt_file(
tmpdir=app_defaults.get("TMPDIR"),
values=data.get("values"),
# Generate this file on the fly!
trait_filename=(f"{data.get('dataset_groupname')}_"
f"{data.get('trait_name')}_"
f"{__hash}.txt"))}
gemma_wrapper_kwargs = {}
if data.get("loco"):
gemma_wrapper_kwargs["loco"] = f"--input {data.get('loco')}"
k_computation_cmd = generate_gemma_computation_cmd(
gemma_cmd=app_defaults.get("GEMMA_WRAPPER_CMD"),
gemma_wrapper_kwargs={"loco": f"--input {data.get('loco')}"},
gemma_kwargs=gemma_kwargs,
output_file=(f"{app_defaults.get('TMPDIR')}/gn2/"
f"{data.get('dataset_name')}_K_"
f"{__hash}.json"))
gemma_kwargs["lmm"] = data.get("lmm", 9)
gemma_wrapper_kwargs["input"] = (f"{data.get('dataset_name')}_K_"
f"{__hash}.json")
gwa_cmd = generate_gemma_computation_cmd(
gemma_wrapper_kwargs=gemma_wrapper_kwargs,
gemma_cmd=app_defaults.get("GEMMA_WRAPPER_CMD"),
gemma_kwargs=gemma_kwargs,
output_file=(f"{data.get('dataset_name')}_GWA_"
f"{__hash}.txt"))
if not all([k_computation_cmd, gwa_cmd]):
return jsonify(status=128,
error="Unable to generate cmds for computation!"), 500
return jsonify(
unique_id=queue_cmd(conn=redis.Redis(),
email=data.get("email"),
job_queue=app_defaults.get("REDIS_JOB_QUEUE"),
cmd=f"{k_computation_cmd} && {gwa_cmd}"),
status="queued",
output_file=(f"{data.get('dataset_name')}_GWA_"
f"{__hash}.txt"))
@gemma.route("/status/<unique_id>", methods=["GET"])
def check_cmd_status(unique_id):
"""Given a (url-encoded) UNIQUE-ID which is returned when hitting any of the
gemma endpoints, return the status of the command
"""
status = redis.Redis().hget(name=unique_id,
key="status")
if not status:
return jsonify(status=128,
error="The unique id you used does not exist!"), 500
return jsonify(status=status.decode("utf-8"))
@gemma.route("/k-compute/<token>", methods=["POST"])
def compute_k(token):
"""Given a genofile, traitfile, snpsfile, and the token, compute the k-valuen
and return <hash-of-inputs>.json with a UNIQUE-ID of the job. The genofile,
traitfile, and snpsfile are extracted from a metadata.json file.
"""
working_dir = os.path.join(current_app.config.get("TMPDIR"),
token)
_dict = jsonfile_to_dict(os.path.join(working_dir,
"metadata.json"))
try:
genofile, phenofile, snpsfile = [os.path.join(working_dir,
_dict.get(x))
for x in ["geno", "pheno", "snps"]]
gemma_kwargs = {"g": genofile, "p": phenofile, "a": snpsfile}
_hash = get_hash_of_files([genofile, phenofile, snpsfile])
k_output_filename = f"{_hash}-k-output.json"
k_computation_cmd = generate_gemma_computation_cmd(
gemma_cmd=current_app.config.get("GEMMA_WRAPPER_CMD"),
gemma_wrapper_kwargs=None,
gemma_kwargs=gemma_kwargs,
output_file=(f"{current_app.config.get('TMPDIR')}/"
f"{token}/{k_output_filename}"))
return jsonify(
unique_id=queue_cmd(
conn=redis.Redis(),
email=(request.get_json() or {}).get('email'),
job_queue=current_app.config.get("REDIS_JOB_QUEUE"),
cmd=f"{k_computation_cmd}"),
status="queued",
output_file=k_output_filename)
# pylint: disable=W0703
except Exception:
return jsonify(status=128,
# use better message
message="Metadata file non-existent!")
@gemma.route("/k-compute/loco/<chromosomes>/<token>", methods=["POST"])
def compute_k_loco(chromosomes, token):
"""Similar to 'compute_k' with the extra option of using loco given chromosome
values.
"""
working_dir = os.path.join(current_app.config.get("TMPDIR"),
token)
_dict = jsonfile_to_dict(os.path.join(working_dir,
"metadata.json"))
try:
genofile, phenofile, snpsfile = [os.path.join(working_dir,
_dict.get(x))
for x in ["geno", "pheno", "snps"]]
gemma_kwargs = {"g": genofile, "p": phenofile, "a": snpsfile}
_hash = get_hash_of_files([genofile, phenofile, snpsfile])
k_output_filename = f"{_hash}-k-output.json"
k_computation_cmd = generate_gemma_computation_cmd(
gemma_cmd=current_app.config.get("GEMMA_WRAPPER_CMD"),
gemma_wrapper_kwargs={"loco": f"--input {chromosomes}"},
gemma_kwargs=gemma_kwargs,
output_file=(f"{current_app.config.get('TMPDIR')}/"
f"{token}/{k_output_filename}"))
return jsonify(
unique_id=queue_cmd(
conn=redis.Redis(),
email=(request.get_json() or {}).get('email'),
job_queue=current_app.config.get("REDIS_JOB_QUEUE"),
cmd=f"{k_computation_cmd}"),
status="queued",
output_file=k_output_filename)
# pylint: disable=W0703
except Exception:
return jsonify(status=128,
# use better message
message="Metadata file non-existent!")
@gemma.route("/gwa-compute/<k_filename>/<token>", methods=["POST"])
def compute_gwa(k_filename, token):
"""Compute GWA values. No loco no covariates provided.
"""
working_dir = os.path.join(current_app.config.get("TMPDIR"),
token)
_dict = jsonfile_to_dict(os.path.join(working_dir,
"metadata.json"))
try:
genofile, phenofile, snpsfile = [
os.path.join(working_dir,
_dict.get(x))
for x in ["geno", "pheno", "snps"]]
gemma_kwargs = {"g": genofile, "p": phenofile,
"a": snpsfile, "lmm": _dict.get("lmm", 9)}
_hash = get_hash_of_files([genofile, phenofile, snpsfile])
_output_filename = f"{_hash}-gwa-output.json"
return jsonify(
unique_id=queue_cmd(
conn=redis.Redis(),
email=(request.get_json() or {}).get('email'),
job_queue=current_app.config.get("REDIS_JOB_QUEUE"),
cmd=generate_gemma_computation_cmd(
gemma_cmd=current_app.config.get("GEMMA_WRAPPER_CMD"),
gemma_wrapper_kwargs={
"input": os.path.join(working_dir, k_filename)
},
gemma_kwargs=gemma_kwargs,
output_file=(f"{current_app.config.get('TMPDIR')}/"
f"{token}/{_output_filename}"))),
status="queued",
output_file=_output_filename)
# pylint: disable=W0703
except Exception:
return jsonify(status=128,
# use better message
message="Metadata file non-existent!")
@gemma.route("/gwa-compute/covars/<k_filename>/<token>", methods=["POST"])
def compute_gwa_with_covar(k_filename, token):
"""Compute GWA values. Covariates provided.
"""
working_dir = os.path.join(current_app.config.get("TMPDIR"),
token)
_dict = jsonfile_to_dict(os.path.join(working_dir,
"metadata.json"))
try:
genofile, phenofile, snpsfile, covarfile = [
os.path.join(working_dir,
_dict.get(x))
for x in ["geno", "pheno", "snps", "covar"]]
gemma_kwargs = {"g": genofile, "p": phenofile,
"a": snpsfile, "c": covarfile,
"lmm": _dict.get("lmm", 9)}
_hash = get_hash_of_files([genofile, phenofile, snpsfile, covarfile])
_output_filename = f"{_hash}-gwa-output.json"
return jsonify(
unique_id=queue_cmd(
conn=redis.Redis(),
email=(request.get_json() or {}).get('email'),
job_queue=current_app.config.get("REDIS_JOB_QUEUE"),
cmd=generate_gemma_computation_cmd(
gemma_cmd=current_app.config.get("GEMMA_WRAPPER_CMD"),
gemma_wrapper_kwargs={
"input": os.path.join(working_dir, k_filename)
},
gemma_kwargs=gemma_kwargs,
output_file=(f"{current_app.config.get('TMPDIR')}/"
f"{token}/{_output_filename}"))),
status="queued",
output_file=_output_filename)
# pylint: disable=W0703
except Exception:
return jsonify(status=128,
# use better message
message="Metadata file non-existent!")
@gemma.route("/gwa-compute/<k_filename>/loco/maf/<maf>/<token>",
methods=["POST"])
def compute_gwa_with_loco_maf(k_filename, maf, token):
"""Compute GWA values. No Covariates provided. Only loco and maf vals given.
"""
working_dir = os.path.join(current_app.config.get("TMPDIR"),
token)
_dict = jsonfile_to_dict(os.path.join(working_dir,
"metadata.json"))
try:
genofile, phenofile, snpsfile = [
os.path.join(working_dir,
_dict.get(x))
for x in ["geno", "pheno", "snps"]]
if not do_paths_exist([genofile, phenofile, snpsfile]):
raise FileNotFoundError
gemma_kwargs = {"g": genofile, "p": phenofile,
"a": snpsfile, "lmm": _dict.get("lmm", 9),
'maf': float(maf)}
_hash = get_hash_of_files([genofile, phenofile, snpsfile])
_output_filename = f"{_hash}-gwa-output.json"
return jsonify(
unique_id=queue_cmd(
conn=redis.Redis(),
email=(request.get_json() or {}).get('email'),
job_queue=current_app.config.get("REDIS_JOB_QUEUE"),
cmd=compose_gemma_cmd(
gemma_wrapper_cmd=current_app.config.get("GEMMA_"
"WRAPPER_CMD"),
gemma_wrapper_kwargs={
"loco": ("--input "
f"{os.path.join(working_dir, k_filename)}")
},
gemma_kwargs=gemma_kwargs,
gemma_args=["-gk", ">",
(f"{current_app.config.get('TMPDIR')}/"
f"{token}/{_output_filename}")])),
status="queued",
output_file=_output_filename)
# pylint: disable=W0703
except Exception:
return jsonify(status=128,
# use better message
message="Metadata file non-existent!")
|
