From 719b256996facb1f4bee2a0288bce1f18168f99b Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 16 Sep 2021 17:59:49 +0000 Subject: Added back the option to edit privileges on trait pages, since the metadata edit button had replaced it --- wqflask/wqflask/templates/show_trait_details.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 36d3c15a..6a541c8c 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -242,6 +242,9 @@ {% if this_trait.dataset.type == 'ProbeSet' %} {% endif %} + {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} + + {% endif %} {% endif %} -- cgit 1.4.1 From a9509a01191883e61cd5013453d18e89db022df2 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 20 Sep 2021 17:34:26 +0000 Subject: Removed the reaper_version selection option, since original qtlreaper is no longer supported --- wqflask/wqflask/templates/show_trait_mapping_tools.html | 9 --------- 1 file changed, 9 deletions(-) diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index 5365140d..c6b6c0e1 100755 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -95,15 +95,6 @@ {% elif mapping_method == "QTLReaper" %}
-
- -
- -
-
-- cgit 1.4.1 From 2ad5b87f1c9bc5ce5bd3f54f7d83f18123ce3c3e Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 20 Sep 2021 17:37:18 +0000 Subject: Returned the GN1 url to gn1.genenetwork.org since it's apparently working well enough again now that some issues have been fixed --- wqflask/wqflask/templates/show_trait_details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 6a541c8c..2a21dd24 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -233,7 +233,7 @@ {% endif %} {% endif %} - + {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} {% if this_trait.dataset.type == 'Publish' %} -- cgit 1.4.1 From 92df90d68c3d3067f6856c6894be92c7c5dbcaa6 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 22 Sep 2021 17:41:53 +0000 Subject: Remove references to reaper_version and code for running old qtlreaper --- .../marker_regression/display_mapping_results.py | 3 +- .../wqflask/marker_regression/qtlreaper_mapping.py | 95 ---------------------- wqflask/wqflask/marker_regression/run_mapping.py | 59 +++++--------- .../new/javascript/show_trait_mapping_tools.js | 2 +- wqflask/wqflask/views.py | 1 - 5 files changed, 24 insertions(+), 136 deletions(-) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 77d6e2db..6254b9b9 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -357,8 +357,7 @@ class DisplayMappingResults: if 'use_loco' in list(start_vars.keys()) and self.mapping_method == "gemma": self.use_loco = start_vars['use_loco'] - if 'reaper_version' in list(start_vars.keys()) and self.mapping_method == "reaper": - self.reaper_version = start_vars['reaper_version'] + if self.mapping_method == "reaper": if 'output_files' in start_vars: self.output_files = ",".join( [(the_file if the_file is not None else "") for the_file in start_vars['output_files']]) diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py index 4d6715ba..801674e1 100644 --- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py +++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py @@ -178,101 +178,6 @@ def parse_reaper_output(gwa_filename, permu_filename, bootstrap_filename): return marker_obs, permu_vals, bootstrap_vals -def run_original_reaper(this_trait, dataset, samples_before, trait_vals, json_data, num_perm, bootCheck, num_bootstrap, do_control, control_marker, manhattan_plot): - genotype = dataset.group.read_genotype_file(use_reaper=True) - - if manhattan_plot != True: - genotype = genotype.addinterval() - - trimmed_samples = [] - trimmed_values = [] - for i in range(0, len(samples_before)): - try: - trimmed_values.append(float(trait_vals[i])) - trimmed_samples.append(str(samples_before[i])) - except: - pass - - perm_output = [] - bootstrap_results = [] - - if num_perm < 100: - suggestive = 0 - significant = 0 - else: - perm_output = genotype.permutation( - strains=trimmed_samples, trait=trimmed_values, nperm=num_perm) - suggestive = perm_output[int(num_perm * 0.37 - 1)] - significant = perm_output[int(num_perm * 0.95 - 1)] - # highly_significant = perm_output[int(num_perm*0.99-1)] #ZS: Currently not used, but leaving it here just in case - - json_data['suggestive'] = suggestive - json_data['significant'] = significant - - if control_marker != "" and do_control == "true": - reaper_results = genotype.regression(strains=trimmed_samples, - trait=trimmed_values, - control=str(control_marker)) - if bootCheck: - control_geno = [] - control_geno2 = [] - _FIND = 0 - for _chr in genotype: - for _locus in _chr: - if _locus.name == control_marker: - control_geno2 = _locus.genotype - _FIND = 1 - break - if _FIND: - break - if control_geno2: - _prgy = list(genotype.prgy) - for _strain in trimmed_samples: - _idx = _prgy.index(_strain) - control_geno.append(control_geno2[_idx]) - - bootstrap_results = genotype.bootstrap(strains=trimmed_samples, - trait=trimmed_values, - control=control_geno, - nboot=num_bootstrap) - else: - reaper_results = genotype.regression(strains=trimmed_samples, - trait=trimmed_values) - - if bootCheck: - bootstrap_results = genotype.bootstrap(strains=trimmed_samples, - trait=trimmed_values, - nboot=num_bootstrap) - - json_data['chr'] = [] - json_data['pos'] = [] - json_data['lod.hk'] = [] - json_data['markernames'] = [] - # if self.additive: - # self.json_data['additive'] = [] - - # Need to convert the QTL objects that qtl reaper returns into a json serializable dictionary - qtl_results = [] - for qtl in reaper_results: - reaper_locus = qtl.locus - # ZS: Convert chr to int - converted_chr = reaper_locus.chr - if reaper_locus.chr != "X" and reaper_locus.chr != "X/Y": - converted_chr = int(reaper_locus.chr) - json_data['chr'].append(converted_chr) - json_data['pos'].append(reaper_locus.Mb) - json_data['lod.hk'].append(qtl.lrs) - json_data['markernames'].append(reaper_locus.name) - # if self.additive: - # self.json_data['additive'].append(qtl.additive) - locus = {"name": reaper_locus.name, "chr": reaper_locus.chr, - "cM": reaper_locus.cM, "Mb": reaper_locus.Mb} - qtl = {"lrs_value": qtl.lrs, "chr": converted_chr, "Mb": reaper_locus.Mb, - "cM": reaper_locus.cM, "name": reaper_locus.name, "additive": qtl.additive, "dominance": qtl.dominance} - qtl_results.append(qtl) - return qtl_results, json_data, perm_output, suggestive, significant, bootstrap_results - - def natural_sort(marker_list): """ Function to naturally sort numbers + strings, adopted from user Mark Byers here: https://stackoverflow.com/questions/4836710/does-python-have-a-built-in-function-for-string-natural-sort diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index 1df53fef..290c4a14 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -271,47 +271,32 @@ class RunMapping: self.bootCheck = False self.num_bootstrap = 0 - self.reaper_version = start_vars['reaper_version'] - self.control_marker = start_vars['control_marker'] self.do_control = start_vars['do_control'] logger.info("Running qtlreaper") - if self.reaper_version == "new": - self.first_run = True - self.output_files = None - # ZS: check if first run so existing result files can be used if it isn't (for example zooming on a chromosome, etc) - if 'first_run' in start_vars: - self.first_run = False - if 'output_files' in start_vars: - self.output_files = start_vars['output_files'].split( - ",") - - results, self.perm_output, self.suggestive, self.significant, self.bootstrap_results, self.output_files = qtlreaper_mapping.run_reaper(self.this_trait, - self.dataset, - self.samples, - self.vals, - self.json_data, - self.num_perm, - self.bootCheck, - self.num_bootstrap, - self.do_control, - self.control_marker, - self.manhattan_plot, - self.first_run, - self.output_files) - else: - results, self.json_data, self.perm_output, self.suggestive, self.significant, self.bootstrap_results = qtlreaper_mapping.run_original_reaper(self.this_trait, - self.dataset, - self.samples, - self.vals, - self.json_data, - self.num_perm, - self.bootCheck, - self.num_bootstrap, - self.do_control, - self.control_marker, - self.manhattan_plot) + self.first_run = True + self.output_files = None + # ZS: check if first run so existing result files can be used if it isn't (for example zooming on a chromosome, etc) + if 'first_run' in start_vars: + self.first_run = False + if 'output_files' in start_vars: + self.output_files = start_vars['output_files'].split( + ",") + + results, self.perm_output, self.suggestive, self.significant, self.bootstrap_results, self.output_files = qtlreaper_mapping.run_reaper(self.this_trait, + self.dataset, + self.samples, + self.vals, + self.json_data, + self.num_perm, + self.bootCheck, + self.num_bootstrap, + self.do_control, + self.control_marker, + self.manhattan_plot, + self.first_run, + self.output_files) elif self.mapping_method == "plink": self.score_type = "-logP" self.manhattan_plot = True diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js index b75d658e..e42fe8c4 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js +++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js @@ -145,7 +145,7 @@ var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form 'sample_vals', 'vals_hash', 'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars', 'num_bootstrap', 'bootCheck', 'bootstrap_results', 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker', 'do_control', 'genofile', 'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', - 'haplotypeAnalystCheck', 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'reaper_version', 'primary_samples'] + 'haplotypeAnalystCheck', 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'primary_samples'] $(".rqtl-geno-tab, #rqtl_geno_compute").on("click", (function(_this) { return function() { diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 85aa6b17..980b9362 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1097,7 +1097,6 @@ def mapping_results_page(): 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', - 'reaper_version', 'n_samples', 'transform' ) -- cgit 1.4.1 From 7cb23fe9c03e4f8051340e2a8f32645e88acce3a Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 22 Sep 2021 17:52:37 +0000 Subject: Added type='button' to the Delete Collection button within collections/view, since without it it attempts to submit the form encapsulating the button (export in this case) --- wqflask/wqflask/templates/collections/view.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index 9ec98ab1..a3090bcf 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -49,7 +49,7 @@ - +
-- cgit 1.4.1 From 9bd85361d58eb67ef9e59eff6031ed9fd00a3411 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 22 Sep 2021 18:30:13 +0000 Subject: Only show most search page options if results exist; otherwise provide a link back to the home page --- wqflask/wqflask/templates/search_result_page.html | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 7ec335d5..c499aa8f 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -53,6 +53,7 @@ A total of {{ results|count }} records were found.

+ {% if results|count > 0 %} {% if go_term is not none %}

The associated genes include:

{% for word in search_terms %}{{ word.search_term[0] }}{% endfor %}

{% endif %} @@ -133,8 +134,11 @@
{% endif %} + {% else %} +
+ + {% endif %}
-
@@ -171,6 +175,7 @@ return params; }; + {% if results|count > 0 %} //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters trait_table = $('#trait_table').DataTable( { 'drawCallback': function( settings ) { @@ -412,6 +417,7 @@ var table = $('#trait_table').DataTable(); table.colReorder.reset() }); + {% endif %} submit_special = function(url) { $("#trait_submission_form").attr("action", url); -- cgit 1.4.1 From cbb9905aa4406551973bfe47dcdb00f6b8ce9628 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 22 Sep 2021 18:39:38 +0000 Subject: Fixed issue where the control marker parameter was being passed incorrectly to GN3 (should be just 'control' but was previously passed as 'control_marker') --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index a148b49c..1dca1b1b 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -39,7 +39,7 @@ def run_rqtl(trait_name, vals, samples, dataset, mapping_scale, model, method, n } if do_control == "true" and control_marker: - post_data["control_marker"] = control_marker + post_data["control"] = control_marker if not manhattan_plot: post_data["interval"] = True -- cgit 1.4.1 From cb9422ec5fc985530fcb4fb44d2729e460424d60 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 22 Sep 2021 18:44:01 +0000 Subject: Removed the Control for (marker) option from R/qtl for now since Danny mentioned it isn't working correctly --- wqflask/wqflask/templates/show_trait_mapping_tools.html | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index c6b6c0e1..3af94ed6 100755 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -253,21 +253,6 @@
{% endif %} -
- -
- - - -
-
-
-- cgit 1.4.1 From 3494a9af7f82691f419ba1e3bd616be5ce68a00a Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Sep 2021 13:07:11 +0300 Subject: Remove lengthy stack trace for 404, 400, 408 status codes The stack trace causes the log file to grow into unmanageable sizes over time. --- wqflask/wqflask/views.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 980b9362..1139ad43 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -160,18 +160,26 @@ def shutdown_session(exception=None): @app.errorhandler(Exception) -def handle_bad_request(e): +def handle_generic_exceptions(e): + import werkzeug err_msg = str(e) - logger.error(err_msg) - logger.error(request.url) - # get the stack trace and send it to the logger - exc_type, exc_value, exc_traceback = sys.exc_info() - logger.error(traceback.format_exc()) now = datetime.datetime.utcnow() time_str = now.strftime('%l:%M%p UTC %b %d, %Y') - formatted_lines = [request.url - + " (" + time_str + ")"] + traceback.format_exc().splitlines() - + # get the stack trace and send it to the logger + exc_type, exc_value, exc_traceback = sys.exc_info() + formatted_lines = {f"{request.url} ({time_str}) " + f" {traceback.format_exc().splitlines()}"} + + _message_templates = { + werkzeug.exceptions.NotFound: ("404: Not Found: " + f"{time_str}: {request.url}"), + werkzeug.exceptions.BadRequest: ("400: Bad Request: " + f"{time_str}: {request.url}"), + werkzeug.exceptions.RequestTimeout: ("408: Request Timeout: " + f"{time_str}: {request.url}")} + # Default to the lengthy stack trace! + logger.error(_message_templates.get(exc_type, + formatted_lines)) # Handle random animations # Use a cookie to have one animation on refresh animation = request.cookies.get(err_msg[:32]) -- cgit 1.4.1 From 8135bd22898130ae6f58aa74cb80c83ab334d0d4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Sep 2021 13:08:49 +0300 Subject: Remove variable-- named after a python keyword-- called "list" --- wqflask/wqflask/views.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 1139ad43..007b0f9e 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -184,9 +184,8 @@ def handle_generic_exceptions(e): # Use a cookie to have one animation on refresh animation = request.cookies.get(err_msg[:32]) if not animation: - list = [fn for fn in os.listdir( - "./wqflask/static/gif/error") if fn.endswith(".gif")] - animation = random.choice(list) + animation = random.choice([fn for fn in os.listdir( + "./wqflask/static/gif/error") if fn.endswith(".gif")]) resp = make_response(render_template("error.html", message=err_msg, stack=formatted_lines, error_image=animation, version=GN_VERSION)) -- cgit 1.4.1 From 16947a15bd955be7da45bb5f078870649b1b63e5 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Sep 2021 13:09:24 +0300 Subject: Apply pep-8 formatting --- wqflask/wqflask/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 007b0f9e..b297da08 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -188,7 +188,9 @@ def handle_generic_exceptions(e): "./wqflask/static/gif/error") if fn.endswith(".gif")]) resp = make_response(render_template("error.html", message=err_msg, - stack=formatted_lines, error_image=animation, version=GN_VERSION)) + stack=formatted_lines, + error_image=animation, + version=GN_VERSION)) # logger.error("Set cookie %s with %s" % (err_msg, animation)) resp.set_cookie(err_msg[:32], animation) -- cgit 1.4.1 From 266d4c4a425ca0a215c8d789e2978d213d5ff37d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 21 Sep 2021 21:25:13 +0300 Subject: Rename "admin_login_required" to "edit_access_required" --- wqflask/wqflask/decorators.py | 2 +- wqflask/wqflask/views.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index f0978fd3..f6e3eb8a 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -3,7 +3,7 @@ from flask import g from functools import wraps -def admin_login_required(f): +def edit_access_required(f): """Use this for endpoints where admins are required""" @wraps(f) def wrap(*args, **kwargs): diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index b297da08..5067ca0e 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -85,7 +85,7 @@ from wqflask.export_traits import export_search_results_csv from wqflask.gsearch import GSearch from wqflask.update_search_results import GSearch as UpdateGSearch from wqflask.docs import Docs, update_text -from wqflask.decorators import admin_login_required +from wqflask.decorators import edit_access_required from wqflask.db_info import InfoPage from utility import temp_data @@ -420,7 +420,7 @@ def submit_trait_form(): @app.route("/trait//edit/inbredset-id/") -@admin_login_required +@edit_access_required def edit_phenotype(name, inbredset_id): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), @@ -477,7 +477,7 @@ def edit_phenotype(name, inbredset_id): @app.route("/trait/edit/probeset-name/") -@admin_login_required +@edit_access_required def edit_probeset(dataset_name): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), @@ -520,7 +520,7 @@ def edit_probeset(dataset_name): @app.route("/trait/update", methods=["POST"]) -@admin_login_required +@edit_access_required def update_phenotype(): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), @@ -646,7 +646,7 @@ def update_phenotype(): @app.route("/probeset/update", methods=["POST"]) -@admin_login_required +@edit_access_required def update_probeset(): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), @@ -1381,7 +1381,7 @@ def get_sample_data_as_csv(trait_name: int, phenotype_id: int): @app.route("/admin/data-sample/diffs/") -@admin_login_required +@edit_access_required def display_diffs_admin(): TMPDIR = current_app.config.get("TMPDIR") DIFF_DIR = f"{TMPDIR}/sample-data/diffs" -- cgit 1.4.1 From 7f317126d7d422b073cb4e4a8698757fe1e763f3 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 21 Sep 2021 21:36:32 +0300 Subject: Replace hard-coded e-mails with gn-proxy queries * wqflask/wqflask/decorators.py (edit_access_required.wrap): Query the proxy to see the access rights of a given user. --- wqflask/wqflask/decorators.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index f6e3eb8a..54aa6795 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -1,14 +1,36 @@ """This module contains gn2 decorators""" from flask import g +from typing import Dict from functools import wraps +from utility.hmac import hmac_creation + +import json +import requests def edit_access_required(f): """Use this for endpoints where admins are required""" @wraps(f) def wrap(*args, **kwargs): - if g.user_session.record.get(b"user_email_address") not in [ - b"labwilliams@gmail.com"]: + resource_id: str = "" + if kwargs.get("inbredset_id"): # data type: dataset-publish + resource_id = hmac_creation("dataset-publish:" + f"{kwargs.get('inbredset_id')}:" + f"{kwargs.get('name')}") + if kwargs.get("dataset_name"): # data type: dataset-probe + resource_id = hmac_creation("dataset-probeset:" + f"{kwargs.get('dataset_name')}") + response: Dict = {} + try: + _user_id = g.user_session.record.get(b"user_id", + "").decode("utf-8") + response = json.loads( + requests.get("http://localhost:8080/" + "available?resource=" + f"{resource_id}&user={_user_id}").content) + except: + response = {} + if "edit" not in response.get("data", []): return "You need to be admin", 401 return f(*args, **kwargs) return wrap -- cgit 1.4.1 From ac14326be6695f185f843d29bf3ff016f5eb3016 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 22 Sep 2021 14:30:09 +0300 Subject: new_security: login_user.html: Delete commented out block --- .../wqflask/templates/new_security/login_user.html | 26 ---------------------- 1 file changed, 26 deletions(-) diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index 095036f0..88eab6bc 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -114,31 +114,5 @@ label.error,div.error{ {% endblock %} {% block js %} - - - {% include "new_security/_scripts.html" %} {% endblock %} -- cgit 1.4.1 From 84a0cce8a341b8b45b3b0037379818c32d5614b2 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 22 Sep 2021 14:32:03 +0300 Subject: Remove "_scripts.html" and all it's references --- wqflask/wqflask/templates/new_security/_scripts.html | 1 - wqflask/wqflask/templates/new_security/forgot_password.html | 1 - wqflask/wqflask/templates/new_security/forgot_password_step2.html | 1 - wqflask/wqflask/templates/new_security/password_reset.html | 1 - wqflask/wqflask/templates/new_security/register_user.html | 1 - wqflask/wqflask/templates/new_security/registered.html | 1 - wqflask/wqflask/templates/new_security/thank_you.html | 1 - wqflask/wqflask/templates/new_security/verification_still_needed.html | 1 - 8 files changed, 8 deletions(-) delete mode 100644 wqflask/wqflask/templates/new_security/_scripts.html diff --git a/wqflask/wqflask/templates/new_security/_scripts.html b/wqflask/wqflask/templates/new_security/_scripts.html deleted file mode 100644 index 5fefe305..00000000 --- a/wqflask/wqflask/templates/new_security/_scripts.html +++ /dev/null @@ -1 +0,0 @@ - diff --git a/wqflask/wqflask/templates/new_security/forgot_password.html b/wqflask/wqflask/templates/new_security/forgot_password.html index e5c42a45..60a221da 100644 --- a/wqflask/wqflask/templates/new_security/forgot_password.html +++ b/wqflask/wqflask/templates/new_security/forgot_password.html @@ -48,6 +48,5 @@ {% endblock %} {% block js %} - {% include "new_security/_scripts.html" %} {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/forgot_password_step2.html b/wqflask/wqflask/templates/new_security/forgot_password_step2.html index b4bf41c7..1835fd4c 100644 --- a/wqflask/wqflask/templates/new_security/forgot_password_step2.html +++ b/wqflask/wqflask/templates/new_security/forgot_password_step2.html @@ -20,7 +20,6 @@ {% endblock %} {% block js %} - {% include "new_security/_scripts.html" %} {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/password_reset.html b/wqflask/wqflask/templates/new_security/password_reset.html index 684c12b1..e21f075c 100644 --- a/wqflask/wqflask/templates/new_security/password_reset.html +++ b/wqflask/wqflask/templates/new_security/password_reset.html @@ -73,7 +73,6 @@ {% block js %} - {% include "new_security/_scripts.html" %} {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/register_user.html b/wqflask/wqflask/templates/new_security/register_user.html index 3ae4488b..c2895517 100644 --- a/wqflask/wqflask/templates/new_security/register_user.html +++ b/wqflask/wqflask/templates/new_security/register_user.html @@ -100,7 +100,6 @@ {% block js %} - {% include "new_security/_scripts.html" %} {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/registered.html b/wqflask/wqflask/templates/new_security/registered.html index f2f58ec1..29889a97 100644 --- a/wqflask/wqflask/templates/new_security/registered.html +++ b/wqflask/wqflask/templates/new_security/registered.html @@ -19,7 +19,6 @@ {% block js %} - {% include "new_security/_scripts.html" %} {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/thank_you.html b/wqflask/wqflask/templates/new_security/thank_you.html index 0ff7ee8d..d4f5e574 100644 --- a/wqflask/wqflask/templates/new_security/thank_you.html +++ b/wqflask/wqflask/templates/new_security/thank_you.html @@ -18,7 +18,6 @@ {% endblock %} {% block js %} - {% include "new_security/_scripts.html" %} {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/verification_still_needed.html b/wqflask/wqflask/templates/new_security/verification_still_needed.html index dc0f9e68..1f91fd8d 100644 --- a/wqflask/wqflask/templates/new_security/verification_still_needed.html +++ b/wqflask/wqflask/templates/new_security/verification_still_needed.html @@ -21,7 +21,6 @@ {% endblock %} {% block js %} - {% include "new_security/_scripts.html" %} {% endblock %} -- cgit 1.4.1 From 6e0ea75ca427721aed0a5f394b501b2cde9bf769 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 23 Sep 2021 13:17:55 +0300 Subject: Add script for creating/ updating groups during authorisation --- scripts/authentication/group.py | 130 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 scripts/authentication/group.py diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py new file mode 100644 index 00000000..b89bc3ec --- /dev/null +++ b/scripts/authentication/group.py @@ -0,0 +1,130 @@ +"""A script for adding users to a specific group. + +Example: + +Assuming there are no groups and 'test@bonfacemunyoki.com' does not +exist in Redis: + +.. code-block:: bash + python group.py -g "editors" -m "test@bonfacemunyoki.com" + +results in:: + + Successfully created the group: 'editors' + Data: '{"admins": [], "members": []}' + +If 'me@bonfacemunyoki.com' exists in 'users' in Redis and we run: + +.. code-block:: bash + python group.py -g "editors" -m "me@bonfacemunyoki.com" + +now results in:: + + No new group was created. + Updated Data: {'admins': [], 'members': ['me@bonfacemunyoki.com']} + +""" + +import argparse +import redis +import json + +from typing import Dict, List, Optional, Set +from glom import glom # type: ignore + + +def create_group_data(users: Dict, target_group: str, + members: Optional[str] = None, + admins: Optional[str] = None) -> Dict: + """Return a dictionary that contains the following keys: "key", + "field", and "value" that can be used in a redis hash as follows: + HSET key field value + + Parameters: + + - `users`: a list of users for example: + + {'8ad942fe-490d-453e-bd37-56f252e41603': + '{"email_address": "me@test.com", + "full_name": "John Doe", + "organization": "Genenetwork", + "password": {"algorithm": "pbkdf2", + "hashfunc": "sha256", + "salt": "gJrd1HnPSSCmzB5veMPaVk2ozzDlS1Z7Ggcyl1+pciA=", + "iterations": 100000, "keylength": 32, + "created_timestamp": "2021-09-22T11:32:44.971912", + "password": "edcdaa60e84526c6"}, + "user_id": "8ad942fe", "confirmed": 1, + "registration_info": { + "timestamp": "2021-09-22T11:32:45.028833", + "ip_address": "127.0.0.1", + "user_agent": "Mozilla/5.0"}}'} + + - `target_group`: the group name that will be stored inside the + "groups" hash in Redis. + + - `members`: a comma-separated list of values that contain members + of the `target_group` e.g. "me@test1.com, me@test2.com, + me@test3.com" + + - `admins`: a comma-separated list of values that contain + administrators of the `target_group` e.g. "me@test1.com, + me@test2.com, me@test3.com" + + """ + _members = "".join(members.split()).split(",") if members else [] + _admins: List = "".join(admins.split()).split(",") if admins else [] + + user_emails: Set = {glom(json.loads(user_details), "email_address") + for _, user_details in users.items()} + + return {"key": "groups", + "field": target_group, + "value": json.dumps({ + "admins": [admin for admin in _admins + if admin in user_emails], + "members": [member for member in _members + if member in user_emails] + })} + + +if __name__ == "__main__": + # Initialising the parser CLI arguments + parser = argparse.ArgumentParser() + parser.add_argument("-g", "--group-name", + help="This is the name of the GROUP mask") + parser.add_argument("-m", "--members", + help="Members of the GROUP mask") + parser.add_argument("-a", "--admins", + help="Admins of the GROUP mask") + args = parser.parse_args() + + if not args.group_name: + exit("\nExiting. Please specify a group name to use!\n") + + members = args.members if args.members else None + admins = args.admins if args.admins else None + REDIS_CONN = redis.Redis() + USERS = {key.decode(): val.decode() + for key, val in REDIS_CONN.hgetall("users").items()} + + if not any([members, admins]): + exit("\nExiting. Please provide a value for " + "MEMBERS(-m) or ADMINS(-a)!\n") + + data = create_group_data( + users=USERS, + target_group=args.group_name, + members=members, + admins=admins) + created_p = REDIS_CONN.hset(data.get("key", ""), + data.get("field", ""), + data.get("value", "")) + + groups = json.loads(REDIS_CONN.hget("groups", + args.group_name)) # type: ignore + if created_p: + exit(f"\nSuccessfully created the group: '{args.group_name}'\n" + f"Data: {groups}\n") + exit("\nNo new group was created.\n" + f"Updated Data: {groups}\n") -- cgit 1.4.1 From c585945c5516092b362efecc16325ad9ecc54291 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 23 Sep 2021 15:39:32 +0300 Subject: Add script that adds "editors" group to all resources in Redis --- scripts/authentication/resource.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 scripts/authentication/resource.py diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py new file mode 100644 index 00000000..75ef9e93 --- /dev/null +++ b/scripts/authentication/resource.py @@ -0,0 +1,25 @@ +"""A script that adds the group: 'editors' to every +resource. 'editors' should have the right to edit both metadata and +data. + +To use this script, simply run: + +.. code-block:: python + python resource.py + +""" +import json +import redis + + +if __name__ == "__main__": + REDIS_CONN = redis.Redis() + resources = REDIS_CONN.hgetall("resources_clone") + for resource_id, resource in resources.items(): + deserialized_resource = json.loads(resource) + deserialized_resource["group_masks"] = { + "editors": {"metadata": "edit", + "data": "edit"}} + REDIS_CONN.hset("resources_clone", + resource_id, + json.dumps(deserialized_resource)) -- cgit 1.4.1 From edb6fe1b9dd98d84deb0925a2d83726e739d8677 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Sep 2021 16:28:12 +0300 Subject: wqflask: resource_manager: Remove logger --- wqflask/wqflask/resource_manager.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index b28c1b04..c54dd0b3 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -8,8 +8,6 @@ from wqflask import app from utility.authentication_tools import check_owner_or_admin from utility.redis_tools import get_resource_info, get_group_info, get_groups_like_unique_column, get_user_id, get_user_by_unique_column, get_users_like_unique_column, add_access_mask, add_resource, change_resource_owner -from utility.logger import getLogger -logger = getLogger(__name__) @app.route("/resources/manage", methods=('GET', 'POST')) -- cgit 1.4.1 From df487791c91a5aa1a9a3b4e1a6c9ce17a58eafe6 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 4 Oct 2021 12:21:16 +0300 Subject: Modify resource editing script to enable data backups & restoration --- scripts/authentication/resource.py | 101 ++++++++++++++++++++++++++++++++----- 1 file changed, 88 insertions(+), 13 deletions(-) diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py index 75ef9e93..8fcf09d7 100644 --- a/scripts/authentication/resource.py +++ b/scripts/authentication/resource.py @@ -1,25 +1,100 @@ -"""A script that adds the group: 'editors' to every -resource. 'editors' should have the right to edit both metadata and -data. +"""A script that: -To use this script, simply run: +- Optionally restores data from a json file. + +- By default, without any args provided, adds the group: 'editors' to +every resource. 'editors' should have the right to edit both metadata +and data. + +- Optionally creates a back-up every time you edit a resource. + + +To restore a back-up: + +.. code-block:: python + python resource.py --restore + +To add editors to every resource without creating a back-up: .. code-block:: python python resource.py +To add editors to every resource while creating a back-up before any +destructive edits: + +.. code-block:: python + python resource.py --enable-backup + """ +import argparse import json import redis +import os + +from datetime import datetime + + +def recover_hash(name: str, file_path: str, set_function) -> bool: + """Recover back-ups using the `set_function` + + Parameters: + + - `name`: Redis hash where `file_path` will be restored + + - `file_path`: File path where redis hash is sourced from + + - `set_function`: Function used to do the Redis backup for + example: HSET + + """ + try: + with open(file_path, "r") as f: + resources = json.load(f) + for resource_id, resource in resources.items(): + set_function(name=name, + key=resource_id, + value=resource) + return True + except Exception as e: + print(e) + return False if __name__ == "__main__": - REDIS_CONN = redis.Redis() - resources = REDIS_CONN.hgetall("resources_clone") - for resource_id, resource in resources.items(): - deserialized_resource = json.loads(resource) - deserialized_resource["group_masks"] = { - "editors": {"metadata": "edit", - "data": "edit"}} - REDIS_CONN.hset("resources_clone", + # Initialising the parser CLI arguments + parser = argparse.ArgumentParser() + parser.add_argument("--restore", + help="Restore from a given backup") + parser.add_argument("--enable-backup", action="store_true", + help="Create a back up before edits") + args = parser.parse_args() + + if args.restore: + if recover_hash(name="resources", + file_path=args.back_up, + set_function=redis.Redis(decode_responses=True).hset): + exit(f"\n Done restoring {args.back_up}!\n") + else: + exit(f"\n There was an error restoring {args.back_up}!\n") + + REDIS_CONN = redis.Redis(decode_responses=True) + RESOURCES = REDIS_CONN.hgetall("resources") + BACKUP_DIR = os.path.join(os.getenv("HOME"), "redis") + if args.enable_backup: + FILENAME = ("resources-" + f"{datetime.now().strftime('%Y-%m-%d-%I:%M:%S-%p')}" + ".json") + if not os.path.exists(BACKUP_DIR): + os.mkdir(BACKUP_DIR) + with open(os.path.join(BACKUP_DIR, FILENAME), "w") as f: + json.dump(RESOURCES, f, indent=4) + print(f"\nDone backing upto {FILENAME}") + + for resource_id, resource in RESOURCES.items(): + _resource = json.loads(resource) # str -> dict conversion + _resource["group_masks"] = {"editors": {"metadata": "edit", + "data": "edit"}} + REDIS_CONN.hset("resources", resource_id, - json.dumps(deserialized_resource)) + json.dumps(_resource)) + exit("Done updating `resources`\n") -- cgit 1.4.1 From c5215d1ed224480a274476933beded9d2ba7f7dc Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 4 Oct 2021 13:40:06 +0300 Subject: Decode redis response by default --- scripts/authentication/group.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index b89bc3ec..265e8664 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -104,9 +104,8 @@ if __name__ == "__main__": members = args.members if args.members else None admins = args.admins if args.admins else None - REDIS_CONN = redis.Redis() - USERS = {key.decode(): val.decode() - for key, val in REDIS_CONN.hgetall("users").items()} + REDIS_CONN = redis.Redis(decode_responses=True) + USERS = REDIS_CONN.hgetall("users") if not any([members, admins]): exit("\nExiting. Please provide a value for " -- cgit 1.4.1 From 8fae92c83d49042da68638319385df02061df44b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 4 Oct 2021 13:45:59 +0300 Subject: scripts: group.py: Remove "glom" dependency --- scripts/authentication/group.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index 265e8664..02f782b3 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -30,7 +30,6 @@ import redis import json from typing import Dict, List, Optional, Set -from glom import glom # type: ignore def create_group_data(users: Dict, target_group: str, @@ -74,10 +73,12 @@ def create_group_data(users: Dict, target_group: str, """ _members = "".join(members.split()).split(",") if members else [] _admins: List = "".join(admins.split()).split(",") if admins else [] - - user_emails: Set = {glom(json.loads(user_details), "email_address") - for _, user_details in users.items()} - + user_emails: Set = set() + for _, user_details in users.items(): + _details = json.loads(user_details) + if _details.get("email_address"): + user_emails.add(_details.get("email_address")) + print(user_emails) return {"key": "groups", "field": target_group, "value": json.dumps({ -- cgit 1.4.1 From 7c1dd1211f96ca1021debc27a80d1700e70b9c6b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 4 Oct 2021 13:49:12 +0300 Subject: scripts: group.py: Modify exit message when displaying updated data --- scripts/authentication/group.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index 02f782b3..cc0b037e 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -125,6 +125,6 @@ if __name__ == "__main__": args.group_name)) # type: ignore if created_p: exit(f"\nSuccessfully created the group: '{args.group_name}'\n" - f"Data: {groups}\n") + f"`HGETALL groups {args.group_name}`: {groups}\n") exit("\nNo new group was created.\n" - f"Updated Data: {groups}\n") + f"`HGETALL groups {args.group_name}`: {groups}\n") -- cgit 1.4.1 From 609865fc42b7436d8c34cdcefd159c3352c5d91c Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 4 Oct 2021 20:58:14 +0000 Subject: Add group link for user member groups --- wqflask/wqflask/templates/admin/group_manager.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/admin/group_manager.html b/wqflask/wqflask/templates/admin/group_manager.html index c0b99e75..692a7abc 100644 --- a/wqflask/wqflask/templates/admin/group_manager.html +++ b/wqflask/wqflask/templates/admin/group_manager.html @@ -81,7 +81,7 @@ {{ loop.index }} - {{ group.name }} + {{ group.name }} {{ group.admins|length + group.members|length }} {{ group.created_timestamp }} {{ group.changed_timestamp }} -- cgit 1.4.1 From 5b116de4aaf796be138ee0ad06c2242b3f3c33c7 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 4 Oct 2021 21:02:37 +0000 Subject: Changed get_user_groups to pull both the ID and details in the for loop from group_list by using group_list.items() --- wqflask/utility/redis_tools.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py index 99295c67..de9dde46 100644 --- a/wqflask/utility/redis_tools.py +++ b/wqflask/utility/redis_tools.py @@ -127,20 +127,20 @@ def check_verification_code(code): def get_user_groups(user_id): - # ZS: Get the groups where a user is an admin or a member and + # Get the groups where a user is an admin or a member and # return lists corresponding to those two sets of groups - admin_group_ids = [] # ZS: Group IDs where user is an admin - user_group_ids = [] # ZS: Group IDs where user is a regular user + admin_group_ids = [] # Group IDs where user is an admin + user_group_ids = [] # Group IDs where user is a regular user groups_list = Redis.hgetall("groups") - for key in groups_list: + for group_id, group_details in groups_list.items(): try: - group_ob = json.loads(groups_list[key]) - group_admins = set([this_admin if this_admin else None for this_admin in group_ob['admins']]) - group_members = set([this_member if this_member else None for this_member in group_ob['members']]) + _details = json.loads(group_details) + group_admins = set([this_admin if this_admin else None for this_admin in _details['admins']]) + group_members = set([this_member if this_member else None for this_member in _details['members']]) if user_id in group_admins: - admin_group_ids.append(group_ob['id']) + admin_group_ids.append(group_id) elif user_id in group_members: - user_group_ids.append(group_ob['id']) + user_group_ids.append(group_id) else: continue except: -- cgit 1.4.1 From 35a970adba5ee1d60769a81b446122a60eac9494 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 4 Oct 2021 21:03:52 +0000 Subject: Changed the group.py script to replace user e-mails with IDs and to include id, name, changed_timestamp, and created_timestamp in group details --- scripts/authentication/group.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index cc0b037e..a5f75aad 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -26,11 +26,13 @@ now results in:: """ import argparse +import datetime import redis import json from typing import Dict, List, Optional, Set +REDIS_CONN = redis.Redis(decode_responses=True) def create_group_data(users: Dict, target_group: str, members: Optional[str] = None, @@ -71,21 +73,26 @@ def create_group_data(users: Dict, target_group: str, me@test2.com, me@test3.com" """ - _members = "".join(members.split()).split(",") if members else [] + + _members: List = "".join(members.split()).split(",") if members else [] _admins: List = "".join(admins.split()).split(",") if admins else [] - user_emails: Set = set() - for _, user_details in users.items(): + + user_ids: Dict = dict() + for user_id, user_details in users.items(): _details = json.loads(user_details) if _details.get("email_address"): - user_emails.add(_details.get("email_address")) - print(user_emails) + user_ids[_details.get("email_address")] = user_id + print(user_ids) return {"key": "groups", "field": target_group, "value": json.dumps({ - "admins": [admin for admin in _admins - if admin in user_emails], - "members": [member for member in _members - if member in user_emails] + "id": target_group, + "name": target_group, + "admins": [user_ids[admin] for admin in _admins + if admin in user_ids], + "members": [user_ids[member] for member in _members + if member in user_ids], + "changed_timestamp": datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') })} @@ -105,7 +112,6 @@ if __name__ == "__main__": members = args.members if args.members else None admins = args.admins if args.admins else None - REDIS_CONN = redis.Redis(decode_responses=True) USERS = REDIS_CONN.hgetall("users") if not any([members, admins]): @@ -117,6 +123,12 @@ if __name__ == "__main__": target_group=args.group_name, members=members, admins=admins) + + if not REDIS_CONN.hget("groups", data.get("field", "")): + updated_data = json.loads(data["value"]) + updated_data["created_timestamp"] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + data["value"] = json.dumps(updated_data) + created_p = REDIS_CONN.hset(data.get("key", ""), data.get("field", ""), data.get("value", "")) -- cgit 1.4.1 From 4c6a7e46dd7afe311c0bed38c4a69ddadf3cb416 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 4 Oct 2021 21:09:15 +0000 Subject: Moved REDIS_CONN back into if __name__ == '__main__' since it doesn't need to be globally accessed anymore (I think I intiially moved it because I was calling it in create_group_data, but that ended up being unnecessary --- scripts/authentication/group.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index a5f75aad..76c7fb4f 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -32,8 +32,6 @@ import json from typing import Dict, List, Optional, Set -REDIS_CONN = redis.Redis(decode_responses=True) - def create_group_data(users: Dict, target_group: str, members: Optional[str] = None, admins: Optional[str] = None) -> Dict: @@ -112,6 +110,8 @@ if __name__ == "__main__": members = args.members if args.members else None admins = args.admins if args.admins else None + + REDIS_CONN = redis.Redis(decode_responses=True) USERS = REDIS_CONN.hgetall("users") if not any([members, admins]): -- cgit 1.4.1 From 2236508f24c8c266a6a9dff28a1307bb3d0dd31e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 5 Oct 2021 11:07:09 +0300 Subject: tests: test_run_mapping: Add missing positional argument "vals_hash" --- wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py index c220a072..31f56c07 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py @@ -181,7 +181,8 @@ class TestRunMapping(unittest.TestCase): with mock.patch("wqflask.marker_regression.run_mapping.datetime.datetime", new=datetime_mock): export_mapping_results(dataset=self.dataset, trait=self.trait, markers=markers, results_path="~/results", mapping_scale="physic", score_type="-log(p)", - transform="qnorm", covariates="Dataset1:Trait1,Dataset2:Trait2", n_samples="100") + transform="qnorm", covariates="Dataset1:Trait1,Dataset2:Trait2", n_samples="100", + vals_hash="") write_calls = [ mock.call('Time/Date: 09/01/19 / 10:12:12\n'), -- cgit 1.4.1 From 321632bf70c72ed987ba7c4f605dd92deaa62380 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 5 Oct 2021 11:10:04 +0300 Subject: tests: test_run_mapping: Add missing attribute to "self.trait" --- wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py index 31f56c07..26903eed 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py @@ -47,7 +47,7 @@ class TestRunMapping(unittest.TestCase): self.chromosomes = AttributeSetter({"chromosomes": chromosomes}) self.trait = AttributeSetter( - {"symbol": "IGFI", "chr": "X1", "mb": 123313}) + {"symbol": "IGFI", "chr": "X1", "mb": 123313, "display_name": "Test Name"}) def tearDown(self): self.dataset = AttributeSetter( -- cgit 1.4.1 From e97719eea27ede1b3e943e6d21d18e5d383679f0 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 5 Oct 2021 11:12:29 +0300 Subject: tests: test_run_mapping: Fix failing assert when getting perm strata --- wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py index 26903eed..3e9e4ef3 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py @@ -233,25 +233,20 @@ class TestRunMapping(unittest.TestCase): "c1": "c1_value", "c2": "c2_value", "w1": "w1_value" - }, "S2": { "w1": "w2_value", "w2": "w2_value" - }, "S3": { "c1": "c1_value", "c2": "c2_value" - }, - }}) - results = get_perm_strata(this_trait={}, sample_list=sample_list, categorical_vars=categorical_vars, used_samples=used_samples) - self.assertEqual(results, [2, 1]) + self.assertEqual(results, [1, 1]) def test_get_chr_length(self): """test for getting chromosome length""" -- cgit 1.4.1 From 1f5be42d8f090fd4fe77a9275b12a9c9b1383d09 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 5 Oct 2021 11:19:52 +0300 Subject: tests: test_run_mapping: Add missing calls After adding the missing attribute to "self.trait", we need to also need to update some missing calls. --- wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py index 3e9e4ef3..3747aeb8 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py @@ -188,6 +188,8 @@ class TestRunMapping(unittest.TestCase): mock.call('Time/Date: 09/01/19 / 10:12:12\n'), mock.call('Population: Human GP1_\n'), mock.call( 'Data Set: dataser_1\n'), + mock.call('Trait: Test Name\n'), + mock.call('Trait Hash: \n'), mock.call('N Samples: 100\n'), mock.call( 'Transform - Quantile Normalized\n'), mock.call('Gene Symbol: IGFI\n'), mock.call( -- cgit 1.4.1 From a4ec2505fb9dd6f0480c9b452fd623e038c07939 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 5 Oct 2021 11:28:33 +0300 Subject: tests: test_run_mapping: Add missing "name" attribute to tests --- .../unit/wqflask/marker_regression/test_gemma_mapping.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_gemma_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_gemma_mapping.py index 4003d68f..58a44b2a 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_gemma_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_gemma_mapping.py @@ -81,10 +81,12 @@ class TestGemmaMapping(unittest.TestCase): def test_gen_pheno_txt_file(self): """add tests for generating pheno txt file""" with mock.patch("builtins.open", mock.mock_open())as mock_open: - gen_pheno_txt_file(this_dataset={}, genofile_name="", vals=[ - "x", "w", "q", "we", "R"], trait_filename="fitr.re") + gen_pheno_txt_file( + this_dataset=AttributeSetter({"name": "A"}), + genofile_name="", vals=[ + "x", "w", "q", "we", "R"]) mock_open.assert_called_once_with( - '/home/user/data/gn2/fitr.re.txt', 'w') + '/home/user/data/gn2/PHENO_KiAEKlCvM6iGTM9Kh_TAlQ.txt', 'w') filehandler = mock_open() values = ["x", "w", "q", "we", "R"] write_calls = [mock.call('NA\n'), mock.call('w\n'), mock.call( @@ -112,7 +114,7 @@ class TestGemmaMapping(unittest.TestCase): create_trait.side_effect = create_trait_side_effect group = MockGroup({"name": "group_X", "samplelist": samplelist}) - this_dataset = AttributeSetter({"group": group}) + this_dataset = AttributeSetter({"group": group, "name": "A"}) flat_files.return_value = "Home/Genenetwork" with mock.patch("builtins.open", mock.mock_open())as mock_open: @@ -132,7 +134,7 @@ class TestGemmaMapping(unittest.TestCase): flat_files.assert_called_once_with('mapping') mock_open.assert_called_once_with( - 'Home/Genenetwork/group_X_covariates.txt', 'w') + 'Home/Genenetwork/COVAR_anFZ_LfZYV0Ulywo+7tRCw.txt', 'w') filehandler = mock_open() filehandler.write.assert_has_calls([mock.call( '-9\t'), mock.call('-9\t'), mock.call('-9\t'), mock.call('-9\t'), mock.call('\n')]) -- cgit 1.4.1 From 7a15d24a6598f30801dd897ddc72d3773641e7bd Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 5 Oct 2021 11:32:17 +0300 Subject: doc: docker-container.org: Remove python2 gn2 docker set-up command --- doc/docker-container.org | 7 ------- 1 file changed, 7 deletions(-) diff --git a/doc/docker-container.org b/doc/docker-container.org index ef0d71fc..79b8272f 100644 --- a/doc/docker-container.org +++ b/doc/docker-container.org @@ -28,13 +28,6 @@ which will be added to a base mariaDB image. First create the gn2 tar archive by running: #+begin_src sh -# For the Python 2 version: -env GUIX_PACKAGE_PATH="/home/bonface/projects/guix-bioinformatics:/home/bonface/projects/guix-past/modules" \ - ./pre-inst-env guix pack --no-grafts\ - -S /gn2-profile=/ \ - screen python2-genenetwork2 - -# For the Python 3 version: env GUIX_PACKAGE_PATH="/home/bonface/projects/guix-bioinformatics:/home/bonface/projects/guix-past/modules" \ ./pre-inst-env guix pack --no-grafts\ -S /gn2-profile=/ \ -- cgit 1.4.1