From 398174af99d6e50af9e10bbb95f78bcf74388f81 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 24 Feb 2021 21:13:53 +0000 Subject: Committing in-progress changes to replacing trait creation with directly querying for trait attributes --- wqflask/wqflask/search_results.py | 180 +++++++++++++++++++++++--------------- 1 file changed, 108 insertions(+), 72 deletions(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index f23c0582..8edf6147 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -4,6 +4,7 @@ from math import * import time import re import requests +from types import SimpleNamespace from pprint import pformat as pf @@ -11,6 +12,7 @@ import json from base.data_set import create_dataset from base.trait import create_trait +from base.webqtlConfig import PUBMEDLINK_URL from wqflask import parser from wqflask import do_search from db import webqtlDatabaseFunction @@ -18,12 +20,15 @@ from db import webqtlDatabaseFunction from flask import Flask, g from utility import hmac, helper_functions +from utility.authentication_tools import check_resource_availability from utility.tools import GN2_BASE_URL from utility.type_checking import is_str from utility.logger import getLogger logger = getLogger(__name__ ) +from utility.benchmark import Bench + class SearchResultPage(object): #maxReturn = 3000 @@ -39,9 +44,7 @@ class SearchResultPage(object): self.uc_id = uuid.uuid4() self.go_term = None - logger.debug("uc_id:", self.uc_id) # contains a unique id - logger.debug("kw is:", kw) # dict containing search terms if kw['search_terms_or']: self.and_or = "or" self.search_terms = kw['search_terms_or'] @@ -70,11 +73,11 @@ class SearchResultPage(object): assert(is_str(kw.get('dataset'))) self.dataset = create_dataset(kw['dataset'], dataset_type) - logger.debug("search_terms:", self.search_terms) #ZS: I don't like using try/except, but it seems like the easiest way to account for all possible bad searches here try: - self.search() + with Bench("Doing Query"): + self.search() except: self.search_term_exists = False @@ -95,83 +98,118 @@ class SearchResultPage(object): trait_list = [] json_trait_list = [] - species = webqtlDatabaseFunction.retrieve_species(self.dataset.group.name) # result_set represents the results for each search term; a search of # "shh grin2b" would have two sets of results, one for each term - logger.debug("self.results is:", pf(self.results)) + + if self.dataset.type == "ProbeSet": + self.header_data_names = ['index', 'display_name', 'symbol', 'description', 'location', 'mean', 'lrs_score', 'lrs_location', 'additive'] + elif self.dataset.type == "Publish": + self.header_data_names = ['index', 'display_name', 'description', 'mean', 'authors', 'pubmed_text', 'lrs_score', 'lrs_location', 'additive'] + elif self.dataset.type == "Geno": + self.header_data_names = ['index', 'display_name', 'location'] for index, result in enumerate(self.results): if not result: continue - #### Excel file needs to be generated #### - trait_dict = {} - trait_id = result[0] - this_trait = create_trait(dataset=self.dataset, name=trait_id, get_qtl_info=True, get_sample_info=False) - if this_trait: - trait_dict['index'] = index + 1 - trait_dict['name'] = this_trait.name - if this_trait.dataset.type == "Publish": - trait_dict['display_name'] = this_trait.display_name + trait_dict['index'] = index + 1 + trait_dict['name'] = result[0] + + #ZS: Check permissions on a trait-by-trait basis for phenotype traits + if self.dataset.type == "Publish": + permissions = check_resource_availability(self.dataset, trait_dict['name']) + if "view" not in permissions['data']: + continue + + trait_dict['display_name'] = result[0] + if self.dataset.type == "Publish": + if self.dataset.group_code: + trait_dict['display_name'] = self.dataset.group_code + "_" + result[0] + + trait_dict['dataset'] = self.dataset.name + trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait_dict['name'], trait_dict['dataset'])) + if self.dataset.type == "ProbeSet": + trait_dict['symbol'] = "N/A" + if result[10]: + trait_dict['symbol'] = result[10] + trait_dict['description'] = "N/A" + description_string = result[2] + if str(description_string) != "" and description_string != None: + description_display = description_string + else: + description_display = trait_dict['symbol'] + + target_string = result[3] + if str(target_string) != "" and target_string != None: + description_display = description_display + "; " + target_string.strip() + trait_dict['description'] = description_display + + trait_dict['location'] = "N/A" + if (result[8] != "NULL" and result[8] != "") and (result[9] != 0): + trait_dict['location'] = f"Chr{result[8]}: {float(result[9]):.6f}" + trait_dict['mean'] = "N/A" + trait_dict['additive'] = "N/A" + if result[4] != "" and result[4] != None: + trait_dict['mean'] = f"{result[4]:.3f}" + try: + trait_dict['lod_score'] = f"{float(result[5]) / 4.61:.1f}" + except: + trait_dict['lod_score'] = "N/A" + try: + trait_dict['lrs_location'] = f"Chr{result[12]}: {float(result[13]):.6f}" + except: + trait_dict['lrs_location'] = "N/A" + if result[5] != "": + trait_dict['additive'] = f"{result[7]:.3f}" + elif self.dataset.type == "Geno": + trait_dict['location'] = "N/A" + if (result[4] != "NULL" and result[4] != "") and (result[5] != 0): + trait_dict['location'] = f"Chr{result[4]}: {float(result[5]):.6f}" + elif self.dataset.type == "Publish": + trait_dict['description'] = "N/A" + trait_dict['pubmed_id'] = "N/A" + trait_dict['pubmed_link'] = "N/A" + trait_dict['pubmed_text'] = "N/A" + trait_dict['mean'] = "N/A" + trait_dict['additive'] = "N/A" + pre_pub_description = result[2].strip() + post_pub_description = result[3].strip() + if result[1] != "NULL" and result[1] != None: + trait_dict['pubmed_id'] = result[1] + trait_dict['pubmed_link'] = PUBMEDLINK_URL % trait_dict['pubmed_id'] + trait_dict['description'] = post_pub_description else: - trait_dict['display_name'] = this_trait.name - trait_dict['dataset'] = this_trait.dataset.name - trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset.name)) - if this_trait.dataset.type == "ProbeSet": - trait_dict['symbol'] = this_trait.symbol - trait_dict['description'] = "N/A" - if this_trait.description_display: - trait_dict['description'] = this_trait.description_display - trait_dict['location'] = this_trait.location_repr - trait_dict['mean'] = "N/A" - trait_dict['additive'] = "N/A" - if this_trait.mean != "" and this_trait.mean != None: - trait_dict['mean'] = f"{this_trait.mean:.3f}" - try: - trait_dict['lod_score'] = f"{float(this_trait.LRS_score_repr) / 4.61:.1f}" - except: - trait_dict['lod_score'] = "N/A" - trait_dict['lrs_location'] = this_trait.LRS_location_repr - if this_trait.additive != "": - trait_dict['additive'] = f"{this_trait.additive:.3f}" - elif this_trait.dataset.type == "Geno": - trait_dict['location'] = this_trait.location_repr - elif this_trait.dataset.type == "Publish": - trait_dict['description'] = "N/A" - if this_trait.description_display: - trait_dict['description'] = this_trait.description_display - trait_dict['authors'] = this_trait.authors - trait_dict['pubmed_id'] = "N/A" - if this_trait.pubmed_id: - trait_dict['pubmed_id'] = this_trait.pubmed_id - trait_dict['pubmed_link'] = this_trait.pubmed_link - trait_dict['pubmed_text'] = this_trait.pubmed_text - trait_dict['mean'] = "N/A" - if this_trait.mean != "" and this_trait.mean != None: - trait_dict['mean'] = f"{this_trait.mean:.3f}" - try: - trait_dict['lod_score'] = f"{float(this_trait.LRS_score_repr) / 4.61:.1f}" - except: - trait_dict['lod_score'] = "N/A" - trait_dict['lrs_location'] = this_trait.LRS_location_repr - trait_dict['additive'] = "N/A" - if this_trait.additive != "": - trait_dict['additive'] = f"{this_trait.additive:.3f}" - # Convert any bytes in dict to a normal utf-8 string - for key in trait_dict.keys(): - if isinstance(trait_dict[key], bytes): - trait_dict[key] = trait_dict[key].decode('utf-8') - trait_list.append(trait_dict) + trait_dict['description'] = pre_pub_description - self.trait_list = trait_list + if result[6].isdigit(): + trait_dict['pubmed_text'] = result[6] - if self.dataset.type == "ProbeSet": - self.header_data_names = ['index', 'display_name', 'symbol', 'description', 'location', 'mean', 'lrs_score', 'lrs_location', 'additive'] - elif self.dataset.type == "Publish": - self.header_data_names = ['index', 'display_name', 'description', 'mean', 'authors', 'pubmed_text', 'lrs_score', 'lrs_location', 'additive'] - elif self.dataset.type == "Geno": - self.header_data_names = ['index', 'display_name', 'location'] + trait_dict['authors'] = result[5] + + if result[4] != "" and result[4] != None: + trait_dict['mean'] = f"{result[4]:.3f}" + + try: + trait_dict['lod_score'] = f"{float(result[5]) / 4.61:.1f}" + except: + trait_dict['lod_score'] = "N/A" + + try: + trait_dict['lrs_location'] = f"Chr{result[9]}: {float(result[10]):.6f}" + except: + trait_dict['lrs_location'] = "N/A" + + if result[5] != "": + trait_dict['additive'] = f"{result[6]:.3f}" + + # Convert any bytes in dict to a normal utf-8 string + for key in trait_dict.keys(): + if isinstance(trait_dict[key], bytes): + trait_dict[key] = trait_dict[key].decode('utf-8') + trait_list.append(trait_dict) + + self.trait_list = trait_list def search(self): """ @@ -179,13 +217,11 @@ class SearchResultPage(object): """ self.search_terms = parser.parse(self.search_terms) - logger.debug("After parsing:", self.search_terms) combined_from_clause = "" combined_where_clause = "" previous_from_clauses = [] #The same table can't be referenced twice in the from clause - logger.debug("len(search_terms)>1") symbol_list = [] if self.dataset.type == "ProbeSet": for a_search in self.search_terms: -- cgit v1.2.3 From 74a6e3f3d21edb9204b8e6e1188b04424c2f7446 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 25 Feb 2021 19:37:55 +0000 Subject: Phenotype regular search now runs without trait creation --- wqflask/wqflask/do_search.py | 23 +++++++++++++++++++---- wqflask/wqflask/search_results.py | 29 ++++++++++++++--------------- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 00636563..1478d2b6 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -189,10 +189,25 @@ class PhenotypeSearch(DoSearch): DoSearch.search_types['Publish'] = "PhenotypeSearch" base_query = """SELECT PublishXRef.Id, - PublishFreeze.createtime as thistable, - Publication.PubMed_ID as Publication_PubMed_ID, - Phenotype.Post_publication_description as Phenotype_Name - FROM Phenotype, PublishFreeze, Publication, PublishXRef """ + CAST(Phenotype.`Pre_publication_description` AS BINARY), + CAST(Phenotype.`Post_publication_description` AS BINARY), + Publication.`Authors`, + Publication.`Year`, + Publication.`PubMed_ID`, + PublishXRef.`mean`, + PublishXRef.`LRS`, + PublishXRef.`additive`, + PublishXRef.`Locus`, + InbredSet.`InbredSetCode`, + Geno.`Chr`, + Geno.`Mb` + FROM Species + INNER JOIN InbredSet ON InbredSet.`SpeciesId` = Species.`Id` + INNER JOIN PublishXRef ON PublishXRef.`InbredSetId` = InbredSet.`Id` + INNER JOIN PublishFreeze ON PublishFreeze.`InbredSetId` = InbredSet.`Id` + INNER JOIN Publication ON Publication.`Id` = PublishXRef.`PublicationId` + INNER JOIN Phenotype ON Phenotype.`Id` = PublishXRef.`PhenotypeId` + LEFT JOIN Geno ON PublishXRef.Locus = Geno.Name AND Geno.SpeciesId = Species.Id """ search_fields = ('Phenotype.Post_publication_description', 'Phenotype.Pre_publication_description', diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 8edf6147..b8e9f5de 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -124,8 +124,8 @@ class SearchResultPage(object): trait_dict['display_name'] = result[0] if self.dataset.type == "Publish": - if self.dataset.group_code: - trait_dict['display_name'] = self.dataset.group_code + "_" + result[0] + if self.dataset.group.code: + trait_dict['display_name'] = self.dataset.group.code + "_" + str(result[0]) trait_dict['dataset'] = self.dataset.name trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait_dict['name'], trait_dict['dataset'])) @@ -173,35 +173,34 @@ class SearchResultPage(object): trait_dict['pubmed_text'] = "N/A" trait_dict['mean'] = "N/A" trait_dict['additive'] = "N/A" - pre_pub_description = result[2].strip() - post_pub_description = result[3].strip() - if result[1] != "NULL" and result[1] != None: - trait_dict['pubmed_id'] = result[1] + pre_pub_description = "N/A" if result[1] is None else result[1].strip() + post_pub_description = "N/A" if result[2] is None else result[2].strip() + if result[5] != "NULL" and result[5] != None: + trait_dict['pubmed_id'] = result[5] trait_dict['pubmed_link'] = PUBMEDLINK_URL % trait_dict['pubmed_id'] trait_dict['description'] = post_pub_description else: trait_dict['description'] = pre_pub_description - if result[6].isdigit(): - trait_dict['pubmed_text'] = result[6] + if result[4].isdigit(): + trait_dict['pubmed_text'] = result[4] - trait_dict['authors'] = result[5] + trait_dict['authors'] = result[3] - if result[4] != "" and result[4] != None: - trait_dict['mean'] = f"{result[4]:.3f}" + if result[6] != "" and result[6] != None: + trait_dict['mean'] = f"{result[6]:.3f}" try: - trait_dict['lod_score'] = f"{float(result[5]) / 4.61:.1f}" + trait_dict['lod_score'] = f"{float(result[7]) / 4.61:.1f}" except: trait_dict['lod_score'] = "N/A" try: - trait_dict['lrs_location'] = f"Chr{result[9]}: {float(result[10]):.6f}" + trait_dict['lrs_location'] = f"Chr{result[11]}: {float(result[12]):.6f}" except: trait_dict['lrs_location'] = "N/A" - if result[5] != "": - trait_dict['additive'] = f"{result[6]:.3f}" + trait_dict['additive'] = "N/A" if not result[8] else f"{result[8]:.3f}" # Convert any bytes in dict to a normal utf-8 string for key in trait_dict.keys(): -- cgit v1.2.3 From 818897630e0bd2e97de9bbc3fca805eed64fd44f Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 25 Feb 2021 20:26:07 +0000 Subject: mRNA Assay searches without trait creation should work now --- wqflask/wqflask/do_search.py | 35 +++++++++++++++++++++++++---------- wqflask/wqflask/search_results.py | 39 +++++++++++---------------------------- 2 files changed, 36 insertions(+), 38 deletions(-) diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 1478d2b6..4448e81e 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -78,16 +78,31 @@ class MrnaAssaySearch(DoSearch): DoSearch.search_types['ProbeSet'] = "MrnaAssaySearch" - base_query = """SELECT distinct ProbeSet.Name as TNAME, - 0 as thistable, - ProbeSetXRef.Mean as TMEAN, - ProbeSetXRef.LRS as TLRS, - ProbeSetXRef.PVALUE as TPVALUE, - ProbeSet.Chr_num as TCHR_NUM, - ProbeSet.Mb as TMB, - ProbeSet.Symbol as TSYMBOL, - ProbeSet.name_num as TNAME_NUM - FROM ProbeSetXRef, ProbeSet """ + base_query = """ + SELECT + ProbeSetFreeze.`Name`, + ProbeSetFreeze.`FullName`, + ProbeSet.`Name`, + ProbeSet.`Symbol`, + CAST(ProbeSet.`description` AS BINARY), + CAST(ProbeSet.`Probe_Target_Description` AS BINARY), + ProbeSet.`Chr`, + ProbeSet.`Mb`, + ProbeSetXRef.`Mean`, + ProbeSetXRef.`LRS`, + ProbeSetXRef.`Locus`, + ProbeSetXRef.`pValue`, + ProbeSetXRef.`additive`, + Geno.`Chr` as geno_chr, + Geno.`Mb` as geno_mb + FROM Species + INNER JOIN InbredSet ON InbredSet.`SpeciesId`= Species.`Id` + INNER JOIN ProbeFreeze ON ProbeFreeze.`InbredSetId` = InbredSet.`Id` + INNER JOIN Tissue ON ProbeFreeze.`TissueId` = Tissue.`Id` + INNER JOIN ProbeSetFreeze ON ProbeSetFreeze.`ProbeFreezeId` = ProbeFreeze.`Id` + INNER JOIN ProbeSetXRef ON ProbeSetXRef.`ProbeSetFreezeId` = ProbeSetFreeze.`Id` + INNER JOIN ProbeSet ON ProbeSet.`Id` = ProbeSetXRef.`ProbeSetId` + LEFT JOIN Geno ON ProbeSetXRef.`Locus` = Geno.`Name` AND Geno.`SpeciesId` = Species.`Id` """ header_fields = ['Index', 'Record', diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index b8e9f5de..97cc6581 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -130,38 +130,21 @@ class SearchResultPage(object): trait_dict['dataset'] = self.dataset.name trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait_dict['name'], trait_dict['dataset'])) if self.dataset.type == "ProbeSet": - trait_dict['symbol'] = "N/A" - if result[10]: - trait_dict['symbol'] = result[10] - trait_dict['description'] = "N/A" - description_string = result[2] - if str(description_string) != "" and description_string != None: - description_display = description_string - else: - description_display = trait_dict['symbol'] + trait_dict['symbol'] = "N/A" if result[3] is None else result[3].strip() + description_text = "N/A" if result[4] is None or str(result[4]) == "" else trait_dict['symbol'] - target_string = result[3] - if str(target_string) != "" and target_string != None: - description_display = description_display + "; " + target_string.strip() + target_string = result[5] + description_display = description_text if target_string is None or str(target_string) == "" else description_text + "; " + str(target_string).strip() trait_dict['description'] = description_display trait_dict['location'] = "N/A" - if (result[8] != "NULL" and result[8] != "") and (result[9] != 0): - trait_dict['location'] = f"Chr{result[8]}: {float(result[9]):.6f}" - trait_dict['mean'] = "N/A" - trait_dict['additive'] = "N/A" - if result[4] != "" and result[4] != None: - trait_dict['mean'] = f"{result[4]:.3f}" - try: - trait_dict['lod_score'] = f"{float(result[5]) / 4.61:.1f}" - except: - trait_dict['lod_score'] = "N/A" - try: - trait_dict['lrs_location'] = f"Chr{result[12]}: {float(result[13]):.6f}" - except: - trait_dict['lrs_location'] = "N/A" - if result[5] != "": - trait_dict['additive'] = f"{result[7]:.3f}" + if (result[6] is not None) and (result[6] != "") and (result[7] is not None) and (result[7] != 0): + trait_dict['location'] = f"Chr{result[6]}: {float(result[7]):.6f}" + + trait_dict['mean'] = "N/A" if result[8] is None or result[8] == "" else f"{result[8]:.3f}" + trait_dict['additive'] = "N/A" if result[12] is None or result[12] == "" else f"{result[12]:.3f}" + trait_dict['lod_score'] = "N/A" if result[9] is None or result[9] == "" else f"{float(result[9]) / 4.61:.1f}" + trait_dict['lrs_location'] = "N/A" if result[13] is None or result[13] == "" or result[14] is None else f"Chr{result[13]}: {float(result[14]):.6f}" elif self.dataset.type == "Geno": trait_dict['location'] = "N/A" if (result[4] != "NULL" and result[4] != "") and (result[5] != 0): -- cgit v1.2.3 From bd421438f1f0b4de913fa40cd49cfcda27e6b16f Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 2 Mar 2021 22:47:16 +0000 Subject: Changed formatting RifSearch clauses and also changes from clause to an INNER JOIN --- wqflask/wqflask/do_search.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 4448e81e..f3ef68b2 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -403,12 +403,10 @@ class RifSearch(MrnaAssaySearch): DoSearch.search_types['ProbeSet_RIF'] = "RifSearch" def get_from_clause(self): - return ", GeneRIF_BASIC " + return f" INNER JOIN GeneRIF_BASIC ON GeneRIF_BASIC.`symbol` = { self.dataset.type }.`symbol` " def get_where_clause(self): - where_clause = """( %s.symbol = GeneRIF_BASIC.symbol and - MATCH (GeneRIF_BASIC.comment) - AGAINST ('+%s' IN BOOLEAN MODE)) """ % (self.dataset.type, self.search_term[0]) + where_clause = f"(MATCH (GeneRIF_BASIC.comment) AGAINST ('+{ self.search_term[0] }' IN BOOLEAN MODE)) " return where_clause -- cgit v1.2.3 From 6e78df4be9abb7e1ce959e9b83b9d38bd77fcade Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 2 Mar 2021 22:49:52 +0000 Subject: Adoping code from the following link to make resizeable columns for search result table - https://datatables.net/forums/discussion/63231/resizing-columns-using-jquery-ui --- wqflask/wqflask/templates/search_result_page.html | 519 +++++++++++++--------- 1 file changed, 303 insertions(+), 216 deletions(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index e7a7bc51..a33d1b1a 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -7,6 +7,7 @@ + {% endblock %} {% block content %} @@ -171,225 +172,311 @@ return params; }; - //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters - trait_table = $('#trait_table').DataTable( { - 'drawCallback': function( settings ) { - $('#trait_table tr').off().on("click", function(event) { - if (event.target.type !== 'checkbox' && event.target.tagName.toLowerCase() !== 'a') { - var obj =$(this).find('input'); - obj.prop('checked', !obj.is(':checked')); - } - if ($(this).hasClass("selected") && event.target.tagName.toLowerCase() !== 'a'){ - $(this).removeClass("selected") - } else if (event.target.tagName.toLowerCase() !== 'a') { - $(this).addClass("selected") - } - change_buttons() - }); - }, - 'createdRow': function ( row, data, index ) { - $('td', row).eq(0).attr("style", "text-align: center; padding: 0px 10px 2px 10px;"); - $('td', row).eq(1).attr("align", "right"); - $('td', row).eq(1).attr('data-export', index+1); - $('td', row).eq(2).attr('data-export', $('td', row).eq(2).text()); - {% if dataset.type == 'ProbeSet' %} - $('td', row).eq(3).attr('title', $('td', row).eq(3).text()); - $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); - if ($('td', row).eq(3).text().length > 20) { - $('td', row).eq(3).text($('td', row).eq(3).text().substring(0, 20)); - $('td', row).eq(3).text($('td', row).eq(3).text() + '...') - } - $('td', row).eq(4).attr('title', $('td', row).eq(4).text()); - $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text()); - $('td', row).slice(5,10).attr("align", "right"); - $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text()); - $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text()); - $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text()); - $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text()); - $('td', row).eq(9).attr('data-export', $('td', row).eq(9).text()); - {% elif dataset.type == 'Publish' %} - $('td', row).eq(3).attr('title', $('td', row).eq(3).text()); - $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); - $('td', row).eq(4).attr('title', $('td', row).eq(4).text()); - $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text()); - $('td', row).eq(4).attr('align', 'right'); - $('td', row).slice(6,10).attr("align", "right"); - $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text()); - $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text()); - $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text()); - $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text()); - $('td', row).eq(9).attr('data-export', $('td', row).eq(8).text()); - {% elif dataset.type == 'Geno' %} - $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); - {% endif %} + var tableId = "trait_table"; + + columnDefs = [ + { + 'data': null, + 'width': "25px", + 'orderDataType': "dom-checkbox", + 'orderable': false, + 'render': function(data, type, row, meta) { + return '' + } + }, + { + 'title': "Index", + 'type': "natural", + 'width': "30px", + 'data': "index" + }, + { + 'title': "Record", + 'type': "natural-minus-na", + 'data': null, + 'width': "60px", + 'render': function(data, type, row, meta) { + return '' + data.display_name + '' + } + }{% if dataset.type == 'ProbeSet' %}, + { + 'title': "Symbol", + 'type': "natural", + 'width': "120px", + 'data': "symbol" + }, + { + 'title': "Description", + 'type': "natural", + 'data': null, + 'render': function(data, type, row, meta) { + try { + return decodeURIComponent(escape(data.description)) + } catch(err){ + return escape(data.description) + } + } + }, + { + 'title': "
Location
", + 'type': "natural-minus-na", + 'width': "125px", + 'data': "location" + }, + { + 'title': "
Mean
", + 'type': "natural-minus-na", + 'width': "30px", + 'data': "mean", + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak  
LOD  
", + 'type': "natural-minus-na", + 'data': "lod_score", + 'width': "60px", + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak Location
", + 'type': "natural-minus-na", + 'width': "125px", + 'data': "lrs_location" + }, + { + 'title': "
Effect  
Size  
", + 'type': "natural-minus-na", + 'data': "additive", + 'width': "60px", + 'orderSequence': [ "desc", "asc"] + }{% elif dataset.type == 'Publish' %}, + { + 'title': "Description", + 'type': "natural", + 'width': "500px", + 'data': null, + 'render': function(data, type, row, meta) { + try { + return decodeURIComponent(escape(data.description)) + } catch(err){ + return data.description + } + } + }, + { + 'title': "
Mean
", + 'type': "natural-minus-na", + 'width': "30px", + 'data': "mean", + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "Authors", + 'type': "natural", + 'width': "300px", + 'data': null, + 'render': function(data, type, row, meta) { + author_list = data.authors.split(",") + if (author_list.length >= 6) { + author_string = author_list.slice(0, 6).join(",") + ", et al." + } else{ + author_string = data.authors + } + return author_string + } + }, + { + 'title': "
Year
", + 'type': "natural-minus-na", + 'data': null, + 'width': "25px", + 'render': function(data, type, row, meta) { + if (data.pubmed_id != "N/A"){ + return '' + data.pubmed_text + '' + } else { + return data.pubmed_text + } }, - 'data': trait_list, - 'columns': [ - { - 'data': null, - 'width': "25px", - 'orderDataType': "dom-checkbox", - 'orderable': false, - 'render': function(data, type, row, meta) { - return '' - } - }, - { - 'title': "Index", - 'type': "natural", - 'width': "30px", - 'data': "index" - }, - { - 'title': "Record", - 'type': "natural-minus-na", - 'data': null, - 'width': "60px", - 'render': function(data, type, row, meta) { - return '' + data.display_name + '' - } - }{% if dataset.type == 'ProbeSet' %}, - { - 'title': "Symbol", - 'type': "natural", - 'width': "120px", - 'data': "symbol" - }, - { - 'title': "Description", - 'type': "natural", - 'data': null, - 'render': function(data, type, row, meta) { - try { - return decodeURIComponent(escape(data.description)) - } catch(err){ - return escape(data.description) - } - } - }, - { - 'title': "
Location
", - 'type': "natural-minus-na", - 'width': "125px", - 'data': "location" - }, - { - 'title': "
Mean
", - 'type': "natural-minus-na", - 'width': "30px", - 'data': "mean", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak  
LOD  
", - 'type': "natural-minus-na", - 'data': "lod_score", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak Location
", - 'type': "natural-minus-na", - 'width': "125px", - 'data': "lrs_location" - }, - { - 'title': "
Effect  
Size  
", - 'type': "natural-minus-na", - 'data': "additive", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }{% elif dataset.type == 'Publish' %}, - { - 'title': "Description", - 'type': "natural", - 'width': "500px", - 'data': null, - 'render': function(data, type, row, meta) { - try { - return decodeURIComponent(escape(data.description)) - } catch(err){ - return data.description - } + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak  
LOD  
", + 'type': "natural-minus-na", + 'data': "lod_score", + 'width': "60px", + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak Location
", + 'type': "natural-minus-na", + 'width': "120px", + 'data': "lrs_location" + }, + { + 'title': "
Effect  
Size  
", + 'type': "natural-minus-na", + 'width': "60px", + 'data': "additive", + 'orderSequence': [ "desc", "asc"] + }{% elif dataset.type == 'Geno' %}, + { + 'title': "
Location
", + 'type': "natural-minus-na", + 'width': "120px", + 'data': "location" + }{% endif %} + ]; + + loadDataTable(); + + function loadDataTable(){ + //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters + trait_table = $('#' + tableId).DataTable( { + 'drawCallback': function( settings ) { + $('#' + tableId + ' tr').off().on("click", function(event) { + if (event.target.type !== 'checkbox' && event.target.tagName.toLowerCase() !== 'a') { + var obj =$(this).find('input'); + obj.prop('checked', !obj.is(':checked')); + } + if ($(this).hasClass("selected") && event.target.tagName.toLowerCase() !== 'a'){ + $(this).removeClass("selected") + } else if (event.target.tagName.toLowerCase() !== 'a') { + $(this).addClass("selected") + } + change_buttons() + }); + }, + 'createdRow': function ( row, data, index ) { + $('td', row).eq(0).attr("style", "text-align: center; padding: 0px 10px 2px 10px;"); + $('td', row).eq(1).attr("align", "right"); + $('td', row).eq(1).attr('data-export', index+1); + $('td', row).eq(2).attr('data-export', $('td', row).eq(2).text()); + {% if dataset.type == 'ProbeSet' %} + $('td', row).eq(3).attr('title', $('td', row).eq(3).text()); + $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); + if ($('td', row).eq(3).text().length > 20) { + $('td', row).eq(3).text($('td', row).eq(3).text().substring(0, 20)); + $('td', row).eq(3).text($('td', row).eq(3).text() + '...') } - }, - { - 'title': "
Mean
", - 'type': "natural-minus-na", - 'width': "30px", - 'data': "mean", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "Authors", - 'type': "natural", - 'width': "300px", - 'data': null, - 'render': function(data, type, row, meta) { - author_list = data.authors.split(",") - if (author_list.length >= 6) { - author_string = author_list.slice(0, 6).join(",") + ", et al." - } else{ - author_string = data.authors - } - return author_string + $('td', row).eq(4).attr('title', $('td', row).eq(4).text()); + $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text()); + $('td', row).slice(5,10).attr("align", "right"); + $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text()); + $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text()); + $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text()); + $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text()); + $('td', row).eq(9).attr('data-export', $('td', row).eq(9).text()); + {% elif dataset.type == 'Publish' %} + $('td', row).eq(3).attr('title', $('td', row).eq(3).text()); + $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); + $('td', row).eq(4).attr('title', $('td', row).eq(4).text()); + $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text()); + $('td', row).eq(4).attr('align', 'right'); + $('td', row).slice(6,10).attr("align", "right"); + $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text()); + $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text()); + $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text()); + $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text()); + $('td', row).eq(9).attr('data-export', $('td', row).eq(8).text()); + {% elif dataset.type == 'Geno' %} + $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); + {% endif %} + }, + "data": trait_list, + "columns": columnDefs, + "order": [[1, "asc" ]], + "sDom": "iti", + "destroy": true, + "autoWidth": false, + "deferRender": true, + "bSortClasses": false, + {% if trait_list|length > 20 %} + "scrollY": "100vh", + "scroller": true, + "scrollCollapse": true, + {% else %} + "iDisplayLength": -1, + {% endif %} + "initComplete": function (settings) { + //Add JQueryUI resizable functionality to each th in the ScrollHead table + $('#' + tableId + '_wrapper .dataTables_scrollHead thead th').resizable({ + handles: "e", + alsoResize: '#' + tableId + '_wrapper .dataTables_scrollHead table', //Not essential but makes the resizing smoother + stop: function () { + saveColumnSettings(); + loadDataTable(); } - }, - { - 'title': "
Year
", - 'type': "natural-minus-na", - 'data': null, - 'width': "25px", - 'render': function(data, type, row, meta) { - if (data.pubmed_id != "N/A"){ - return '' + data.pubmed_text + '' - } else { - return data.pubmed_text - } - }, - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak  
LOD  
", - 'type': "natural-minus-na", - 'data': "lod_score", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak Location
", - 'type': "natural-minus-na", - 'width': "120px", - 'data': "lrs_location" - }, - { - 'title': "
Effect  
Size  
", - 'type': "natural-minus-na", - 'width': "60px", - 'data': "additive", - 'orderSequence': [ "desc", "asc"] - }{% elif dataset.type == 'Geno' %}, - { - 'title': "
Location
", - 'type': "natural-minus-na", - 'width': "120px", - 'data': "location" - }{% endif %} - ], - "order": [[1, "asc" ]], - 'sDom': "iti", - "autoWidth": true, - "bSortClasses": false, - {% if trait_list|length > 20 %} - "scrollY": "100vh", - "scroller": true, - "scrollCollapse": true - {% else %} - "iDisplayLength": -1 - {% endif %} - } ); + }); + }, + } ); + } + + function setUserColumnsDefWidths() { + + var userColumnDef; + + // Get the settings for this table from localStorage + var userColumnDefs = JSON.parse(localStorage.getItem(tableId)) || []; + + if (userColumnDefs.length === 0 ) return; + + columnDefs.forEach( function(columnDef) { + + // Check if there is a width specified for this column + userColumnDef = userColumnDefs.find( function(column) { + return column.targets === columnDef.targets; + }); + + // If there is, set the width of this columnDef in px + if ( userColumnDef ) { + + columnDef.width = userColumnDef.width + 'px'; + + } + + }); + + } + + + function saveColumnSettings() { + + var userColumnDefs = JSON.parse(localStorage.getItem(tableId)) || []; + + var width, header, existingSetting; + + trait_table.columns().every( function ( targets ) { + + // Check if there is a setting for this column in localStorage + existingSetting = userColumnDefs.findIndex( function(column) { return column.targets === targets;}); + + // Get the width of this column + header = this.header(); + width = $(header).width(); + + if ( existingSetting !== -1 ) { + + // Update the width + userColumnDefs[existingSetting].width = width; + + } else { + + // Add the width for this column + userColumnDefs.push({ + targets: targets, + width: width, + }); + + } + + }); + + // Save (or update) the settings in localStorage + localStorage.setItem(tableId, JSON.stringify(userColumnDefs)); + + } - trait_table.draw(); //ZS: This makes the table adjust its height properly on initial load + //trait_table.draw(); //ZS: This makes the table adjust its height properly on initial load $('.toggle-vis').on( 'click', function (e) { e.preventDefault(); @@ -409,7 +496,7 @@ $('#redraw').click(function() { - var table = $('#trait_table').DataTable(); + var table = $('#' + tableId).DataTable(); table.colReorder.reset() }); -- cgit v1.2.3 From 3c4043e445d0708490b2a14805b8e3eaa75c23fd Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 11 Mar 2021 22:35:11 +0000 Subject: Added some CSS to make cell content cut off with ellipses if a column is narrower than the content's width --- wqflask/wqflask/static/new/css/trait_list.css | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/wqflask/wqflask/static/new/css/trait_list.css b/wqflask/wqflask/static/new/css/trait_list.css index c7249721..6d49f009 100644 --- a/wqflask/wqflask/static/new/css/trait_list.css +++ b/wqflask/wqflask/static/new/css/trait_list.css @@ -52,3 +52,21 @@ div.dts div.dataTables_paginate,div.dts div.dataTables_length{ display:none } +/* This is the important CSS */ +table.dataTable { + table-layout: fixed; + width: 100%; + white-space: nowrap; + overflow: hidden; +} + +table.dataTable th { + white-space: nowrap; + overflow: hidden; +} + +table.dataTable td { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} \ No newline at end of file -- cgit v1.2.3 From c03dded58477b48ce54a8a03da90aae45a256758 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 11 Mar 2021 22:43:35 +0000 Subject: Made a variety of changes to make column resizeability work - the main issue was related to there being both an sWidth and width parameter --- wqflask/wqflask/templates/search_result_page.html | 317 +++++++++++----------- 1 file changed, 166 insertions(+), 151 deletions(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index a33d1b1a..210cef30 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -125,8 +125,8 @@ {% endif %} {% endif %} -
- +
+
@@ -175,161 +175,174 @@ var tableId = "trait_table"; columnDefs = [ - { - 'data': null, - 'width': "25px", - 'orderDataType': "dom-checkbox", - 'orderable': false, - 'render': function(data, type, row, meta) { - return '' - } - }, - { - 'title': "Index", - 'type': "natural", - 'width': "30px", - 'data': "index" - }, - { - 'title': "Record", - 'type': "natural-minus-na", - 'data': null, - 'width': "60px", - 'render': function(data, type, row, meta) { - return '' + data.display_name + '' + { + 'data': null, + 'width': "25px", + 'orderDataType': "dom-checkbox", + 'orderable': false, + 'targets': 0, + 'render': function(data, type, row, meta) { + return '' + } + }, + { + 'title': "Index", + 'type': "natural", + 'width': "30px", + 'targets': 1, + 'data': "index" + }, + { + 'title': "Record", + 'type': "natural-minus-na", + 'data': null, + 'width': "60px", + 'targets': 2, + 'render': function(data, type, row, meta) { + return '' + data.display_name + '' + } + }{% if dataset.type == 'ProbeSet' %}, + { + 'title': "Symbol", + 'type': "natural", + 'width': "120px", + 'targets': 3, + 'data': "symbol" + }, + { + 'title': "Description", + 'type': "natural", + 'data': null, + 'targets': 4, + 'render': function(data, type, row, meta) { + try { + return decodeURIComponent(escape(data.description)) + } catch(err){ + return escape(data.description) } - }{% if dataset.type == 'ProbeSet' %}, - { - 'title': "Symbol", - 'type': "natural", - 'width': "120px", - 'data': "symbol" - }, - { - 'title': "Description", - 'type': "natural", - 'data': null, - 'render': function(data, type, row, meta) { - try { + } + }, + { + 'title': "
Location
", + 'type': "natural-minus-na", + 'width': "125px", + 'targets': 5, + 'data': "location" + }, + { + 'title': "
Mean
", + 'type': "natural-minus-na", + 'width': "30px", + 'data': "mean", + 'targets': 6, + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak  
LOD  
", + 'type': "natural-minus-na", + 'data': "lod_score", + 'width': "60px", + 'targets': 7, + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak Location
", + 'type': "natural-minus-na", + 'width': "125px", + 'targets': 8, + 'data': "lrs_location" + }, + { + 'title': "
Effect  
Size  
", + 'type': "natural-minus-na", + 'data': "additive", + 'width': "60px", + 'targets': 9, + 'orderSequence': [ "desc", "asc"] + }{% elif dataset.type == 'Publish' %}, + { + 'title': "Description", + 'type': "natural", + 'width': "500px", + 'data': null, + 'render': function(data, type, row, meta) { + try { return decodeURIComponent(escape(data.description)) - } catch(err){ - return escape(data.description) - } + } catch(err){ + return data.description } - }, - { - 'title': "
Location
", - 'type': "natural-minus-na", - 'width': "125px", - 'data': "location" - }, - { - 'title': "
Mean
", - 'type': "natural-minus-na", - 'width': "30px", - 'data': "mean", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak  
LOD  
", - 'type': "natural-minus-na", - 'data': "lod_score", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak Location
", - 'type': "natural-minus-na", - 'width': "125px", - 'data': "lrs_location" - }, - { - 'title': "
Effect  
Size  
", - 'type': "natural-minus-na", - 'data': "additive", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }{% elif dataset.type == 'Publish' %}, - { - 'title': "Description", - 'type': "natural", - 'width': "500px", - 'data': null, - 'render': function(data, type, row, meta) { - try { - return decodeURIComponent(escape(data.description)) - } catch(err){ - return data.description - } + } + }, + { + 'title': "
Mean
", + 'type': "natural-minus-na", + 'width': "30px", + 'data': "mean", + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "Authors", + 'type': "natural", + 'width': "300px", + 'data': null, + 'render': function(data, type, row, meta) { + author_list = data.authors.split(",") + if (author_list.length >= 6) { + author_string = author_list.slice(0, 6).join(",") + ", et al." + } else{ + author_string = data.authors } - }, - { - 'title': "
Mean
", - 'type': "natural-minus-na", - 'width': "30px", - 'data': "mean", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "Authors", - 'type': "natural", - 'width': "300px", - 'data': null, - 'render': function(data, type, row, meta) { - author_list = data.authors.split(",") - if (author_list.length >= 6) { - author_string = author_list.slice(0, 6).join(",") + ", et al." - } else{ - author_string = data.authors - } - return author_string + return author_string + } + }, + { + 'title': "
Year
", + 'type': "natural-minus-na", + 'data': null, + 'width': "25px", + 'render': function(data, type, row, meta) { + if (data.pubmed_id != "N/A"){ + return '' + data.pubmed_text + '' + } else { + return data.pubmed_text } }, - { - 'title': "
Year
", - 'type': "natural-minus-na", - 'data': null, - 'width': "25px", - 'render': function(data, type, row, meta) { - if (data.pubmed_id != "N/A"){ - return '' + data.pubmed_text + '' - } else { - return data.pubmed_text - } - }, - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak  
LOD  
", - 'type': "natural-minus-na", - 'data': "lod_score", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak Location
", - 'type': "natural-minus-na", - 'width': "120px", - 'data': "lrs_location" - }, - { - 'title': "
Effect  
Size  
", - 'type': "natural-minus-na", - 'width': "60px", - 'data': "additive", - 'orderSequence': [ "desc", "asc"] - }{% elif dataset.type == 'Geno' %}, - { - 'title': "
Location
", - 'type': "natural-minus-na", - 'width': "120px", - 'data': "location" - }{% endif %} + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak  
LOD  
", + 'type': "natural-minus-na", + 'data': "lod_score", + 'width': "60px", + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak Location
", + 'type': "natural-minus-na", + 'width': "120px", + 'data': "lrs_location" + }, + { + 'title': "
Effect  
Size  
", + 'type': "natural-minus-na", + 'width': "60px", + 'data': "additive", + 'orderSequence': [ "desc", "asc"] + }{% elif dataset.type == 'Geno' %}, + { + 'title': "
Location
", + 'type': "natural-minus-na", + 'width': "120px", + 'data': "location" + }{% endif %} ]; loadDataTable(); function loadDataTable(){ + + setUserColumnsDefWidths(); + //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters trait_table = $('#' + tableId).DataTable( { 'drawCallback': function( settings ) { @@ -391,7 +404,8 @@ "deferRender": true, "bSortClasses": false, {% if trait_list|length > 20 %} - "scrollY": "100vh", + "scrollY": "500px", + "scrollX": true, "scroller": true, "scrollCollapse": true, {% else %} @@ -411,6 +425,10 @@ } ); } + window.addEventListener('resize', function(){ + trait_table.columns.adjust(); + }); + function setUserColumnsDefWidths() { var userColumnDef; @@ -430,12 +448,12 @@ // If there is, set the width of this columnDef in px if ( userColumnDef ) { + columnDef.sWidth = userColumnDef.width + 'px'; columnDef.width = userColumnDef.width + 'px'; } }); - } @@ -473,11 +491,8 @@ // Save (or update) the settings in localStorage localStorage.setItem(tableId, JSON.stringify(userColumnDefs)); - } - //trait_table.draw(); //ZS: This makes the table adjust its height properly on initial load - $('.toggle-vis').on( 'click', function (e) { e.preventDefault(); -- cgit v1.2.3 From 8d727b8de708e48b298f42c673f8e902ac3df6a4 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 11 Mar 2021 22:57:49 +0000 Subject: Added a parameter to prevent columnDefs from being overwritten on initial table generation + changed the default table width to 100% --- wqflask/wqflask/templates/search_result_page.html | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 210cef30..973e9c23 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -125,8 +125,8 @@ {% endif %} {% endif %} -
-

Loading...
+
+
@@ -337,11 +337,13 @@ }{% endif %} ]; - loadDataTable(); + loadDataTable(true); - function loadDataTable(){ + function loadDataTable(first_run=false){ - setUserColumnsDefWidths(); + if (!first_run){ + setUserColumnsDefWidths(); + } //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters trait_table = $('#' + tableId).DataTable( { -- cgit v1.2.3 From 814258b95436c5aabd76d932fde8386fea187e84 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 11 Mar 2021 23:08:43 +0000 Subject: Set scrollX in all cases so the adjustablee columns also work for shorter results that don't use Scroller (<20 results) --- wqflask/wqflask/templates/search_result_page.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 973e9c23..b2820496 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -405,9 +405,9 @@ "autoWidth": false, "deferRender": true, "bSortClasses": false, + "scrollX": true, {% if trait_list|length > 20 %} "scrollY": "500px", - "scrollX": true, "scroller": true, "scrollCollapse": true, {% else %} -- cgit v1.2.3 From f421a7b319696bb57b9ea4c0a3ec94a32fa4aa65 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 24 Mar 2021 19:31:04 +0000 Subject: Use autoWidth on initial table load, though I'll change this to only apply when certain fields (like Description) are below a certain width Added some code tracking the change in width when a column's width is adjusted, so that the table_container width can be changed accordingly --- wqflask/wqflask/templates/search_result_page.html | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index b2820496..22a22e68 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -126,7 +126,7 @@ {% endif %}
-

Loading...
+
@@ -174,6 +174,8 @@ var tableId = "trait_table"; + var width_change = 0; //ZS: For storing the change in width so overall table width can be adjusted by that amount + columnDefs = [ { 'data': null, @@ -196,7 +198,6 @@ 'title': "Record", 'type': "natural-minus-na", 'data': null, - 'width': "60px", 'targets': 2, 'render': function(data, type, row, meta) { return '' + data.display_name + '' @@ -232,7 +233,7 @@ { 'title': "
Mean
", 'type': "natural-minus-na", - 'width': "30px", + 'width': "40px", 'data': "mean", 'targets': 6, 'orderSequence': [ "desc", "asc"] @@ -346,7 +347,7 @@ } //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters - trait_table = $('#' + tableId).DataTable( { + table_settings = { 'drawCallback': function( settings ) { $('#' + tableId + ' tr').off().on("click", function(event) { if (event.target.type !== 'checkbox' && event.target.tagName.toLowerCase() !== 'a') { @@ -402,7 +403,7 @@ "order": [[1, "asc" ]], "sDom": "iti", "destroy": true, - "autoWidth": false, + "autoWidth": true, "deferRender": true, "bSortClasses": false, "scrollX": true, @@ -418,13 +419,22 @@ $('#' + tableId + '_wrapper .dataTables_scrollHead thead th').resizable({ handles: "e", alsoResize: '#' + tableId + '_wrapper .dataTables_scrollHead table', //Not essential but makes the resizing smoother + resize: function( event, ui ) { + width_change = ui.size.width - ui.originalSize.width; + }, stop: function () { saveColumnSettings(); loadDataTable(); } }); }, - } ); + } + + if (!first_run){ + table_settings['autoWidth'] = false; + $('#table_container').css("width", String($('#trait_table').width() + width_change) + "px"); //ZS : Change the container width by the change in width of the adjusted column, so the overall table size adjusts properly + } + trait_table = $('#' + tableId).DataTable(table_settings); } window.addEventListener('resize', function(){ @@ -488,7 +498,6 @@ }); } - }); // Save (or update) the settings in localStorage -- cgit v1.2.3 From ed911b95571948c8c34ff5f3286f322cc9cfbbcf Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 24 Mar 2021 22:12:14 +0000 Subject: Store a dictionary of maximum widths for each field + create a variable indicating if there are any wide columns (too wide to use DataTables' 'autoWidth' setting) --- wqflask/wqflask/search_results.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index f23c0582..c4d8d4ce 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -98,7 +98,6 @@ class SearchResultPage(object): species = webqtlDatabaseFunction.retrieve_species(self.dataset.group.name) # result_set represents the results for each search term; a search of # "shh grin2b" would have two sets of results, one for each term - logger.debug("self.results is:", pf(self.results)) for index, result in enumerate(self.results): if not result: @@ -164,6 +163,19 @@ class SearchResultPage(object): trait_dict[key] = trait_dict[key].decode('utf-8') trait_list.append(trait_dict) + self.max_widths = {} + for i, trait in enumerate(trait_list): + for key in trait.keys(): + self.max_widths[key] = max(len(str(trait[key])), self.max_widths[key]) if key in self.max_widths else len(str(trait[key])) + + self.wide_columns_exist = False + if this_trait.dataset.type == "Publish": + if (self.max_widths['display_name'] > 25 or self.max_widths['description'] > 100 or self.max_widths['authors']> 80): + self.wide_columns_exist = True + if this_trait.dataset.type == "ProbeSet": + if (self.max_widths['display_name'] > 25 or self.max_widths['symbol'] > 25 or self.max_widths['description'] > 100): + self.wide_columns_exist = True + self.trait_list = trait_list if self.dataset.type == "ProbeSet": -- cgit v1.2.3 From 52dcfe71be8cd86bfc19fa310fa3d65920d6f2af Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 24 Mar 2021 22:12:57 +0000 Subject: Added some default widths for situations where some wide columns exist and autoWidth can't be used on initial load + changed the logic for how to recalculate table width when a column is manually resized --- wqflask/wqflask/templates/search_result_page.html | 55 +++++++++++++++++------ 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 22a22e68..8ecddc53 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -125,7 +125,7 @@ {% endif %} {% endif %} -
+

Loading...
@@ -190,23 +190,29 @@ { 'title': "Index", 'type': "natural", - 'width': "30px", + 'width': "35px", 'targets': 1, 'data': "index" - }, + } + {% if dataset.type == 'ProbeSet' %}, { 'title': "Record", 'type': "natural-minus-na", 'data': null, + {% if wide_columns_exist == true %} + 'width': "{{ max_widths.display_name * 5 }}px", + {% endif %} 'targets': 2, 'render': function(data, type, row, meta) { return '' + data.display_name + '' } - }{% if dataset.type == 'ProbeSet' %}, + }, { 'title': "Symbol", 'type': "natural", - 'width': "120px", + {% if wide_columns_exist == true %} + 'width': "200px", + {% endif %} 'targets': 3, 'data': "symbol" }, @@ -214,6 +220,9 @@ 'title': "Description", 'type': "natural", 'data': null, + {% if wide_columns_exist == true %} + 'width': "600px", + {% endif %} 'targets': 4, 'render': function(data, type, row, meta) { try { @@ -261,10 +270,21 @@ 'targets': 9, 'orderSequence': [ "desc", "asc"] }{% elif dataset.type == 'Publish' %}, + { + 'title': "Record", + 'type': "natural-minus-na", + 'data': null, + {% if wide_columns_exist == true %} + 'width': "100px", + {% endif %} + 'targets': 2, + 'render': function(data, type, row, meta) { + return '' + data.display_name + '' + } + }, { 'title': "Description", 'type': "natural", - 'width': "500px", 'data': null, 'render': function(data, type, row, meta) { try { @@ -277,14 +297,16 @@ { 'title': "
Mean
", 'type': "natural-minus-na", - 'width': "30px", + 'width': "60px", 'data': "mean", 'orderSequence': [ "desc", "asc"] }, { 'title': "Authors", 'type': "natural", - 'width': "300px", + {% if wide_columns_exist == true %} + 'width': "400px", + {% endif %} 'data': null, 'render': function(data, type, row, meta) { author_list = data.authors.split(",") @@ -300,7 +322,7 @@ 'title': "
Year
", 'type': "natural-minus-na", 'data': null, - 'width': "25px", + 'width': "50px", 'render': function(data, type, row, meta) { if (data.pubmed_id != "N/A"){ return '' + data.pubmed_text + '' @@ -403,7 +425,11 @@ "order": [[1, "asc" ]], "sDom": "iti", "destroy": true, + {% if wide_columns_exist != true %} "autoWidth": true, + {% else %} + "autoWidth": false, + {% endif %} "deferRender": true, "bSortClasses": false, "scrollX": true, @@ -432,9 +458,15 @@ if (!first_run){ table_settings['autoWidth'] = false; - $('#table_container').css("width", String($('#trait_table').width() + width_change) + "px"); //ZS : Change the container width by the change in width of the adjusted column, so the overall table size adjusts properly + $('#table_container').css("width", String($('#trait_table').width() + width_change {% if trait_list|length > 20 %}+ 17{% endif %}) + "px"); //ZS : Change the container width by the change in width of the adjusted column, so the overall table size adjusts properly } trait_table = $('#' + tableId).DataTable(table_settings); + + {% if trait_list|length > 20 %} + if (first_run){ + $('#table_container').css("width", String($('#trait_table').width() + 17) + "px"); + } + {% endif %} } window.addEventListener('resize', function(){ @@ -442,7 +474,6 @@ }); function setUserColumnsDefWidths() { - var userColumnDef; // Get the settings for this table from localStorage @@ -468,9 +499,7 @@ }); } - function saveColumnSettings() { - var userColumnDefs = JSON.parse(localStorage.getItem(tableId)) || []; var width, header, existingSetting; -- cgit v1.2.3 From 498a7760fa70788827a28b90ea6f2fd0d94931a5 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 18 Jun 2021 19:28:47 +0000 Subject: Resolving conflict in search_result_page.html --- wqflask/wqflask/templates/search_result_page.html | 519 +++++++++++++--------- 1 file changed, 303 insertions(+), 216 deletions(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 7ec335d5..13f3c7c1 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -7,6 +7,7 @@ + {% endblock %} {% block content %} @@ -171,225 +172,311 @@ return params; }; - //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters - trait_table = $('#trait_table').DataTable( { - 'drawCallback': function( settings ) { - $('#trait_table tr').off().on("click", function(event) { - if (event.target.type !== 'checkbox' && event.target.tagName.toLowerCase() !== 'a') { - var obj =$(this).find('input'); - obj.prop('checked', !obj.is(':checked')); - } - if ($(this).hasClass("selected") && event.target.tagName.toLowerCase() !== 'a'){ - $(this).removeClass("selected") - } else if (event.target.tagName.toLowerCase() !== 'a') { - $(this).addClass("selected") - } - change_buttons() - }); - }, - 'createdRow': function ( row, data, index ) { - $('td', row).eq(0).attr("style", "text-align: center; padding: 0px 10px 2px 13px;"); - $('td', row).eq(1).attr("align", "right"); - $('td', row).eq(1).attr('data-export', index+1); - $('td', row).eq(2).attr('data-export', $('td', row).eq(2).text()); - {% if dataset.type == 'ProbeSet' %} - $('td', row).eq(3).attr('title', $('td', row).eq(3).text()); - $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); - if ($('td', row).eq(3).text().length > 20) { - $('td', row).eq(3).text($('td', row).eq(3).text().substring(0, 20)); - $('td', row).eq(3).text($('td', row).eq(3).text() + '...') - } - $('td', row).eq(4).attr('title', $('td', row).eq(4).text()); - $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text()); - $('td', row).slice(5,10).attr("align", "right"); - $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text()); - $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text()); - $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text()); - $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text()); - $('td', row).eq(9).attr('data-export', $('td', row).eq(9).text()); - {% elif dataset.type == 'Publish' %} - $('td', row).eq(3).attr('title', $('td', row).eq(3).text()); - $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); - $('td', row).eq(4).attr('title', $('td', row).eq(4).text()); - $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text()); - $('td', row).eq(4).attr('align', 'right'); - $('td', row).slice(6,10).attr("align", "right"); - $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text()); - $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text()); - $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text()); - $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text()); - $('td', row).eq(9).attr('data-export', $('td', row).eq(8).text()); - {% elif dataset.type == 'Geno' %} - $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); - {% endif %} + var tableId = "trait_table"; + + columnDefs = [ + { + 'data': null, + 'width': "25px", + 'orderDataType': "dom-checkbox", + 'orderable': false, + 'render': function(data, type, row, meta) { + return '' + } + }, + { + 'title': "Index", + 'type': "natural", + 'width': "30px", + 'data': "index" + }, + { + 'title': "Record", + 'type': "natural-minus-na", + 'data': null, + 'width': "60px", + 'render': function(data, type, row, meta) { + return '' + data.display_name + '' + } + }{% if dataset.type == 'ProbeSet' %}, + { + 'title': "Symbol", + 'type': "natural", + 'width': "120px", + 'data': "symbol" + }, + { + 'title': "Description", + 'type': "natural", + 'data': null, + 'render': function(data, type, row, meta) { + try { + return decodeURIComponent(escape(data.description)) + } catch(err){ + return escape(data.description) + } + } + }, + { + 'title': "
Location
", + 'type': "natural-minus-na", + 'width': "125px", + 'data': "location" + }, + { + 'title': "
Mean
", + 'type': "natural-minus-na", + 'width': "30px", + 'data': "mean", + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak  
LOD  
", + 'type': "natural-minus-na", + 'data': "lod_score", + 'width': "60px", + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak Location
", + 'type': "natural-minus-na", + 'width': "125px", + 'data': "lrs_location" + }, + { + 'title': "
Effect  
Size  
", + 'type': "natural-minus-na", + 'data': "additive", + 'width': "60px", + 'orderSequence': [ "desc", "asc"] + }{% elif dataset.type == 'Publish' %}, + { + 'title': "Description", + 'type': "natural", + 'width': "500px", + 'data': null, + 'render': function(data, type, row, meta) { + try { + return decodeURIComponent(escape(data.description)) + } catch(err){ + return data.description + } + } + }, + { + 'title': "
Mean
", + 'type': "natural-minus-na", + 'width': "30px", + 'data': "mean", + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "Authors", + 'type': "natural", + 'width': "300px", + 'data': null, + 'render': function(data, type, row, meta) { + author_list = data.authors.split(",") + if (author_list.length >= 6) { + author_string = author_list.slice(0, 6).join(",") + ", et al." + } else{ + author_string = data.authors + } + return author_string + } + }, + { + 'title': "
Year
", + 'type': "natural-minus-na", + 'data': null, + 'width': "25px", + 'render': function(data, type, row, meta) { + if (data.pubmed_id != "N/A"){ + return '' + data.pubmed_text + '' + } else { + return data.pubmed_text + } }, - 'data': trait_list, - 'columns': [ - { - 'data': null, - 'width': "10px", - 'orderDataType': "dom-checkbox", - 'orderable': false, - 'render': function(data, type, row, meta) { - return '' - } - }, - { - 'title': "Index", - 'type': "natural", - 'width': "30px", - 'data': "index" - }, - { - 'title': "Record", - 'type': "natural-minus-na", - 'data': null, - 'width': "60px", - 'render': function(data, type, row, meta) { - return '' + data.display_name + '' - } - }{% if dataset.type == 'ProbeSet' %}, - { - 'title': "Symbol", - 'type': "natural", - 'width': "120px", - 'data': "symbol" - }, - { - 'title': "Description", - 'type': "natural", - 'data': null, - 'render': function(data, type, row, meta) { - try { - return decodeURIComponent(escape(data.description)) - } catch(err){ - return escape(data.description) - } - } - }, - { - 'title': "
Location
", - 'type': "natural-minus-na", - 'width': "125px", - 'data': "location" - }, - { - 'title': "
Mean
", - 'type': "natural-minus-na", - 'width': "30px", - 'data': "mean", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak  
LOD  
", - 'type': "natural-minus-na", - 'data': "lod_score", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak Location
", - 'type': "natural-minus-na", - 'width': "125px", - 'data': "lrs_location" - }, - { - 'title': "
Effect  
Size  
", - 'type': "natural-minus-na", - 'data': "additive", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }{% elif dataset.type == 'Publish' %}, - { - 'title': "Description", - 'type': "natural", - 'width': "500px", - 'data': null, - 'render': function(data, type, row, meta) { - try { - return decodeURIComponent(escape(data.description)) - } catch(err){ - return data.description - } + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak  
LOD  
", + 'type': "natural-minus-na", + 'data': "lod_score", + 'width': "60px", + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak Location
", + 'type': "natural-minus-na", + 'width': "120px", + 'data': "lrs_location" + }, + { + 'title': "
Effect  
Size  
", + 'type': "natural-minus-na", + 'width': "60px", + 'data': "additive", + 'orderSequence': [ "desc", "asc"] + }{% elif dataset.type == 'Geno' %}, + { + 'title': "
Location
", + 'type': "natural-minus-na", + 'width': "120px", + 'data': "location" + }{% endif %} + ]; + + loadDataTable(); + + function loadDataTable(){ + //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters + trait_table = $('#' + tableId).DataTable( { + 'drawCallback': function( settings ) { + $('#' + tableId + ' tr').off().on("click", function(event) { + if (event.target.type !== 'checkbox' && event.target.tagName.toLowerCase() !== 'a') { + var obj =$(this).find('input'); + obj.prop('checked', !obj.is(':checked')); + } + if ($(this).hasClass("selected") && event.target.tagName.toLowerCase() !== 'a'){ + $(this).removeClass("selected") + } else if (event.target.tagName.toLowerCase() !== 'a') { + $(this).addClass("selected") + } + change_buttons() + }); + }, + 'createdRow': function ( row, data, index ) { + $('td', row).eq(0).attr("style", "text-align: center; padding: 0px 10px 2px 10px;"); + $('td', row).eq(1).attr("align", "right"); + $('td', row).eq(1).attr('data-export', index+1); + $('td', row).eq(2).attr('data-export', $('td', row).eq(2).text()); + {% if dataset.type == 'ProbeSet' %} + $('td', row).eq(3).attr('title', $('td', row).eq(3).text()); + $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); + if ($('td', row).eq(3).text().length > 20) { + $('td', row).eq(3).text($('td', row).eq(3).text().substring(0, 20)); + $('td', row).eq(3).text($('td', row).eq(3).text() + '...') } - }, - { - 'title': "
Mean
", - 'type': "natural-minus-na", - 'width': "30px", - 'data': "mean", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "Authors", - 'type': "natural", - 'width': "300px", - 'data': null, - 'render': function(data, type, row, meta) { - author_list = data.authors.split(",") - if (author_list.length >= 6) { - author_string = author_list.slice(0, 6).join(",") + ", et al." - } else{ - author_string = data.authors - } - return author_string + $('td', row).eq(4).attr('title', $('td', row).eq(4).text()); + $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text()); + $('td', row).slice(5,10).attr("align", "right"); + $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text()); + $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text()); + $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text()); + $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text()); + $('td', row).eq(9).attr('data-export', $('td', row).eq(9).text()); + {% elif dataset.type == 'Publish' %} + $('td', row).eq(3).attr('title', $('td', row).eq(3).text()); + $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); + $('td', row).eq(4).attr('title', $('td', row).eq(4).text()); + $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text()); + $('td', row).eq(4).attr('align', 'right'); + $('td', row).slice(6,10).attr("align", "right"); + $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text()); + $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text()); + $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text()); + $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text()); + $('td', row).eq(9).attr('data-export', $('td', row).eq(8).text()); + {% elif dataset.type == 'Geno' %} + $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); + {% endif %} + }, + "data": trait_list, + "columns": columnDefs, + "order": [[1, "asc" ]], + "sDom": "iti", + "destroy": true, + "autoWidth": false, + "deferRender": true, + "bSortClasses": false, + {% if trait_list|length > 20 %} + "scrollY": "100vh", + "scroller": true, + "scrollCollapse": true, + {% else %} + "iDisplayLength": -1, + {% endif %} + "initComplete": function (settings) { + //Add JQueryUI resizable functionality to each th in the ScrollHead table + $('#' + tableId + '_wrapper .dataTables_scrollHead thead th').resizable({ + handles: "e", + alsoResize: '#' + tableId + '_wrapper .dataTables_scrollHead table', //Not essential but makes the resizing smoother + stop: function () { + saveColumnSettings(); + loadDataTable(); } - }, - { - 'title': "
Year
", - 'type': "natural-minus-na", - 'data': null, - 'width': "25px", - 'render': function(data, type, row, meta) { - if (data.pubmed_id != "N/A"){ - return '' + data.pubmed_text + '' - } else { - return data.pubmed_text - } - }, - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak  
LOD  
", - 'type': "natural-minus-na", - 'data': "lod_score", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak Location
", - 'type': "natural-minus-na", - 'width': "120px", - 'data': "lrs_location" - }, - { - 'title': "
Effect  
Size  
", - 'type': "natural-minus-na", - 'width': "60px", - 'data': "additive", - 'orderSequence': [ "desc", "asc"] - }{% elif dataset.type == 'Geno' %}, - { - 'title': "
Location
", - 'type': "natural-minus-na", - 'width': "120px", - 'data': "location" - }{% endif %} - ], - "order": [[1, "asc" ]], - 'sDom': "iti", - "autoWidth": true, - "bSortClasses": false, - {% if trait_list|length > 20 %} - "scrollY": "100vh", - "scroller": true, - "scrollCollapse": true - {% else %} - "iDisplayLength": -1 - {% endif %} - } ); + }); + }, + } ); + } + + function setUserColumnsDefWidths() { + + var userColumnDef; + + // Get the settings for this table from localStorage + var userColumnDefs = JSON.parse(localStorage.getItem(tableId)) || []; + + if (userColumnDefs.length === 0 ) return; + + columnDefs.forEach( function(columnDef) { + + // Check if there is a width specified for this column + userColumnDef = userColumnDefs.find( function(column) { + return column.targets === columnDef.targets; + }); + + // If there is, set the width of this columnDef in px + if ( userColumnDef ) { + + columnDef.width = userColumnDef.width + 'px'; + + } + + }); + + } + + + function saveColumnSettings() { + + var userColumnDefs = JSON.parse(localStorage.getItem(tableId)) || []; + + var width, header, existingSetting; + + trait_table.columns().every( function ( targets ) { + + // Check if there is a setting for this column in localStorage + existingSetting = userColumnDefs.findIndex( function(column) { return column.targets === targets;}); + + // Get the width of this column + header = this.header(); + width = $(header).width(); + + if ( existingSetting !== -1 ) { + + // Update the width + userColumnDefs[existingSetting].width = width; + + } else { + + // Add the width for this column + userColumnDefs.push({ + targets: targets, + width: width, + }); + + } + + }); + + // Save (or update) the settings in localStorage + localStorage.setItem(tableId, JSON.stringify(userColumnDefs)); + + } - trait_table.draw(); //ZS: This makes the table adjust its height properly on initial load + //trait_table.draw(); //ZS: This makes the table adjust its height properly on initial load $('.toggle-vis').on( 'click', function (e) { e.preventDefault(); @@ -409,7 +496,7 @@ $('#redraw').click(function() { - var table = $('#trait_table').DataTable(); + var table = $('#' + tableId).DataTable(); table.colReorder.reset() }); -- cgit v1.2.3 From c81255153ac44462ee378908af12646a2697933c Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 11 Mar 2021 22:35:11 +0000 Subject: Added some CSS to make cell content cut off with ellipses if a column is narrower than the content's width --- wqflask/wqflask/static/new/css/trait_list.css | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/wqflask/wqflask/static/new/css/trait_list.css b/wqflask/wqflask/static/new/css/trait_list.css index c7249721..6d49f009 100644 --- a/wqflask/wqflask/static/new/css/trait_list.css +++ b/wqflask/wqflask/static/new/css/trait_list.css @@ -52,3 +52,21 @@ div.dts div.dataTables_paginate,div.dts div.dataTables_length{ display:none } +/* This is the important CSS */ +table.dataTable { + table-layout: fixed; + width: 100%; + white-space: nowrap; + overflow: hidden; +} + +table.dataTable th { + white-space: nowrap; + overflow: hidden; +} + +table.dataTable td { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} \ No newline at end of file -- cgit v1.2.3 From a3d4cdd47ec843612c4e33a5c45db9e152bb81c6 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 11 Mar 2021 22:43:35 +0000 Subject: Made a variety of changes to make column resizeability work - the main issue was related to there being both an sWidth and width parameter --- wqflask/wqflask/templates/search_result_page.html | 317 +++++++++++----------- 1 file changed, 166 insertions(+), 151 deletions(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 13f3c7c1..2cf1856b 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -125,8 +125,8 @@ {% endif %} {% endif %} -
-

Loading...
+
+
@@ -175,161 +175,174 @@ var tableId = "trait_table"; columnDefs = [ - { - 'data': null, - 'width': "25px", - 'orderDataType': "dom-checkbox", - 'orderable': false, - 'render': function(data, type, row, meta) { - return '' - } - }, - { - 'title': "Index", - 'type': "natural", - 'width': "30px", - 'data': "index" - }, - { - 'title': "Record", - 'type': "natural-minus-na", - 'data': null, - 'width': "60px", - 'render': function(data, type, row, meta) { - return '' + data.display_name + '' + { + 'data': null, + 'width': "25px", + 'orderDataType': "dom-checkbox", + 'orderable': false, + 'targets': 0, + 'render': function(data, type, row, meta) { + return '' + } + }, + { + 'title': "Index", + 'type': "natural", + 'width': "30px", + 'targets': 1, + 'data': "index" + }, + { + 'title': "Record", + 'type': "natural-minus-na", + 'data': null, + 'width': "60px", + 'targets': 2, + 'render': function(data, type, row, meta) { + return '' + data.display_name + '' + } + }{% if dataset.type == 'ProbeSet' %}, + { + 'title': "Symbol", + 'type': "natural", + 'width': "120px", + 'targets': 3, + 'data': "symbol" + }, + { + 'title': "Description", + 'type': "natural", + 'data': null, + 'targets': 4, + 'render': function(data, type, row, meta) { + try { + return decodeURIComponent(escape(data.description)) + } catch(err){ + return escape(data.description) } - }{% if dataset.type == 'ProbeSet' %}, - { - 'title': "Symbol", - 'type': "natural", - 'width': "120px", - 'data': "symbol" - }, - { - 'title': "Description", - 'type': "natural", - 'data': null, - 'render': function(data, type, row, meta) { - try { + } + }, + { + 'title': "
Location
", + 'type': "natural-minus-na", + 'width': "125px", + 'targets': 5, + 'data': "location" + }, + { + 'title': "
Mean
", + 'type': "natural-minus-na", + 'width': "30px", + 'data': "mean", + 'targets': 6, + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak  
LOD  
", + 'type': "natural-minus-na", + 'data': "lod_score", + 'width': "60px", + 'targets': 7, + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak Location
", + 'type': "natural-minus-na", + 'width': "125px", + 'targets': 8, + 'data': "lrs_location" + }, + { + 'title': "
Effect  
Size  
", + 'type': "natural-minus-na", + 'data': "additive", + 'width': "60px", + 'targets': 9, + 'orderSequence': [ "desc", "asc"] + }{% elif dataset.type == 'Publish' %}, + { + 'title': "Description", + 'type': "natural", + 'width': "500px", + 'data': null, + 'render': function(data, type, row, meta) { + try { return decodeURIComponent(escape(data.description)) - } catch(err){ - return escape(data.description) - } + } catch(err){ + return data.description } - }, - { - 'title': "
Location
", - 'type': "natural-minus-na", - 'width': "125px", - 'data': "location" - }, - { - 'title': "
Mean
", - 'type': "natural-minus-na", - 'width': "30px", - 'data': "mean", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak  
LOD  
", - 'type': "natural-minus-na", - 'data': "lod_score", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak Location
", - 'type': "natural-minus-na", - 'width': "125px", - 'data': "lrs_location" - }, - { - 'title': "
Effect  
Size  
", - 'type': "natural-minus-na", - 'data': "additive", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }{% elif dataset.type == 'Publish' %}, - { - 'title': "Description", - 'type': "natural", - 'width': "500px", - 'data': null, - 'render': function(data, type, row, meta) { - try { - return decodeURIComponent(escape(data.description)) - } catch(err){ - return data.description - } + } + }, + { + 'title': "
Mean
", + 'type': "natural-minus-na", + 'width': "30px", + 'data': "mean", + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "Authors", + 'type': "natural", + 'width': "300px", + 'data': null, + 'render': function(data, type, row, meta) { + author_list = data.authors.split(",") + if (author_list.length >= 6) { + author_string = author_list.slice(0, 6).join(",") + ", et al." + } else{ + author_string = data.authors } - }, - { - 'title': "
Mean
", - 'type': "natural-minus-na", - 'width': "30px", - 'data': "mean", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "Authors", - 'type': "natural", - 'width': "300px", - 'data': null, - 'render': function(data, type, row, meta) { - author_list = data.authors.split(",") - if (author_list.length >= 6) { - author_string = author_list.slice(0, 6).join(",") + ", et al." - } else{ - author_string = data.authors - } - return author_string + return author_string + } + }, + { + 'title': "
Year
", + 'type': "natural-minus-na", + 'data': null, + 'width': "25px", + 'render': function(data, type, row, meta) { + if (data.pubmed_id != "N/A"){ + return '' + data.pubmed_text + '' + } else { + return data.pubmed_text } }, - { - 'title': "
Year
", - 'type': "natural-minus-na", - 'data': null, - 'width': "25px", - 'render': function(data, type, row, meta) { - if (data.pubmed_id != "N/A"){ - return '' + data.pubmed_text + '' - } else { - return data.pubmed_text - } - }, - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak  
LOD  
", - 'type': "natural-minus-na", - 'data': "lod_score", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "
Peak Location
", - 'type': "natural-minus-na", - 'width': "120px", - 'data': "lrs_location" - }, - { - 'title': "
Effect  
Size  
", - 'type': "natural-minus-na", - 'width': "60px", - 'data': "additive", - 'orderSequence': [ "desc", "asc"] - }{% elif dataset.type == 'Geno' %}, - { - 'title': "
Location
", - 'type': "natural-minus-na", - 'width': "120px", - 'data': "location" - }{% endif %} + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak  
LOD  
", + 'type': "natural-minus-na", + 'data': "lod_score", + 'width': "60px", + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "
Peak Location
", + 'type': "natural-minus-na", + 'width': "120px", + 'data': "lrs_location" + }, + { + 'title': "
Effect  
Size  
", + 'type': "natural-minus-na", + 'width': "60px", + 'data': "additive", + 'orderSequence': [ "desc", "asc"] + }{% elif dataset.type == 'Geno' %}, + { + 'title': "
Location
", + 'type': "natural-minus-na", + 'width': "120px", + 'data': "location" + }{% endif %} ]; loadDataTable(); function loadDataTable(){ + + setUserColumnsDefWidths(); + //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters trait_table = $('#' + tableId).DataTable( { 'drawCallback': function( settings ) { @@ -391,7 +404,8 @@ "deferRender": true, "bSortClasses": false, {% if trait_list|length > 20 %} - "scrollY": "100vh", + "scrollY": "500px", + "scrollX": true, "scroller": true, "scrollCollapse": true, {% else %} @@ -411,6 +425,10 @@ } ); } + window.addEventListener('resize', function(){ + trait_table.columns.adjust(); + }); + function setUserColumnsDefWidths() { var userColumnDef; @@ -430,12 +448,12 @@ // If there is, set the width of this columnDef in px if ( userColumnDef ) { + columnDef.sWidth = userColumnDef.width + 'px'; columnDef.width = userColumnDef.width + 'px'; } }); - } @@ -473,11 +491,8 @@ // Save (or update) the settings in localStorage localStorage.setItem(tableId, JSON.stringify(userColumnDefs)); - } - //trait_table.draw(); //ZS: This makes the table adjust its height properly on initial load - $('.toggle-vis').on( 'click', function (e) { e.preventDefault(); -- cgit v1.2.3 From 8d0513d9c667f076bca1c54037bab740b04c1bbd Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 11 Mar 2021 22:57:49 +0000 Subject: Added a parameter to prevent columnDefs from being overwritten on initial table generation + changed the default table width to 100% --- wqflask/wqflask/templates/search_result_page.html | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 2cf1856b..6321c687 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -125,8 +125,8 @@ {% endif %} {% endif %} -
-

Loading...
+
+
@@ -337,11 +337,13 @@ }{% endif %} ]; - loadDataTable(); + loadDataTable(true); - function loadDataTable(){ + function loadDataTable(first_run=false){ - setUserColumnsDefWidths(); + if (!first_run){ + setUserColumnsDefWidths(); + } //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters trait_table = $('#' + tableId).DataTable( { -- cgit v1.2.3 From aa3625be807a661e44c7f0c62a539eec27ccca46 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 11 Mar 2021 23:08:43 +0000 Subject: Set scrollX in all cases so the adjustablee columns also work for shorter results that don't use Scroller (<20 results) --- wqflask/wqflask/templates/search_result_page.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 6321c687..06322a86 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -405,9 +405,9 @@ "autoWidth": false, "deferRender": true, "bSortClasses": false, + "scrollX": true, {% if trait_list|length > 20 %} "scrollY": "500px", - "scrollX": true, "scroller": true, "scrollCollapse": true, {% else %} -- cgit v1.2.3 From 682a7eeecc9f80836d11780feeabc3afab57b3e7 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 24 Mar 2021 19:31:04 +0000 Subject: Use autoWidth on initial table load, though I'll change this to only apply when certain fields (like Description) are below a certain width Added some code tracking the change in width when a column's width is adjusted, so that the table_container width can be changed accordingly --- wqflask/wqflask/templates/search_result_page.html | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 06322a86..601fd63a 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -126,7 +126,7 @@ {% endif %}
-

Loading...
+
@@ -174,6 +174,8 @@ var tableId = "trait_table"; + var width_change = 0; //ZS: For storing the change in width so overall table width can be adjusted by that amount + columnDefs = [ { 'data': null, @@ -196,7 +198,6 @@ 'title': "Record", 'type': "natural-minus-na", 'data': null, - 'width': "60px", 'targets': 2, 'render': function(data, type, row, meta) { return '' + data.display_name + '' @@ -232,7 +233,7 @@ { 'title': "
Mean
", 'type': "natural-minus-na", - 'width': "30px", + 'width': "40px", 'data': "mean", 'targets': 6, 'orderSequence': [ "desc", "asc"] @@ -346,7 +347,7 @@ } //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters - trait_table = $('#' + tableId).DataTable( { + table_settings = { 'drawCallback': function( settings ) { $('#' + tableId + ' tr').off().on("click", function(event) { if (event.target.type !== 'checkbox' && event.target.tagName.toLowerCase() !== 'a') { @@ -402,7 +403,7 @@ "order": [[1, "asc" ]], "sDom": "iti", "destroy": true, - "autoWidth": false, + "autoWidth": true, "deferRender": true, "bSortClasses": false, "scrollX": true, @@ -418,13 +419,22 @@ $('#' + tableId + '_wrapper .dataTables_scrollHead thead th').resizable({ handles: "e", alsoResize: '#' + tableId + '_wrapper .dataTables_scrollHead table', //Not essential but makes the resizing smoother + resize: function( event, ui ) { + width_change = ui.size.width - ui.originalSize.width; + }, stop: function () { saveColumnSettings(); loadDataTable(); } }); }, - } ); + } + + if (!first_run){ + table_settings['autoWidth'] = false; + $('#table_container').css("width", String($('#trait_table').width() + width_change) + "px"); //ZS : Change the container width by the change in width of the adjusted column, so the overall table size adjusts properly + } + trait_table = $('#' + tableId).DataTable(table_settings); } window.addEventListener('resize', function(){ @@ -488,7 +498,6 @@ }); } - }); // Save (or update) the settings in localStorage -- cgit v1.2.3 From 56e3de19c28372ff618ef3aba483c4f717fafd91 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 24 Mar 2021 22:12:14 +0000 Subject: Store a dictionary of maximum widths for each field + create a variable indicating if there are any wide columns (too wide to use DataTables' 'autoWidth' setting) --- wqflask/wqflask/search_results.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 3cbda3dd..3095708d 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -101,7 +101,6 @@ class SearchResultPage: self.dataset.group.name) # result_set represents the results for each search term; a search of # "shh grin2b" would have two sets of results, one for each term - logger.debug("self.results is:", pf(self.results)) for index, result in enumerate(self.results): if not result: @@ -169,6 +168,19 @@ class SearchResultPage: trait_dict[key] = trait_dict[key].decode('utf-8') trait_list.append(trait_dict) + self.max_widths = {} + for i, trait in enumerate(trait_list): + for key in trait.keys(): + self.max_widths[key] = max(len(str(trait[key])), self.max_widths[key]) if key in self.max_widths else len(str(trait[key])) + + self.wide_columns_exist = False + if this_trait.dataset.type == "Publish": + if (self.max_widths['display_name'] > 25 or self.max_widths['description'] > 100 or self.max_widths['authors']> 80): + self.wide_columns_exist = True + if this_trait.dataset.type == "ProbeSet": + if (self.max_widths['display_name'] > 25 or self.max_widths['symbol'] > 25 or self.max_widths['description'] > 100): + self.wide_columns_exist = True + self.trait_list = trait_list if self.dataset.type == "ProbeSet": -- cgit v1.2.3 From c73f55287b986844d4c894e80a3cb76c1f05d8f8 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 24 Mar 2021 22:12:57 +0000 Subject: Added some default widths for situations where some wide columns exist and autoWidth can't be used on initial load + changed the logic for how to recalculate table width when a column is manually resized --- wqflask/wqflask/templates/search_result_page.html | 55 +++++++++++++++++------ 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 601fd63a..1826c694 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -125,7 +125,7 @@ {% endif %} {% endif %} -
+

Loading...
@@ -190,23 +190,29 @@ { 'title': "Index", 'type': "natural", - 'width': "30px", + 'width': "35px", 'targets': 1, 'data': "index" - }, + } + {% if dataset.type == 'ProbeSet' %}, { 'title': "Record", 'type': "natural-minus-na", 'data': null, + {% if wide_columns_exist == true %} + 'width': "{{ max_widths.display_name * 5 }}px", + {% endif %} 'targets': 2, 'render': function(data, type, row, meta) { return '' + data.display_name + '' } - }{% if dataset.type == 'ProbeSet' %}, + }, { 'title': "Symbol", 'type': "natural", - 'width': "120px", + {% if wide_columns_exist == true %} + 'width': "200px", + {% endif %} 'targets': 3, 'data': "symbol" }, @@ -214,6 +220,9 @@ 'title': "Description", 'type': "natural", 'data': null, + {% if wide_columns_exist == true %} + 'width': "600px", + {% endif %} 'targets': 4, 'render': function(data, type, row, meta) { try { @@ -261,10 +270,21 @@ 'targets': 9, 'orderSequence': [ "desc", "asc"] }{% elif dataset.type == 'Publish' %}, + { + 'title': "Record", + 'type': "natural-minus-na", + 'data': null, + {% if wide_columns_exist == true %} + 'width': "100px", + {% endif %} + 'targets': 2, + 'render': function(data, type, row, meta) { + return '' + data.display_name + '' + } + }, { 'title': "Description", 'type': "natural", - 'width': "500px", 'data': null, 'render': function(data, type, row, meta) { try { @@ -277,14 +297,16 @@ { 'title': "
Mean
", 'type': "natural-minus-na", - 'width': "30px", + 'width': "60px", 'data': "mean", 'orderSequence': [ "desc", "asc"] }, { 'title': "Authors", 'type': "natural", - 'width': "300px", + {% if wide_columns_exist == true %} + 'width': "400px", + {% endif %} 'data': null, 'render': function(data, type, row, meta) { author_list = data.authors.split(",") @@ -300,7 +322,7 @@ 'title': "
Year
", 'type': "natural-minus-na", 'data': null, - 'width': "25px", + 'width': "50px", 'render': function(data, type, row, meta) { if (data.pubmed_id != "N/A"){ return '' + data.pubmed_text + '' @@ -403,7 +425,11 @@ "order": [[1, "asc" ]], "sDom": "iti", "destroy": true, + {% if wide_columns_exist != true %} "autoWidth": true, + {% else %} + "autoWidth": false, + {% endif %} "deferRender": true, "bSortClasses": false, "scrollX": true, @@ -432,9 +458,15 @@ if (!first_run){ table_settings['autoWidth'] = false; - $('#table_container').css("width", String($('#trait_table').width() + width_change) + "px"); //ZS : Change the container width by the change in width of the adjusted column, so the overall table size adjusts properly + $('#table_container').css("width", String($('#trait_table').width() + width_change {% if trait_list|length > 20 %}+ 17{% endif %}) + "px"); //ZS : Change the container width by the change in width of the adjusted column, so the overall table size adjusts properly } trait_table = $('#' + tableId).DataTable(table_settings); + + {% if trait_list|length > 20 %} + if (first_run){ + $('#table_container').css("width", String($('#trait_table').width() + 17) + "px"); + } + {% endif %} } window.addEventListener('resize', function(){ @@ -442,7 +474,6 @@ }); function setUserColumnsDefWidths() { - var userColumnDef; // Get the settings for this table from localStorage @@ -468,9 +499,7 @@ }); } - function saveColumnSettings() { - var userColumnDefs = JSON.parse(localStorage.getItem(tableId)) || []; var width, header, existingSetting; -- cgit v1.2.3 From 62667ad7bd1a9a9fb555a87c873d7a54ab7f03f9 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 26 Mar 2021 19:39:15 +0000 Subject: Added checks in template for whether to explicitly set column width when wide column contents exist --- wqflask/wqflask/templates/search_result_page.html | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 1826c694..c165ad3c 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -179,7 +179,9 @@ columnDefs = [ { 'data': null, + {% if wide_columns_exist == true %} 'width': "25px", + {% endif %} 'orderDataType': "dom-checkbox", 'orderable': false, 'targets': 0, @@ -190,7 +192,9 @@ { 'title': "Index", 'type': "natural", + {% if wide_columns_exist == true %} 'width': "35px", + {% endif %} 'targets': 1, 'data': "index" } @@ -235,14 +239,18 @@ { 'title': "
Location
", 'type': "natural-minus-na", + {% if wide_columns_exist == true %} 'width': "125px", + {% endif %} 'targets': 5, 'data': "location" }, { 'title': "
Mean
", 'type': "natural-minus-na", + {% if wide_columns_exist == true %} 'width': "40px", + {% endif %} 'data': "mean", 'targets': 6, 'orderSequence': [ "desc", "asc"] @@ -251,14 +259,17 @@ 'title': "
Peak  
LOD  
", 'type': "natural-minus-na", 'data': "lod_score", - 'width': "60px", + {% if wide_columns_exist == true %} + 'width': "60px", {% endif %} 'targets': 7, 'orderSequence': [ "desc", "asc"] }, { 'title': "
Peak Location
", 'type': "natural-minus-na", + {% if wide_columns_exist == true %} 'width': "125px", + {% endif %} 'targets': 8, 'data': "lrs_location" }, @@ -266,7 +277,9 @@ 'title': "
Effect  
Size  
", 'type': "natural-minus-na", 'data': "additive", + {% if wide_columns_exist == true %} 'width': "60px", + {% endif %} 'targets': 9, 'orderSequence': [ "desc", "asc"] }{% elif dataset.type == 'Publish' %}, -- cgit v1.2.3 From d1beaa5ac7a3eea343d8447d771f7661173fabac Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 19 Apr 2021 21:15:14 +0000 Subject: Fixed resizeable columns to work with show/hide columns option + keep rows checked after resizing --- wqflask/wqflask/templates/search_result_page.html | 79 ++++++++++++++++++++--- 1 file changed, 70 insertions(+), 9 deletions(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index c165ad3c..eac3ec61 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -224,9 +224,6 @@ 'title': "Description", 'type': "natural", 'data': null, - {% if wide_columns_exist == true %} - 'width': "600px", - {% endif %} 'targets': 4, 'render': function(data, type, row, meta) { try { @@ -260,7 +257,8 @@ 'type': "natural-minus-na", 'data': "lod_score", {% if wide_columns_exist == true %} - 'width': "60px", {% endif %} + 'width': "60px", + {% endif %} 'targets': 7, 'orderSequence': [ "desc", "asc"] }, @@ -299,6 +297,7 @@ 'title': "Description", 'type': "natural", 'data': null, + 'targets': 3, 'render': function(data, type, row, meta) { try { return decodeURIComponent(escape(data.description)) @@ -310,8 +309,11 @@ { 'title': "
Mean
", 'type': "natural-minus-na", + {% if wide_columns_exist == true %} 'width': "60px", + {% endif %} 'data': "mean", + 'targets': 4, 'orderSequence': [ "desc", "asc"] }, { @@ -321,6 +323,7 @@ 'width': "400px", {% endif %} 'data': null, + 'targets': 5, 'render': function(data, type, row, meta) { author_list = data.authors.split(",") if (author_list.length >= 6) { @@ -335,7 +338,10 @@ 'title': "
Year
", 'type': "natural-minus-na", 'data': null, + {% if wide_columns_exist == true %} 'width': "50px", + {% endif %} + 'targets': 6, 'render': function(data, type, row, meta) { if (data.pubmed_id != "N/A"){ return '' + data.pubmed_text + '' @@ -349,30 +355,70 @@ 'title': "
Peak  
LOD  
", 'type': "natural-minus-na", 'data': "lod_score", + 'targets': 7, + {% if wide_columns_exist == true %} 'width': "60px", + {% endif %} 'orderSequence': [ "desc", "asc"] }, { 'title': "
Peak Location
", 'type': "natural-minus-na", + {% if wide_columns_exist == true %} 'width': "120px", + {% endif %} + 'targets': 8, 'data': "lrs_location" }, { 'title': "
Effect  
Size  
", 'type': "natural-minus-na", + {% if wide_columns_exist == true %} 'width': "60px", + {% endif %} 'data': "additive", + 'targets': 9, 'orderSequence': [ "desc", "asc"] }{% elif dataset.type == 'Geno' %}, { 'title': "
Location
", 'type': "natural-minus-na", + {% if wide_columns_exist == true %} 'width': "120px", + {% endif %} + 'targets': 2, 'data': "location" }{% endif %} ]; + recheck_rows = function(checked_rows){ + //ZS: This is meant to recheck checkboxes after columns are resized + check_cells = trait_table.column(0).nodes().to$(); + for (let i = 0; i < check_cells.length; i++) { + if (checked_rows.includes(i)){ + check_cells[i].childNodes[0].checked = true; + } + } + + check_rows = trait_table.rows().nodes(); + for (let i =0; i < check_rows.length; i++) { + if (checked_rows.includes(i)){ + check_rows[i].classList.add("selected") + } + } + } + + get_checked_rows = function(trait_table){ + let checked_rows = [] + $("#trait_table input").each(function(index){ + if ($(this).prop("checked") == true){ + checked_rows.push(index); + } + }); + + return checked_rows + } + loadDataTable(true); function loadDataTable(first_run=false){ @@ -473,13 +519,22 @@ table_settings['autoWidth'] = false; $('#table_container').css("width", String($('#trait_table').width() + width_change {% if trait_list|length > 20 %}+ 17{% endif %}) + "px"); //ZS : Change the container width by the change in width of the adjusted column, so the overall table size adjusts properly } + + let checked_rows = get_checked_rows(); trait_table = $('#' + tableId).DataTable(table_settings); + if (checked_rows.length > 0){ + recheck_rows(checked_rows); + } - {% if trait_list|length > 20 %} if (first_run){ + {% if trait_list|length > 20 %} $('#table_container').css("width", String($('#trait_table').width() + 17) + "px"); + {% else %} + {% if wide_columns_exist != true %} + $('#table_container').css("width", String($('#trait_table').width()) + "px"); + {% endif %} + {% endif %} } - {% endif %} } window.addEventListener('resize', function(){ @@ -507,8 +562,16 @@ columnDef.sWidth = userColumnDef.width + 'px'; columnDef.width = userColumnDef.width + 'px'; + $('.toggle-vis').each(function(){ + if ($(this).attr('data-column') == columnDef.targets){ + if ($(this).hasClass("active")){ + columnDef.bVisible = false + } else { + columnDef.bVisible = true + } + } + }) } - }); } @@ -527,10 +590,8 @@ width = $(header).width(); if ( existingSetting !== -1 ) { - // Update the width userColumnDefs[existingSetting].width = width; - } else { // Add the width for this column -- cgit v1.2.3 From dc70723199c1cf84d4debee01a86646650afbba9 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 22 Apr 2021 18:19:52 +0000 Subject: Temporarily replacing jquery-ui with the version that works with resizing DataTables columns --- wqflask/wqflask/templates/base.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index 12dddf89..1a0335b6 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -255,7 +255,8 @@ }) - + + -- cgit v1.2.3 From 2d9220c3d88f5f818ae81e3cafdc69288c9cfbd8 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 4 May 2021 19:24:15 +0000 Subject: Changed recheck_rows to include the DataTable as input in preparation for moving it into a separate JS file that can be imported by all table templates --- wqflask/wqflask/templates/search_result_page.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index eac3ec61..279a1886 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -391,9 +391,9 @@ }{% endif %} ]; - recheck_rows = function(checked_rows){ + recheck_rows = function(the_table, checked_rows){ //ZS: This is meant to recheck checkboxes after columns are resized - check_cells = trait_table.column(0).nodes().to$(); + check_cells = the_table.column(0).nodes().to$(); for (let i = 0; i < check_cells.length; i++) { if (checked_rows.includes(i)){ check_cells[i].childNodes[0].checked = true; @@ -523,7 +523,7 @@ let checked_rows = get_checked_rows(); trait_table = $('#' + tableId).DataTable(table_settings); if (checked_rows.length > 0){ - recheck_rows(checked_rows); + recheck_rows(trait_table, checked_rows); } if (first_run){ -- cgit v1.2.3 From 304128b1231df425709fbe3b128bd0c5eda4139b Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 4 May 2021 19:29:01 +0000 Subject: Include table ID as input for get_checked_rows --- wqflask/wqflask/templates/search_result_page.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 279a1886..adc8c264 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -408,9 +408,9 @@ } } - get_checked_rows = function(trait_table){ + get_checked_rows = function(table_id){ let checked_rows = [] - $("#trait_table input").each(function(index){ + $("#" + table_id + " input").each(function(index){ if ($(this).prop("checked") == true){ checked_rows.push(index); } @@ -520,7 +520,7 @@ $('#table_container').css("width", String($('#trait_table').width() + width_change {% if trait_list|length > 20 %}+ 17{% endif %}) + "px"); //ZS : Change the container width by the change in width of the adjusted column, so the overall table size adjusts properly } - let checked_rows = get_checked_rows(); + let checked_rows = get_checked_rows(tableId); trait_table = $('#' + tableId).DataTable(table_settings); if (checked_rows.length > 0){ recheck_rows(trait_table, checked_rows); -- cgit v1.2.3 From 16338316a7529265aaa94d541698c8ed8cc97aba Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 4 May 2021 19:35:31 +0000 Subject: Include table/table_id as parameters in setUseerColumnsDefWidths and saveColumnSettings --- wqflask/wqflask/templates/search_result_page.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index adc8c264..a351256b 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -424,7 +424,7 @@ function loadDataTable(first_run=false){ if (!first_run){ - setUserColumnsDefWidths(); + setUserColumnsDefWidths(tableId); } //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters @@ -508,7 +508,7 @@ width_change = ui.size.width - ui.originalSize.width; }, stop: function () { - saveColumnSettings(); + saveColumnSettings(tableId, trait_table); loadDataTable(); } }); @@ -541,11 +541,11 @@ trait_table.columns.adjust(); }); - function setUserColumnsDefWidths() { + function setUserColumnsDefWidths(table_id) { var userColumnDef; // Get the settings for this table from localStorage - var userColumnDefs = JSON.parse(localStorage.getItem(tableId)) || []; + var userColumnDefs = JSON.parse(localStorage.getItem(table_id)) || []; if (userColumnDefs.length === 0 ) return; @@ -575,8 +575,8 @@ }); } - function saveColumnSettings() { - var userColumnDefs = JSON.parse(localStorage.getItem(tableId)) || []; + function saveColumnSettings(table_id, trait_table) { + var userColumnDefs = JSON.parse(localStorage.getItem(table_id)) || []; var width, header, existingSetting; -- cgit v1.2.3 From fe224fb35f926373ff75c9a19217a4602ca3ea99 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 4 May 2021 19:40:09 +0000 Subject: Moved several table functions out of search_results.html and into new file table_functions.js --- .../static/new/javascript/table_functions.js | 93 +++++++++++++++++++++ wqflask/wqflask/templates/search_result_page.html | 96 +--------------------- 2 files changed, 94 insertions(+), 95 deletions(-) create mode 100644 wqflask/wqflask/static/new/javascript/table_functions.js diff --git a/wqflask/wqflask/static/new/javascript/table_functions.js b/wqflask/wqflask/static/new/javascript/table_functions.js new file mode 100644 index 00000000..e24b3e03 --- /dev/null +++ b/wqflask/wqflask/static/new/javascript/table_functions.js @@ -0,0 +1,93 @@ +recheck_rows = function(the_table, checked_rows){ + //ZS: This is meant to recheck checkboxes after columns are resized + check_cells = the_table.column(0).nodes().to$(); + for (let i = 0; i < check_cells.length; i++) { + if (checked_rows.includes(i)){ + check_cells[i].childNodes[0].checked = true; + } + } + + check_rows = trait_table.rows().nodes(); + for (let i =0; i < check_rows.length; i++) { + if (checked_rows.includes(i)){ + check_rows[i].classList.add("selected") + } + } + } + +get_checked_rows = function(table_id){ + let checked_rows = [] + $("#" + table_id + " input").each(function(index){ + if ($(this).prop("checked") == true){ + checked_rows.push(index); + } + }); + + return checked_rows +} + +function setUserColumnsDefWidths(table_id) { + var userColumnDef; + + // Get the settings for this table from localStorage + var userColumnDefs = JSON.parse(localStorage.getItem(table_id)) || []; + + if (userColumnDefs.length === 0 ) return; + + columnDefs.forEach( function(columnDef) { + + // Check if there is a width specified for this column + userColumnDef = userColumnDefs.find( function(column) { + return column.targets === columnDef.targets; + }); + + // If there is, set the width of this columnDef in px + if ( userColumnDef ) { + + columnDef.sWidth = userColumnDef.width + 'px'; + columnDef.width = userColumnDef.width + 'px'; + + $('.toggle-vis').each(function(){ + if ($(this).attr('data-column') == columnDef.targets){ + if ($(this).hasClass("active")){ + columnDef.bVisible = false + } else { + columnDef.bVisible = true + } + } + }) + } + }); +} + +function saveColumnSettings(table_id, trait_table) { +var userColumnDefs = JSON.parse(localStorage.getItem(table_id)) || []; + +var width, header, existingSetting; + +trait_table.columns().every( function ( targets ) { + + // Check if there is a setting for this column in localStorage + existingSetting = userColumnDefs.findIndex( function(column) { return column.targets === targets;}); + + // Get the width of this column + header = this.header(); + width = $(header).width(); + + if ( existingSetting !== -1 ) { + // Update the width + userColumnDefs[existingSetting].width = width; + } else { + + // Add the width for this column + userColumnDefs.push({ + targets: targets, + width: width, + }); + + } +}); + +// Save (or update) the settings in localStorage +localStorage.setItem(table_id, JSON.stringify(userColumnDefs)); +} \ No newline at end of file diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index a351256b..436eb639 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -154,6 +154,7 @@ + - + - @@ -402,90 +401,89 @@ //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters table_settings = { - 'drawCallback': function( settings ) { - $('#' + tableId + ' tr').off().on("click", function(event) { - if (event.target.type !== 'checkbox' && event.target.tagName.toLowerCase() !== 'a') { - var obj =$(this).find('input'); - obj.prop('checked', !obj.is(':checked')); - } - if ($(this).hasClass("selected") && event.target.tagName.toLowerCase() !== 'a'){ - $(this).removeClass("selected") - } else if (event.target.tagName.toLowerCase() !== 'a') { - $(this).addClass("selected") - } - change_buttons() - }); - }, - 'createdRow': function ( row, data, index ) { - $('td', row).eq(0).attr("style", "text-align: center; padding: 0px 10px 2px 10px;"); - $('td', row).eq(1).attr("align", "right"); - $('td', row).eq(1).attr('data-export', index+1); - $('td', row).eq(2).attr('data-export', $('td', row).eq(2).text()); - {% if dataset.type == 'ProbeSet' %} - $('td', row).eq(3).attr('title', $('td', row).eq(3).text()); - $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); - if ($('td', row).eq(3).text().length > 20) { - $('td', row).eq(3).text($('td', row).eq(3).text().substring(0, 20)); - $('td', row).eq(3).text($('td', row).eq(3).text() + '...') - } - $('td', row).eq(4).attr('title', $('td', row).eq(4).text()); - $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text()); - $('td', row).slice(5,10).attr("align", "right"); - $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text()); - $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text()); - $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text()); - $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text()); - $('td', row).eq(9).attr('data-export', $('td', row).eq(9).text()); - {% elif dataset.type == 'Publish' %} - $('td', row).eq(3).attr('title', $('td', row).eq(3).text()); - $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); - $('td', row).eq(4).attr('title', $('td', row).eq(4).text()); - $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text()); - $('td', row).eq(4).attr('align', 'right'); - $('td', row).slice(6,10).attr("align", "right"); - $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text()); - $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text()); - $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text()); - $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text()); - $('td', row).eq(9).attr('data-export', $('td', row).eq(8).text()); - {% elif dataset.type == 'Geno' %} - $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); - {% endif %} - }, - "data": trait_list, - "columns": columnDefs, - "order": [[1, "asc" ]], - "sDom": "iti", - "destroy": true, - {% if wide_columns_exist != true %} - "autoWidth": true, - {% else %} - "autoWidth": false, - {% endif %} - "deferRender": true, - "bSortClasses": false, - "scrollX": true, - {% if trait_list|length > 20 %} - "scrollY": "500px", - "scroller": true, - "scrollCollapse": true, - {% else %} - "iDisplayLength": -1, - {% endif %} - "initComplete": function (settings) { - //Add JQueryUI resizable functionality to each th in the ScrollHead table - $('#' + tableId + '_wrapper .dataTables_scrollHead thead th').resizable({ - handles: "e", - alsoResize: '#' + tableId + '_wrapper .dataTables_scrollHead table', //Not essential but makes the resizing smoother - resize: function( event, ui ) { - width_change = ui.size.width - ui.originalSize.width; - }, - stop: function () { - saveColumnSettings(tableId, trait_table); - loadDataTable(); - } - }); - }, + 'drawCallback': function( settings ) { + $('#' + tableId + ' tr').off().on("click", function(event) { + if (event.target.type !== 'checkbox' && event.target.tagName.toLowerCase() !== 'a') { + var obj =$(this).find('input'); + obj.prop('checked', !obj.is(':checked')); + } + if ($(this).hasClass("selected") && event.target.tagName.toLowerCase() !== 'a'){ + $(this).removeClass("selected") + } else if (event.target.tagName.toLowerCase() !== 'a') { + $(this).addClass("selected") + } + change_buttons() + }); + }, + 'createdRow': function ( row, data, index ) { + $('td', row).eq(0).attr("style", "text-align: center; padding: 0px 10px 2px 10px;"); + $('td', row).eq(1).attr("align", "right"); + $('td', row).eq(1).attr('data-export', index+1); + $('td', row).eq(2).attr('data-export', $('td', row).eq(2).text()); + {% if dataset.type == 'ProbeSet' %} + $('td', row).eq(3).attr('title', $('td', row).eq(3).text()); + $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); + if ($('td', row).eq(3).text().length > 20) { + $('td', row).eq(3).text($('td', row).eq(3).text().substring(0, 20)); + $('td', row).eq(3).text($('td', row).eq(3).text() + '...') + } + $('td', row).eq(4).attr('title', $('td', row).eq(4).text()); + $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text()); + $('td', row).slice(5,10).attr("align", "right"); + $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text()); + $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text()); + $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text()); + $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text()); + $('td', row).eq(9).attr('data-export', $('td', row).eq(9).text()); + {% elif dataset.type == 'Publish' %} + $('td', row).eq(3).attr('title', $('td', row).eq(3).text()); + $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); + $('td', row).eq(4).attr('title', $('td', row).eq(4).text()); + $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text()); + $('td', row).eq(4).attr('align', 'right'); + $('td', row).slice(6,10).attr("align", "right"); + $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text()); + $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text()); + $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text()); + $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text()); + $('td', row).eq(9).attr('data-export', $('td', row).eq(8).text()); + {% elif dataset.type == 'Geno' %} + $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); + {% endif %} + }, + "data": trait_list, + "columns": columnDefs, + "order": [[1, "asc" ]], + "sDom": "iti", + "destroy": true, + {% if wide_columns_exist != true %} + "autoWidth": true, + {% else %} + "autoWidth": false, + {% endif %} + "deferRender": true, + "bSortClasses": false, + {% if trait_list|length > 20 %} + "scrollY": "500px", + "scroller": true, + "scrollCollapse": true, + {% else %} + "iDisplayLength": -1, + {% endif %} + "initComplete": function (settings) { + //Add JQueryUI resizable functionality to each th in the ScrollHead table + $('#' + tableId + '_wrapper .dataTables_scrollHead thead th').resizable({ + handles: "e", + alsoResize: '#' + tableId + '_wrapper .dataTables_scrollHead table', //Not essential but makes the resizing smoother + resize: function( event, ui ) { + width_change = ui.size.width - ui.originalSize.width; + }, + stop: function () { + saveColumnSettings(tableId, trait_table); + loadDataTable(); + } + }); + } } if (!first_run){ -- cgit v1.2.3 From ef49a5bf5b1b2b010c71f77394c5b178c00b8942 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 21 Jun 2021 20:29:26 +0000 Subject: Fixed indentation in table_functions.js --- .../static/new/javascript/table_functions.js | 181 ++++++++++----------- 1 file changed, 88 insertions(+), 93 deletions(-) diff --git a/wqflask/wqflask/static/new/javascript/table_functions.js b/wqflask/wqflask/static/new/javascript/table_functions.js index e24b3e03..745563c2 100644 --- a/wqflask/wqflask/static/new/javascript/table_functions.js +++ b/wqflask/wqflask/static/new/javascript/table_functions.js @@ -1,93 +1,88 @@ -recheck_rows = function(the_table, checked_rows){ - //ZS: This is meant to recheck checkboxes after columns are resized - check_cells = the_table.column(0).nodes().to$(); - for (let i = 0; i < check_cells.length; i++) { - if (checked_rows.includes(i)){ - check_cells[i].childNodes[0].checked = true; - } - } - - check_rows = trait_table.rows().nodes(); - for (let i =0; i < check_rows.length; i++) { - if (checked_rows.includes(i)){ - check_rows[i].classList.add("selected") - } - } - } - -get_checked_rows = function(table_id){ - let checked_rows = [] - $("#" + table_id + " input").each(function(index){ - if ($(this).prop("checked") == true){ - checked_rows.push(index); - } - }); - - return checked_rows -} - -function setUserColumnsDefWidths(table_id) { - var userColumnDef; - - // Get the settings for this table from localStorage - var userColumnDefs = JSON.parse(localStorage.getItem(table_id)) || []; - - if (userColumnDefs.length === 0 ) return; - - columnDefs.forEach( function(columnDef) { - - // Check if there is a width specified for this column - userColumnDef = userColumnDefs.find( function(column) { - return column.targets === columnDef.targets; - }); - - // If there is, set the width of this columnDef in px - if ( userColumnDef ) { - - columnDef.sWidth = userColumnDef.width + 'px'; - columnDef.width = userColumnDef.width + 'px'; - - $('.toggle-vis').each(function(){ - if ($(this).attr('data-column') == columnDef.targets){ - if ($(this).hasClass("active")){ - columnDef.bVisible = false - } else { - columnDef.bVisible = true - } - } - }) - } - }); -} - -function saveColumnSettings(table_id, trait_table) { -var userColumnDefs = JSON.parse(localStorage.getItem(table_id)) || []; - -var width, header, existingSetting; - -trait_table.columns().every( function ( targets ) { - - // Check if there is a setting for this column in localStorage - existingSetting = userColumnDefs.findIndex( function(column) { return column.targets === targets;}); - - // Get the width of this column - header = this.header(); - width = $(header).width(); - - if ( existingSetting !== -1 ) { - // Update the width - userColumnDefs[existingSetting].width = width; - } else { - - // Add the width for this column - userColumnDefs.push({ - targets: targets, - width: width, - }); - - } -}); - -// Save (or update) the settings in localStorage -localStorage.setItem(table_id, JSON.stringify(userColumnDefs)); -} \ No newline at end of file +recheck_rows = function(the_table, checked_rows){ + //ZS: This is meant to recheck checkboxes after columns are resized + check_cells = the_table.column(0).nodes().to$(); + for (let i = 0; i < check_cells.length; i++) { + if (checked_rows.includes(i)){ + check_cells[i].childNodes[0].checked = true; + } + } + + check_rows = trait_table.rows().nodes(); + for (let i =0; i < check_rows.length; i++) { + if (checked_rows.includes(i)){ + check_rows[i].classList.add("selected") + } + } +} + +get_checked_rows = function(table_id){ + let checked_rows = [] + $("#" + table_id + " input").each(function(index){ + if ($(this).prop("checked") == true){ + checked_rows.push(index); + } + }); + + return checked_rows +} + +function setUserColumnsDefWidths(table_id) { + var userColumnDef; + + // Get the settings for this table from localStorage + var userColumnDefs = JSON.parse(localStorage.getItem(table_id)) || []; + + if (userColumnDefs.length === 0 ) return; + + columnDefs.forEach( function(columnDef) { + // Check if there is a width specified for this column + userColumnDef = userColumnDefs.find( function(column) { + return column.targets === columnDef.targets; + }); + + // If there is, set the width of this columnDef in px + if ( userColumnDef ) { + + columnDef.sWidth = userColumnDef.width + 'px'; + columnDef.width = userColumnDef.width + 'px'; + + $('.toggle-vis').each(function(){ + if ($(this).attr('data-column') == columnDef.targets){ + if ($(this).hasClass("active")){ + columnDef.bVisible = false + } else { + columnDef.bVisible = true + } + } + }) + } + }); +} + +function saveColumnSettings(table_id, trait_table) { + var userColumnDefs = JSON.parse(localStorage.getItem(table_id)) || []; + var width, header, existingSetting; + + trait_table.columns().every( function ( targets ) { + // Check if there is a setting for this column in localStorage + existingSetting = userColumnDefs.findIndex( function(column) { return column.targets === targets;}); + + // Get the width of this column + header = this.header(); + width = $(header).width(); + + if ( existingSetting !== -1 ) { + // Update the width + userColumnDefs[existingSetting].width = width; + } else { + // Add the width for this column + userColumnDefs.push({ + targets: targets, + width: width, + }); + } + }); + + // Save (or update) the settings in localStorage + localStorage.setItem(table_id, JSON.stringify(userColumnDefs)); +} -- cgit v1.2.3 From 6f621612f1f74694b9895c7252c1d72e7b2f7314 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 21 Jun 2021 20:33:11 +0000 Subject: Implemented resizeable columns for gene global search + fixed the way change_buttons is called and row highlighting works in the DataTables callback --- wqflask/wqflask/templates/gsearch_gene.html | 347 +++++++++++++++------------- 1 file changed, 183 insertions(+), 164 deletions(-) diff --git a/wqflask/wqflask/templates/gsearch_gene.html b/wqflask/wqflask/templates/gsearch_gene.html index 5549ac8a..3b8f9ff7 100644 --- a/wqflask/wqflask/templates/gsearch_gene.html +++ b/wqflask/wqflask/templates/gsearch_gene.html @@ -2,6 +2,7 @@ {% block title %}Search Results{% endblock %} {% block css %} + {% endblock %} {% block content %} @@ -31,7 +32,7 @@

-
+

Loading...
@@ -54,6 +55,7 @@ + - {% endblock %} -- cgit v1.2.3 From 48154b81c705f6eb2af16b0394e1e2e5bcca83b6 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 21 Jun 2021 20:35:07 +0000 Subject: Added table_functions.js to gsearch_pheno.html --- wqflask/wqflask/templates/gsearch_pheno.html | 1 + 1 file changed, 1 insertion(+) diff --git a/wqflask/wqflask/templates/gsearch_pheno.html b/wqflask/wqflask/templates/gsearch_pheno.html index 89316cbc..a5e30c8b 100644 --- a/wqflask/wqflask/templates/gsearch_pheno.html +++ b/wqflask/wqflask/templates/gsearch_pheno.html @@ -54,6 +54,7 @@ + - {% endblock %} -- cgit v1.2.3 From d99776e16fc991f75803836f6e3292cea9f6d621 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 24 Jun 2021 21:11:59 +0000 Subject: Gave initial widths for dataset and description columns in gene global search --- wqflask/wqflask/templates/gsearch_gene.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/gsearch_gene.html b/wqflask/wqflask/templates/gsearch_gene.html index 3b8f9ff7..0a3ec027 100644 --- a/wqflask/wqflask/templates/gsearch_gene.html +++ b/wqflask/wqflask/templates/gsearch_gene.html @@ -132,6 +132,7 @@ 'title': "Dataset", 'type': "natural", 'targets': 6, + 'width': "320px", 'data': "dataset_fullname" }, { @@ -145,6 +146,7 @@ 'title': "Description", 'type': "natural", 'data': null, + 'width': "120px", 'targets': 8, 'render': function(data, type, row, meta) { try { @@ -260,7 +262,6 @@ "order": [[1, "asc" ]], 'sDom': "iti", "destroy": true, - "autoWidth": true, "deferRender": true, "bSortClasses": false, {% if trait_count > 20 %} @@ -301,9 +302,8 @@ {% if trait_list|length > 20 %} $('#table_container').css("width", String($('#trait_table').width() + 17) + "px"); {% endif %} + trait_table.draw(); } - - trait_table.draw(); } $('#redraw').click(function() { -- cgit v1.2.3 From a2919c924f647887f6383d1e2efebc914ade0da3 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 30 Jun 2021 20:24:52 +0000 Subject: Removed CSS that truncates cell content and adds an ellipses when the cell is too small to contain content; it now just makes the content wrap --- wqflask/wqflask/static/new/css/trait_list.css | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/wqflask/wqflask/static/new/css/trait_list.css b/wqflask/wqflask/static/new/css/trait_list.css index 6d49f009..ce3075d4 100644 --- a/wqflask/wqflask/static/new/css/trait_list.css +++ b/wqflask/wqflask/static/new/css/trait_list.css @@ -51,22 +51,3 @@ div.dts div.dataTables_scrollBody table { div.dts div.dataTables_paginate,div.dts div.dataTables_length{ display:none } - -/* This is the important CSS */ -table.dataTable { - table-layout: fixed; - width: 100%; - white-space: nowrap; - overflow: hidden; -} - -table.dataTable th { - white-space: nowrap; - overflow: hidden; -} - -table.dataTable td { - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; -} \ No newline at end of file -- cgit v1.2.3 From c3302c1f44af146ab296110c11e68ab25ddae4df Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 1 Jul 2021 17:50:31 +0000 Subject: Changed initialize_show_trait_tables.js to include resizeable columns and deal with both primary/other tables in a single function --- .../new/javascript/initialize_show_trait_tables.js | 214 +++++++++++++-------- 1 file changed, 136 insertions(+), 78 deletions(-) diff --git a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js index 6ca92fb6..d8a1b006 100644 --- a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js +++ b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js @@ -15,6 +15,7 @@ build_columns = function() { 'data': null, 'orderDataType': "dom-checkbox", 'searchable' : false, + 'targets': 0, 'render': function(data, type, row, meta) { return '' } @@ -23,12 +24,14 @@ build_columns = function() { 'title': "ID", 'type': "natural", 'searchable' : false, + 'targets': 1, 'data': "this_id" }, { 'title': "Sample", 'type': "natural", 'data': null, + 'targets': 2, 'render': function(data, type, row, meta) { return '' + data.name + '' } @@ -38,6 +41,7 @@ build_columns = function() { 'orderDataType': "dom-input", 'type': "cust-txt", 'data': null, + 'targets': 3, 'render': function(data, type, row, meta) { if (data.value == null) { return '' @@ -48,12 +52,15 @@ build_columns = function() { } ]; + attr_start = 4 if (js_data.se_exists) { + attr_start += 2 column_list.push( { 'bSortable': false, 'type': "natural", 'data': null, + 'targets': 4, 'searchable' : false, 'render': function(data, type, row, meta) { return '±' @@ -64,6 +71,7 @@ build_columns = function() { 'orderDataType': "dom-input", 'type': "cust-txt", 'data': null, + 'targets': 5, 'render': function(data, type, row, meta) { if (data.variance == null) { return '' @@ -73,24 +81,47 @@ build_columns = function() { } } ); - } - if (js_data.has_num_cases === true) { - column_list.push( - { - 'title': "
N
", - 'orderDataType': "dom-input", - 'type': "cust-txt", - 'data': null, - 'render': function(data, type, row, meta) { - if (data.num_cases == null || data.num_cases == undefined) { - return '' - } else { - return '' + if (js_data.has_num_cases === true) { + attr_start += 1 + column_list.push( + { + 'title': "
N
", + 'orderDataType': "dom-input", + 'type': "cust-txt", + 'data': null, + 'targets': 6, + 'render': function(data, type, row, meta) { + if (data.num_cases == null || data.num_cases == undefined) { + return '' + } else { + return '' + } } } - } - ); + ); + } + } + else { + if (js_data.has_num_cases === true) { + attr_start += 1 + column_list.push( + { + 'title': "
N
", + 'orderDataType': "dom-input", + 'type': "cust-txt", + 'data': null, + 'targets': 4, + 'render': function(data, type, row, meta) { + if (data.num_cases == null || data.num_cases == undefined) { + return '' + } else { + return '' + } + } + } + ); + } } attr_keys = Object.keys(js_data.attributes).sort((a, b) => (js_data.attributes[a].name.toLowerCase() > js_data.attributes[b].name.toLowerCase()) ? 1 : -1) @@ -100,6 +131,7 @@ build_columns = function() { 'title': "
" + js_data.attributes[attr_keys[i]].name + "
", 'type': "natural", 'data': null, + 'targets': attr_start + i, 'render': function(data, type, row, meta) { attr_name = Object.keys(data.extra_attributes).sort()[meta.col - data.first_attr_col] @@ -119,14 +151,35 @@ build_columns = function() { return column_list } -var primary_table = $('#samples_primary').DataTable( { +columnDefs = build_columns() + +loadDataTable(first_run=true, table_id="samples_primary", table_data=js_data['sample_lists'][0]) +if (js_data.sample_lists.length > 1){ + loadDataTable(first_run=true, table_id="samples_other", table_data=js_data['sample_lists'][1]) +} + +function loadDataTable(first_run=false, table_id, table_data){ + + console.log("COL DEFS:", columnDefs) + + if (!first_run){ + setUserColumnsDefWidths(table_id); + } + + if (table_id == "samples_primary"){ + table_type = "Primary" + } else { + table_type = "Other" + } + + table_settings = { 'initComplete': function(settings, json) { $('.edit_sample_value').change(function() { edit_data_change(); }); }, 'createdRow': function ( row, data, index ) { - $(row).attr('id', "Primary_" + data.this_id) + $(row).attr('id', table_type + "_" + data.this_id) $(row).addClass("value_se"); if (data.outlier) { $(row).addClass("outlier"); @@ -154,75 +207,80 @@ var primary_table = $('#samples_primary').DataTable( { $('td', row).eq(attribute_start_pos + i + 1).attr("style", "text-align: " + js_data.attributes[attr_keys[i]].alignment + "; padding-top: 2px; padding-bottom: 0px;") } }, - 'data': js_data['sample_lists'][0], - 'columns': build_columns(), - 'order': [[1, "asc"]], - 'sDom': "Ztr", - 'autoWidth': true, - 'orderClasses': true, + 'data': table_data, + 'columns': columnDefs, + "order": [[1, "asc" ]], + "sDom": "iti", + "destroy": true, + "autoWidth": first_run, + "deferRender": true, + "bSortClasses": false, "scrollY": "100vh", - 'scroller': true, - 'scrollCollapse': true -} ); + "scrollCollapse": true, + "scroller": true, + "iDisplayLength": -1, + "initComplete": function (settings) { + //Add JQueryUI resizable functionality to each th in the ScrollHead table + $('#' + table_id + '_wrapper .dataTables_scrollHead thead th').resizable({ + handles: "e", + alsoResize: '#' + table_id + '_wrapper .dataTables_scrollHead table', //Not essential but makes the resizing smoother + resize: function( event, ui ) { + width_change = ui.size.width - ui.originalSize.width; + }, + stop: function () { + saveColumnSettings(table_id, the_table); + loadDataTable(first_run=false, table_id, table_data); + } + }); + } + } -primary_table.draw(); //ZS: This makes the table adjust its height properly on initial load + var the_table = $('#' + table_id).DataTable(table_settings); -primary_table.on( 'order.dt search.dt draw.dt', function () { - primary_table.column(1, {search:'applied', order:'applied'}).nodes().each( function (cell, i) { - cell.innerHTML = i+1; - } ); -} ).draw(); + the_table.draw(); //ZS: This makes the table adjust its height properly on initial load -$('#primary_searchbox').on( 'keyup', function () { - primary_table.search($(this).val()).draw(); -} ); + the_table.on( 'order.dt search.dt draw.dt', function () { + the_table.column(1, {search:'applied', order:'applied'}).nodes().each( function (cell, i) { + cell.innerHTML = i+1; + } ); + } ).draw(); -if (js_data.sample_lists.length > 1){ - var other_table = $('#samples_other').DataTable( { - 'initComplete': function(settings, json) { - $('.edit_sample_value').change(function() { - edit_data_change(); - }); - }, - 'createdRow': function ( row, data, index ) { - $(row).attr('id', "Primary_" + data.this_id) - $(row).addClass("value_se"); - if (data.outlier) { - $(row).addClass("outlier"); - } - $('td', row).eq(1).addClass("column_name-Index") - $('td', row).eq(2).addClass("column_name-Sample") - $('td', row).eq(3).addClass("column_name-Value") - if (js_data.se_exists) { - $('td', row).eq(5).addClass("column_name-SE") - if (js_data.has_num_cases === true) { - $('td', row).eq(6).addClass("column_name-num_cases") - } else { - if (js_data.has_num_cases === true) { - $('td', row).eq(4).addClass("column_name-num_cases") - } - } + if (!first_run){ + $('#' + table_type.toLowerCase() + '_container').css("width", String($('#' + table_id).width() + width_change + 17) + "px"); //ZS : Change the container width by the change in width of the adjusted column, so the overall table size adjusts properly + + let checked_rows = get_checked_rows(table_id); + if (checked_rows.length > 0){ + recheck_rows(the_table, checked_rows); + } + } + + if (first_run){ + $('#' + table_type.toLowerCase() + '_container').css("width", String($('#' + table_id).width() + 17) + "px"); + } + + $('#' + table_type.toLowerCase() + '_searchbox').on( 'keyup', function () { + the_table.search($(this).val()).draw(); + } ); + + $('.toggle-vis').on('click', function (e) { + e.preventDefault(); + + function toggle_column(column) { + //ZS: Toggle column visibility + column.visible( ! column.visible() ); + if (column.visible()){ + $(this).removeClass("active"); } else { - if (js_data.has_num_cases === true) { - $('td', row).eq(4).addClass("column_name-num_cases") - } + $(this).addClass("active"); } + } - for (i=0; i < attr_keys.length; i++) { - $('td', row).eq(attribute_start_pos + i + 1).addClass("column_name-" + js_data.attributes[attr_keys[i]].name) - $('td', row).eq(attribute_start_pos + i + 1).attr("style", "text-align: " + js_data.attributes[attr_keys[i]].alignment + "; padding-top: 2px; padding-bottom: 0px;") - } - }, - 'data': js_data['sample_lists'][1], - 'columns': build_columns(), - 'order': [[1, "asc"]], - 'sDom': "Ztr", - 'autoWidth': true, - 'orderClasses': true, - "scrollY": "100vh", - 'scroller': true, - 'scrollCollapse': true + // Get the column API object + var target_cols = $(this).attr('data-column').split(",") + for (let i = 0; i < target_cols.length; i++){ + var column = the_table.column( target_cols[i] ); + toggle_column(column); + } } ); - other_table.draw(); //ZS: This makes the table adjust its height properly on initial load } -- cgit v1.2.3 From be51132f68ea92c3917c16f1d2962772e8debe04 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 1 Jul 2021 17:51:32 +0000 Subject: Removed width from sample_group div and change table div ID to be distinct when there are both primary and other samplee tables --- wqflask/wqflask/templates/show_trait_edit_data.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/templates/show_trait_edit_data.html b/wqflask/wqflask/templates/show_trait_edit_data.html index 5939c953..c5944d3f 100644 --- a/wqflask/wqflask/templates/show_trait_edit_data.html +++ b/wqflask/wqflask/templates/show_trait_edit_data.html @@ -53,11 +53,11 @@ {% set outer_loop = loop %} -
+

{{ sample_type.header }}

-
+

Loading...
-- cgit v1.2.3 From bbda9eb2eb542b7d90abdfcd1a8f34c35d38491c Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 1 Jul 2021 17:53:05 +0000 Subject: Added imports necessary for resizeable columns and removed unused code --- wqflask/wqflask/templates/show_trait.html | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html index 3dbf5f57..7074e21e 100644 --- a/wqflask/wqflask/templates/show_trait.html +++ b/wqflask/wqflask/templates/show_trait.html @@ -5,9 +5,10 @@ - + + @@ -155,6 +156,7 @@ + -- cgit v1.2.3 From 614f641624582754e29b84d632e311ed5f186c1e Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 20 Sep 2021 09:06:28 +0300 Subject: Move "Clustered Heatmap" button to separate form Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/clustering.gmi * Move the button out of the "export_form" into a new "heatmaps_form" to avoid some weird JS interaction that showed up. --- wqflask/wqflask/templates/collections/view.html | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index 783458fc..06fd80f4 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -50,13 +50,16 @@ - + + + +
Show/Hide Columns: -- cgit v1.2.3 From cfe0de277021c41eedeb65ec7f1560ac6d67ad0a Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 20 Sep 2021 09:15:10 +0300 Subject: Fix id used. --- wqflask/wqflask/templates/collections/view.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index 06fd80f4..0578460d 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -257,7 +257,7 @@ make_default(); }); - $("#clustered-heatmaps").on("click", function() { + $("#clustered-heatmap").on("click", function() { heatmap_url = $(this).attr("data-url") console.log("heatmap url:", heatmap_url) traits = $(".trait_checkbox:checked").map(function() { -- cgit v1.2.3 From f1876d4d8da5c973375fc398fedaa12825a0b780 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 20 Sep 2021 09:20:44 +0300 Subject: Prevent the default submit action Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/clustering.gmi * Prevent the default submit action. --- wqflask/wqflask/templates/collections/view.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index 0578460d..51b96e10 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -257,6 +257,10 @@ make_default(); }); + $("#heatmaps_form").submit(function(e) { + e.preventDefault(); + }); + $("#clustered-heatmap").on("click", function() { heatmap_url = $(this).attr("data-url") console.log("heatmap url:", heatmap_url) -- cgit v1.2.3 From 98f9027b8ce8f33dda7f0b1b5495b22b4a450349 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Wed, 22 Sep 2021 06:30:42 +0300 Subject: Test heatmap creation from serialized figure Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/clustering.gmi * Attempt using the figure, serialized as JSON, to display the clustered heatmap. --- wqflask/wqflask/templates/collections/view.html | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index 51b96e10..bca629a9 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -128,6 +128,9 @@

Loading...
+
+

@@ -148,6 +151,8 @@ + + +{% endblock %} + -- cgit v1.2.3 From 167ec0df8d8d487832e6a0acaee3eac8963d9804 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Fri, 1 Oct 2021 05:34:20 +0300 Subject: add xterm web terminal --- wqflask/wqflask/templates/test_wgcna_results.html | 225 +++++++++++----------- 1 file changed, 116 insertions(+), 109 deletions(-) diff --git a/wqflask/wqflask/templates/test_wgcna_results.html b/wqflask/wqflask/templates/test_wgcna_results.html index 37ea2aa0..f95766ad 100644 --- a/wqflask/wqflask/templates/test_wgcna_results.html +++ b/wqflask/wqflask/templates/test_wgcna_results.html @@ -1,146 +1,153 @@ {% extends "base.html" %} {% block title %}WCGNA results{% endblock %} +{% block content %} + -{% block content %} - -
+.control_net_colors { + display: flex; + flex-wrap: wrap; + justify-content: space-between; + align-items: center; + text-align: center; +} -
-
- {% for key, value in results["data"]["output"]["soft_threshold"].items()%} -
-

{{key}}

- {% for val in value %} -

{{val|round(3)}}

- - {% endfor %} - -
- {% endfor %} +.control_mod_eigens { + display: grid; + grid-template-columns: repeat(2, 200px); +} -
+#terminal { -
-

Net colors

-
- {% for key,value in results["data"]["output"]["net_colors"].items() %} -
-

{{key}}

-

{{value}}

-
- - {% endfor %} + max-width: 768px; -
- + margin: 10px; -
- - -
-

Module eigen genes

- -
- {% for strain in results["data"]["input"]["sample_names"]%} - {{strain}} - {% endfor %} - {% for mod,values in results["data"]["output"]["ModEigens"].items() %} - {{mod}} {{values}} - - {% endfor %} - -
- -
- - - dsffsdf - +} + +
+
+
+
+
+
+
+ {% for key, value in results["data"]["output"]["soft_threshold"].items()%} +
+

{{key}}

+ {% for val in value %} +

{{val|round(3)}}

+ {% endfor %} +
+ {% endfor %} +
+
+

Net colors

+
+ {% for key,value in results["data"]["output"]["net_colors"].items() %} +
+

{{key}}

+

{{value}}

+
+ {% endfor %} +
+
+
+

Module eigen genes

+
+ {% for strain in results["data"]["input"]["sample_names"]%} + {{strain}} + {% endfor %} + {% for mod,values in results["data"]["output"]["ModEigens"].items() %} + {{mod}} {{values}} + {% endfor %} +
+ + +
+
+
+ + -{% endblock %} + writeToTerminal({ cursorBlink: true, lineHeight: 1.2 }, "terminal")(terminal_output) + +{% endblock %} \ No newline at end of file -- cgit v1.2.3 From 18b50f56d614021d5727d546f3d6e360575e4468 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 5 Oct 2021 19:07:24 +0300 Subject: work on wgcna_setup form --- wqflask/wqflask/templates/wgcna_setup.html | 54 ++++++++++++++++-------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/wqflask/wqflask/templates/wgcna_setup.html b/wqflask/wqflask/templates/wgcna_setup.html index c5461497..44932389 100644 --- a/wqflask/wqflask/templates/wgcna_setup.html +++ b/wqflask/wqflask/templates/wgcna_setup.html @@ -2,8 +2,10 @@ {% block title %}WCGNA analysis{% endblock %} {% block content %} -

WGCNA analysis parameters

+
+
+

WGCNA analysis parameters

{% if request.form['trait_list'].split(",")|length < 4 %} {% else %} -
- -
- -
- -
+ + +
+ +
+
-
- -
- -
+
+
+ +
+
-
- -
- -
+
+ +
+ +
+
-
- -
- -
+
+
+ +
+
+
-
+
- + {% endif %}
+
{% endblock %} -- cgit v1.2.3 From 31071d78f2ffb0ae4c7e0b99e74c7729a1a36e9c Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 5 Oct 2021 19:49:50 +0300 Subject: move xterm code to setup --- wqflask/wqflask/templates/wgcna_setup.html | 140 +++++++++++++++++++---------- 1 file changed, 94 insertions(+), 46 deletions(-) diff --git a/wqflask/wqflask/templates/wgcna_setup.html b/wqflask/wqflask/templates/wgcna_setup.html index 44932389..4b13e54e 100644 --- a/wqflask/wqflask/templates/wgcna_setup.html +++ b/wqflask/wqflask/templates/wgcna_setup.html @@ -1,53 +1,101 @@ {% extends "base.html" %} {% block title %}WCGNA analysis{% endblock %} - -{% block content %} - +{% block content %} + + + + +
-
-

WGCNA analysis parameters

- {% if request.form['trait_list'].split(",")|length < 4 %} - - {% else %} -
- -
- -
- -
-
-
- -
- -
-
- -
- -
- -
-
-
- -
- +
+

WGCNA analysis parameters

+ {% if request.form['trait_list'].split(",")|length < 4 %} -
-
-
- -
+ {% else %} + + +
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+
+ +
+
+ + {% endif %} +
+
+
+
+
- - {% endif %}
-{% endblock %} + +{% endblock %} \ No newline at end of file -- cgit v1.2.3 From 6f739fccbb7cff9f05b53ff4775ecee0761c293b Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 5 Oct 2021 19:50:37 +0300 Subject: js formatting --- wqflask/wqflask/templates/wgcna_setup.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/wgcna_setup.html b/wqflask/wqflask/templates/wgcna_setup.html index 4b13e54e..34690f29 100644 --- a/wqflask/wqflask/templates/wgcna_setup.html +++ b/wqflask/wqflask/templates/wgcna_setup.html @@ -84,11 +84,11 @@ document.addEventListener('DOMContentLoaded', function() { // open socket connection - const socket = io(`${GN_SERVER_URL}`) + const socket = io(`${GN_SERVER_URL}`) // add namespace - socket.on("output",({data})=>{ + socket.on("output", ({ data }) => { - term.writeln(data) + term.writeln(data) }) } else { -- cgit v1.2.3 From 37c37996424e826187ac3eca9ed4cd11b9715736 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 5 Oct 2021 19:52:51 +0300 Subject: code format for wgcna results file --- wqflask/wqflask/templates/test_wgcna_results.html | 61 +++++++++-------------- 1 file changed, 23 insertions(+), 38 deletions(-) diff --git a/wqflask/wqflask/templates/test_wgcna_results.html b/wqflask/wqflask/templates/test_wgcna_results.html index f95766ad..9484595f 100644 --- a/wqflask/wqflask/templates/test_wgcna_results.html +++ b/wqflask/wqflask/templates/test_wgcna_results.html @@ -2,13 +2,17 @@ {% block title %}WCGNA results{% endblock %} {% block content %} + + +
@@ -122,32 +118,21 @@ let terminal_output = results.output let { output } = results.data let sft = output.soft_threshold - - + -{% endblock %} \ No newline at end of file +{% endblock %} +** \ No newline at end of file -- cgit v1.2.3 From 6b4b7a5774f42ff6be1fb5fde6ba1fef632cd1a4 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 6 Oct 2021 11:48:50 +0300 Subject: add datatable for eigengenes --- wqflask/wqflask/templates/test_wgcna_results.html | 80 ++++++++++++++++------- 1 file changed, 57 insertions(+), 23 deletions(-) diff --git a/wqflask/wqflask/templates/test_wgcna_results.html b/wqflask/wqflask/templates/test_wgcna_results.html index 9484595f..ac82647d 100644 --- a/wqflask/wqflask/templates/test_wgcna_results.html +++ b/wqflask/wqflask/templates/test_wgcna_results.html @@ -2,8 +2,17 @@ {% block title %}WCGNA results{% endblock %} {% block content %} + - + + + + + + + + +
@@ -104,11 +113,12 @@
-
- - CLuster dendogram - + {% if image["image_generated"] %} +
+
+ + {% endif %}
@@ -161,12 +171,5 @@ $(document).ready(function(){ } ); }) - - - - - - - {% endblock %} \ No newline at end of file -- cgit v1.2.3 From 6578f6fa44dfa54bee29a16347bf3265ec6d76ad Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sat, 9 Oct 2021 21:55:08 +0300 Subject: add function to process gn3 wgcna output --- wqflask/wqflask/wgcna/gn3_wgcna.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 wqflask/wqflask/wgcna/gn3_wgcna.py diff --git a/wqflask/wqflask/wgcna/gn3_wgcna.py b/wqflask/wqflask/wgcna/gn3_wgcna.py new file mode 100644 index 00000000..2657a099 --- /dev/null +++ b/wqflask/wqflask/wgcna/gn3_wgcna.py @@ -0,0 +1,27 @@ +"""module contains code to consume gn3-wgcna api +and process data to be rendered by datatables +""" + + + +def process_wgcna_data(response): + """function for processing modeigene genes + for create row data for datataba""" + mod_eigens = response["output"]["ModEigens"] + + sample_names = response["input"]["sample_names"] + + mod_dataset = [[sample] for sample in sample_names] + + for _, mod_values in mod_eigens.items(): + for (index, _sample) in enumerate(sample_names): + mod_dataset[index].append(round(mod_values[index], 3)) + + return { + "col_names": ["sample_names", *mod_eigens.keys()], + "mod_dataset": mod_dataset + } + + +def process_image(): + pass \ No newline at end of file -- cgit v1.2.3 From ef80c72194dd8a0b8868ece15589e0a3cf04516f Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sat, 9 Oct 2021 21:55:39 +0300 Subject: unittest for processing wgcna output --- wqflask/tests/unit/wqflask/wgcna/__init__.py | 0 wqflask/tests/unit/wqflask/wgcna/test_wgcna.py | 50 ++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 wqflask/tests/unit/wqflask/wgcna/__init__.py create mode 100644 wqflask/tests/unit/wqflask/wgcna/test_wgcna.py diff --git a/wqflask/tests/unit/wqflask/wgcna/__init__.py b/wqflask/tests/unit/wqflask/wgcna/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wqflask/tests/unit/wqflask/wgcna/test_wgcna.py b/wqflask/tests/unit/wqflask/wgcna/test_wgcna.py new file mode 100644 index 00000000..8e947e2f --- /dev/null +++ b/wqflask/tests/unit/wqflask/wgcna/test_wgcna.py @@ -0,0 +1,50 @@ + +"""module contains for processing gn3 wgcna data""" +from unittest import TestCase + +from wqflask.wgcna.gn3_wgcna import process_wgcna_data + + +class DataProcessingTests(TestCase): + """class contains data processing tests""" + + def test_data_processing(self): + """test for parsing data for datatable""" + output = { + "input": { + "sample_names": ["BXD1", "BXD2", "BXD3", "BXD4", "BXD5", "BXD6"], + + }, + "output": { + "ModEigens": { + "MEturquoise": [ + 0.0646677768085351, + 0.137200224277058, + 0.63451113720732, + -0.544002665501479, + -0.489487590361863, + 0.197111117570427 + ], + "MEgrey": [ + 0.213, + 0.214, + 0.3141, + -0.545, + -0.423, + 0.156, + ] + }}} + + row_data = [['BXD1', 0.065, 0.213], + ['BXD2', 0.137, 0.214], + ['BXD3', 0.635, 0.314], + ['BXD4', -0.544, -0.545], + ['BXD5', -0.489, -0.423], + ['BXD6', 0.197, 0.156]] + + expected_results = { + "col_names": ["sample_names", "MEturquoise", "MEgrey"], + "mod_dataset": row_data + } + + self.assertEqual(process_wgcna_data(output), expected_results) -- cgit v1.2.3 From 599ac567990a3881dc3821ad226a18ce538d1a17 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sat, 9 Oct 2021 21:58:36 +0300 Subject: add function to process image data --- wqflask/wqflask/wgcna/gn3_wgcna.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/wgcna/gn3_wgcna.py b/wqflask/wqflask/wgcna/gn3_wgcna.py index 2657a099..225bef22 100644 --- a/wqflask/wqflask/wgcna/gn3_wgcna.py +++ b/wqflask/wqflask/wgcna/gn3_wgcna.py @@ -3,7 +3,6 @@ and process data to be rendered by datatables """ - def process_wgcna_data(response): """function for processing modeigene genes for create row data for datataba""" @@ -23,5 +22,12 @@ def process_wgcna_data(response): } -def process_image(): - pass \ No newline at end of file +def process_image(response): + """function to process image check if byte string is empty""" + image_data = response["output"]["image_data"] + return ({ + "image_generated": True, + "image_data": image_data + } if image_data else { + "image_generated": False + }) -- cgit v1.2.3 From 1b77d2417ba71ad7e2274429dd20c9272cb0f582 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sat, 9 Oct 2021 22:00:26 +0300 Subject: function to fetch trait data --- wqflask/wqflask/wgcna/gn3_wgcna.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/wqflask/wqflask/wgcna/gn3_wgcna.py b/wqflask/wqflask/wgcna/gn3_wgcna.py index 225bef22..f7ed4cef 100644 --- a/wqflask/wqflask/wgcna/gn3_wgcna.py +++ b/wqflask/wqflask/wgcna/gn3_wgcna.py @@ -3,6 +3,16 @@ and process data to be rendered by datatables """ +def fetch_trait_data(requestform): + """fetch trait data""" + db_obj = SimpleNamespace() + get_trait_db_obs(db_obj, + [trait.strip() + for trait in requestform['trait_list'].split(',')]) + + return process_dataset(db_obj.trait_list) + + def process_wgcna_data(response): """function for processing modeigene genes for create row data for datataba""" -- cgit v1.2.3 From c9ae69a30a972f47232f8457e9e1b8cd514f9832 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sat, 9 Oct 2021 22:04:17 +0300 Subject: add function to process trait sample data --- wqflask/wqflask/wgcna/gn3_wgcna.py | 44 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/wqflask/wqflask/wgcna/gn3_wgcna.py b/wqflask/wqflask/wgcna/gn3_wgcna.py index f7ed4cef..9ab6b3e0 100644 --- a/wqflask/wqflask/wgcna/gn3_wgcna.py +++ b/wqflask/wqflask/wgcna/gn3_wgcna.py @@ -1,6 +1,7 @@ """module contains code to consume gn3-wgcna api and process data to be rendered by datatables """ +from utility.helper_functions import get_trait_db_obs def fetch_trait_data(requestform): @@ -13,6 +14,49 @@ def fetch_trait_data(requestform): return process_dataset(db_obj.trait_list) +def process_dataset(trait_list): + """process datasets and strains""" + + input_data = {} + traits = [] + strains = [] + + # xtodo unique traits and strains + + for trait in trait_list: + traits.append(trait[0].name) + + input_data[trait[0].name] = {} + for strain in trait[0].data: + strains.append(strain) + input_data[trait[0].name][strain] = trait[0].data[strain].value + + return { + "wgcna_input": input_data + } + + def process_dataset(trait_list): + """process datasets and strains""" + + input_data = {} + traits = [] + strains = [] + + # xtodo unique traits and strains + + for trait in trait_list: + traits.append(trait[0].name) + + input_data[trait[0].name] = {} + for strain in trait[0].data: + strains.append(strain) + input_data[trait[0].name][strain] = trait[0].data[strain].value + + return { + "wgcna_input": input_data + } + + def process_wgcna_data(response): """function for processing modeigene genes for create row data for datataba""" -- cgit v1.2.3 From 585cd45c56ae4bc444336815cbde791b0c0d2e7b Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sat, 9 Oct 2021 22:11:50 +0300 Subject: add function to call wgcna api --- wqflask/wqflask/wgcna/gn3_wgcna.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/wqflask/wqflask/wgcna/gn3_wgcna.py b/wqflask/wqflask/wgcna/gn3_wgcna.py index 9ab6b3e0..520f3d04 100644 --- a/wqflask/wqflask/wgcna/gn3_wgcna.py +++ b/wqflask/wqflask/wgcna/gn3_wgcna.py @@ -1,6 +1,7 @@ """module contains code to consume gn3-wgcna api and process data to be rendered by datatables """ +from typing import SimpleNamespace from utility.helper_functions import get_trait_db_obs @@ -85,3 +86,33 @@ def process_image(response): } if image_data else { "image_generated": False }) + + +def run_wgcna(form_data): + """function to run wgcna""" + + wgcna_api = f"{GN3_URL}/api/wgcna/run_wgcna" + + # parse form data + + trait_dataset = fetch_trait_data(form_data) + + response = requests.post(wgcna_api, { + "socket_id": form_data.get("socket_id"), # streaming disabled + "sample_names": list(set(strains)), + "trait_names": form_traits, + "trait_sample_data": form_strains, + "TOMtype": form_data["TOMtype"], + "minModuleSize": int(form_data["MinModuleSize"]), + "corType": form_data["corType"] + + } + ).json() + + if response.status_code == 200: + return { + {"results": response, + "data": process_wgcna_data(response), + "image": process_image(response) + } + } -- cgit v1.2.3 From 9a17787cab82fe1c89dc68521eca9e6c8bb1dbb6 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sun, 10 Oct 2021 01:44:45 +0300 Subject: pep8 formatting fix for parsing response data --- wqflask/wqflask/wgcna/gn3_wgcna.py | 66 ++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 38 deletions(-) diff --git a/wqflask/wqflask/wgcna/gn3_wgcna.py b/wqflask/wqflask/wgcna/gn3_wgcna.py index 520f3d04..96510223 100644 --- a/wqflask/wqflask/wgcna/gn3_wgcna.py +++ b/wqflask/wqflask/wgcna/gn3_wgcna.py @@ -1,10 +1,18 @@ """module contains code to consume gn3-wgcna api and process data to be rendered by datatables """ -from typing import SimpleNamespace + +import requests +from types import SimpleNamespace from utility.helper_functions import get_trait_db_obs +def validate_form(requestform): + return { + "" + } + + def fetch_trait_data(requestform): """fetch trait data""" db_obj = SimpleNamespace() @@ -31,30 +39,14 @@ def process_dataset(trait_list): for strain in trait[0].data: strains.append(strain) input_data[trait[0].name][strain] = trait[0].data[strain].value + # "sample_names": list(set(strains)), + # "trait_names": form_traits, + # "trait_sample_data": form_strains, return { - "wgcna_input": input_data - } - - def process_dataset(trait_list): - """process datasets and strains""" - - input_data = {} - traits = [] - strains = [] - - # xtodo unique traits and strains - - for trait in trait_list: - traits.append(trait[0].name) - - input_data[trait[0].name] = {} - for strain in trait[0].data: - strains.append(strain) - input_data[trait[0].name][strain] = trait[0].data[strain].value - - return { - "wgcna_input": input_data + "input": input_data, + "trait_names": traits, + "sample_names": strains } @@ -91,28 +83,26 @@ def process_image(response): def run_wgcna(form_data): """function to run wgcna""" + GN3_URL = "http://127.0.0.1:8081" + wgcna_api = f"{GN3_URL}/api/wgcna/run_wgcna" # parse form data trait_dataset = fetch_trait_data(form_data) + form_data["minModuleSize"] = int(form_data["MinModuleSize"]) - response = requests.post(wgcna_api, { - "socket_id": form_data.get("socket_id"), # streaming disabled - "sample_names": list(set(strains)), - "trait_names": form_traits, - "trait_sample_data": form_strains, - "TOMtype": form_data["TOMtype"], - "minModuleSize": int(form_data["MinModuleSize"]), - "corType": form_data["corType"] + response = requests.post(wgcna_api, json={ + "sample_names": list(set(trait_dataset["sample_names"])), + "trait_names": trait_dataset["trait_names"], + "trait_sample_data": list(trait_dataset["input"].values()), + **form_data } ).json() - if response.status_code == 200: - return { - {"results": response, - "data": process_wgcna_data(response), - "image": process_image(response) - } - } + return { + "results": response, + "data": process_wgcna_data(response["data"]), + "image": process_image(response["data"]) + } -- cgit v1.2.3 From b8a2b58cbf5bb96d86d59da7e72a9cb5f874fc41 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sun, 10 Oct 2021 01:47:50 +0300 Subject: call run_wgcna in views && render test template --- wqflask/wqflask/views.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 44560427..a462b31a 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -39,8 +39,8 @@ from gn3.db.phenotypes import Probeset from gn3.db.phenotypes import Publication from gn3.db.phenotypes import PublishXRef from gn3.db.phenotypes import probeset_mapping -from gn3.db.traits import get_trait_csv_sample_data -from gn3.db.traits import update_sample_data +# from gn3.db.traits import get_trait_csv_sample_data +# from gn3.db.traits import update_sample_data from flask import current_app @@ -77,6 +77,7 @@ from wqflask.correlation_matrix import show_corr_matrix from wqflask.correlation import corr_scatter_plot # from wqflask.wgcna import wgcna_analysis # from wqflask.ctl import ctl_analysis +from wqflask.wgcna.gn3_wgcna import run_wgcna from wqflask.snp_browser import snp_browser from wqflask.search_results import SearchResultPage from wqflask.export_traits import export_search_results_csv @@ -365,18 +366,11 @@ def wcgna_setup(): return render_template("wgcna_setup.html", **request.form) -# @app.route("/wgcna_results", methods=('POST',)) -# def wcgna_results(): -# logger.info("In wgcna, request.form is:", request.form) -# logger.info(request.url) -# # Start R, load the package and pointers and create the analysis -# wgcna = wgcna_analysis.WGCNA() -# # Start the analysis, a wgcnaA object should be a separate long running thread -# wgcnaA = wgcna.run_analysis(request.form) -# # After the analysis is finished store the result -# result = wgcna.process_results(wgcnaA) -# # Display them using the template -# return render_template("wgcna_results.html", **result) +@app.route("/wgcna_results", methods=('POST',)) +def wcgna_results(): + """call the gn3 api to get wgcna response data""" + results = run_wgcna(dict(request.form)) + return render_template("test_wgcna_results.html", **results) @app.route("/ctl_setup", methods=('POST',)) -- cgit v1.2.3 From f07d5014a3d6756199bd206b4251f6d3b48bf165 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sun, 10 Oct 2021 01:49:14 +0300 Subject: pep8 formatting && remove unused functions --- wqflask/wqflask/wgcna/gn3_wgcna.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/wqflask/wqflask/wgcna/gn3_wgcna.py b/wqflask/wqflask/wgcna/gn3_wgcna.py index 96510223..c4cc2e7f 100644 --- a/wqflask/wqflask/wgcna/gn3_wgcna.py +++ b/wqflask/wqflask/wgcna/gn3_wgcna.py @@ -7,12 +7,6 @@ from types import SimpleNamespace from utility.helper_functions import get_trait_db_obs -def validate_form(requestform): - return { - "" - } - - def fetch_trait_data(requestform): """fetch trait data""" db_obj = SimpleNamespace() -- cgit v1.2.3 From 63a161e2a6d3863720aa6814f1060bed22c22a39 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sun, 10 Oct 2021 01:54:51 +0300 Subject: enable wgcna in gn2 toolbar --- wqflask/wqflask/templates/tool_buttons.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/templates/tool_buttons.html b/wqflask/wqflask/templates/tool_buttons.html index 3f9d8211..3ee5be19 100644 --- a/wqflask/wqflask/templates/tool_buttons.html +++ b/wqflask/wqflask/templates/tool_buttons.html @@ -18,13 +18,13 @@ BNW - +
- +
- - + {% if image["image_generated"] %}
@@ -140,13 +129,8 @@ let results = {{results|safe}} let phenoModules = results["data"]["output"]["net_colors"] -let phenotypes = Array.from(Object.keys(phenoModules)); +let phenotypes = Object.keys(phenoModules) let phenoMods = Object.values(phenoModules) -console.log(phenotypes) - - -console.log(typeof phenotypes); - let {col_names,mod_dataset} = {{data|safe}} $('#eigens').DataTable( { @@ -159,7 +143,6 @@ let {col_names,mod_dataset} = {{data|safe}} } ); $('#phenos').DataTable( { data:phenotypes.map((phenoName,idx)=>{ - console.log(phenoName) return [phenoName,phenoMods[idx]] }), columns: [{ -- cgit v1.2.3 From 1642c6bf0b8dee3bd3d4c32636a2c11ab97025a0 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sun, 10 Oct 2021 12:02:01 +0300 Subject: replace form input with select options --- wqflask/wqflask/templates/wgcna_setup.html | 35 ++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/wqflask/wqflask/templates/wgcna_setup.html b/wqflask/wqflask/templates/wgcna_setup.html index 9d4bbfc7..86d9fa10 100644 --- a/wqflask/wqflask/templates/wgcna_setup.html +++ b/wqflask/wqflask/templates/wgcna_setup.html @@ -35,29 +35,42 @@
-
- -
- -
+ +
+ +
+ +
+
- +
-
- -
- -
+ +
+ +
+ +
+
+ + + {% endif %}
-- cgit v1.2.3 From db9225caf0a78b13af1892d47c69463e00262d03 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 13 Oct 2021 08:22:51 +0300 Subject: disable heatmap image --- wqflask/wqflask/templates/test_wgcna_results.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/templates/test_wgcna_results.html b/wqflask/wqflask/templates/test_wgcna_results.html index 0d253564..1dddd393 100644 --- a/wqflask/wqflask/templates/test_wgcna_results.html +++ b/wqflask/wqflask/templates/test_wgcna_results.html @@ -94,9 +94,9 @@
{% endif %} -
+
-- cgit v1.2.3 From 2b5b6a119e22d0bbe29c08681c34a50f9769f38e Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Wed, 13 Oct 2021 09:06:57 +0300 Subject: Update the action button text Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/non-clustered-heatmaps-and-flipping.gmi * Update the action button text to more closely correspond to the action that the button triggers. --- wqflask/wqflask/templates/collections/view.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index d347a060..0ded66a6 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -54,7 +54,7 @@ class="btn btn-primary" data-url="{{heatmap_data_url}}" title="Generate heatmap from this collection"> - Clustered Heatmap + Generate Heatmap -- cgit v1.2.3 From ba98ef026544d4437e65a7bd248ff9591296b48e Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Wed, 13 Oct 2021 09:37:19 +0300 Subject: Add some documentation for generating heatmaps Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/non-clustered-heatmaps-and-flipping.gmi * Add some documentation on generating the heatmaps, that would be useful for the end user. --- doc/heatmap-generation.org | 34 ++++++++++++++++++++++++++++++++ doc/images/gn2_header_collections.png | Bin 0 -> 7890 bytes doc/images/heatmap_form.png | Bin 0 -> 9363 bytes doc/images/heatmap_with_hover_tools.png | Bin 0 -> 42578 bytes 4 files changed, 34 insertions(+) create mode 100644 doc/heatmap-generation.org create mode 100644 doc/images/gn2_header_collections.png create mode 100644 doc/images/heatmap_form.png create mode 100644 doc/images/heatmap_with_hover_tools.png diff --git a/doc/heatmap-generation.org b/doc/heatmap-generation.org new file mode 100644 index 00000000..a697c70b --- /dev/null +++ b/doc/heatmap-generation.org @@ -0,0 +1,34 @@ +#+STARTUP: inlineimages +#+TITLE: Heatmap Generation +#+AUTHOR: Muriithi Frederick Muriuki + +* Generating Heatmaps + +Like a lot of other features, the heatmap generation requires an existing collection. If none exists, see [[][Creating a new collection]] for how to create a new collection. + +Once you have a collection, you can navigate to the collections page by clicking on the "Collections" link in the header + + +[[./images/gn2_header_collections.png]] + +From that page, pick the collection that you want to work with by clicking on its name on the collections table. + +That takes you to that collection's page, where you can select the data that you want to use to generate the heatmap. + +** Selecting Orientation + +Once you have selected the data, select the orientation of the heatmap you want generated. You do this by selecting either *"vertical"* or *"horizontal"* in the heatmaps form: + +[[./images/heatmap_form.png]] + +Once you have selected the orientation, click on the "Generate Heatmap" button as in the image above. + +The heatmap generation might take a while, but once it is done, an image shows up above the data table. + +** Downloading the PNG copy of the Heatmap + +Once the heatmap image is shown, hovering over it, displays some tools to interact with the image. + +To download, hover over the heatmap image, and click on the "Download plot as png" icon as shown. + +[[./images/heatmap_with_hover_tools.png]] diff --git a/doc/images/gn2_header_collections.png b/doc/images/gn2_header_collections.png new file mode 100644 index 00000000..ac23f9c1 Binary files /dev/null and b/doc/images/gn2_header_collections.png differ diff --git a/doc/images/heatmap_form.png b/doc/images/heatmap_form.png new file mode 100644 index 00000000..163fbb60 Binary files /dev/null and b/doc/images/heatmap_form.png differ diff --git a/doc/images/heatmap_with_hover_tools.png b/doc/images/heatmap_with_hover_tools.png new file mode 100644 index 00000000..4ab79f99 Binary files /dev/null and b/doc/images/heatmap_with_hover_tools.png differ -- cgit v1.2.3 From 25cafac773edf3a053819b53ef860321a678941a Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 13 Oct 2021 17:46:18 +0000 Subject: Fix issue where score_type being set wrong caused an error when exporting mapping results (a while back it was changed from LOD to -logP) --- wqflask/wqflask/marker_regression/run_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index 80094057..769b9240 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -230,7 +230,7 @@ class RunMapping: self.perm_strata = get_perm_strata( self.this_trait, primary_samples, self.categorical_vars, self.samples) - self.score_type = "LOD" + self.score_type = "-logP" self.control_marker = start_vars['control_marker'] self.do_control = start_vars['do_control'] if 'mapmethod_rqtl_geno' in start_vars: -- cgit v1.2.3 From bc1b297acdd85f9bc04cd402f646ce123401b907 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 18 Oct 2021 17:55:09 +0000 Subject: Replace this_trait.dataset.type with just self.dataset.type --- wqflask/wqflask/search_results.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index bc0c08a1..5db469c1 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -200,13 +200,14 @@ class SearchResultPage: self.max_widths[key] = max(len(str(trait[key])), self.max_widths[key]) if key in self.max_widths else len(str(trait[key])) self.wide_columns_exist = False - if this_trait.dataset.type == "Publish": + if self.dataset.type == "Publish": if (self.max_widths['display_name'] > 25 or self.max_widths['description'] > 100 or self.max_widths['authors']> 80): self.wide_columns_exist = True - if this_trait.dataset.type == "ProbeSet": + if self.dataset.type == "ProbeSet": if (self.max_widths['display_name'] > 25 or self.max_widths['symbol'] > 25 or self.max_widths['description'] > 100): self.wide_columns_exist = True + self.trait_list = trait_list def search(self): -- cgit v1.2.3 From 789f91513a551dfe65133a2ae6191e6d98cbfcf2 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 18 Oct 2021 18:29:52 +0000 Subject: Changed table width to 100% for non-Geno datasets, since it can always be manually resized --- wqflask/wqflask/templates/search_result_page.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 72a4b560..5922ac75 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -126,7 +126,7 @@ {% endif %}
{% endif %} -
+
-- cgit v1.2.3 From 7f241d1205eefe5bec871ef2d5b99168e0e136d7 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 18 Oct 2021 18:30:55 +0000 Subject: Fixed the way group code was used when setting phenotype trait display name + prevent encoding error with try/except --- wqflask/wqflask/search_results.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 5db469c1..61970e7e 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -122,8 +122,8 @@ class SearchResultPage: trait_dict['display_name'] = result[0] if self.dataset.type == "Publish": - if self.dataset.group.code: - trait_dict['display_name'] = self.dataset.group.code + "_" + str(result[0]) + if result[10]: + trait_dict['display_name'] = str(result[10]) + "_" + str(result[0]) trait_dict['dataset'] = self.dataset.name trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait_dict['name'], trait_dict['dataset'])) @@ -186,7 +186,11 @@ class SearchResultPage: # Convert any bytes in dict to a normal utf-8 string for key in trait_dict.keys(): if isinstance(trait_dict[key], bytes): - trait_dict[key] = trait_dict[key].decode('utf-8') + try: + trait_dict[key] = trait_dict[key].decode('utf-8') + except UnicodeDecodeError: + trait_dict[key] = trait_dict[key].decode('latin-1') + trait_list.append(trait_dict) if self.results: -- cgit v1.2.3 From 4b1ce6c7ee290f2a11f9323085e7e559c7c7f322 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 19 Oct 2021 05:00:30 +0000 Subject: Fixed heatmap URL in view_collection --- wqflask/wqflask/collect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 3475ae5d..76ef5ca4 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -222,7 +222,7 @@ def view_collection(): collection_info = dict( trait_obs=trait_obs, uc=uc, - heatmap_data_url=f"{GN_SERVER_URL}api/heatmaps/clustered") + heatmap_data_url=f"{GN_SERVER_URL}heatmaps/clustered") if "json" in params: return json.dumps(json_version) -- cgit v1.2.3 From 12b72d28b28ede3e891afa2e679570a27110a094 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 19 Oct 2021 05:15:07 +0000 Subject: Fixed bug where the record ID for ProbeSet traits wasn't being pulled from the correct position in the query results --- wqflask/wqflask/search_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 61970e7e..f8ecd178 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -112,7 +112,7 @@ class SearchResultPage: trait_dict = {} trait_dict['index'] = index + 1 - trait_dict['name'] = result[0] + trait_dict['name'] = result[2] #ZS: Check permissions on a trait-by-trait basis for phenotype traits if self.dataset.type == "Publish": -- cgit v1.2.3 From ebfeee414acc0f59090fc86eaf2d631b7a3f0790 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 19 Oct 2021 05:29:46 +0000 Subject: Fixed some errors/logic for search results related to how the name/display name was being set --- wqflask/wqflask/search_results.py | 26 +++++++++++------------ wqflask/wqflask/templates/search_result_page.html | 12 ++++++++++- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index f8ecd178..dcc8021a 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -112,22 +112,11 @@ class SearchResultPage: trait_dict = {} trait_dict['index'] = index + 1 - trait_dict['name'] = result[2] - - #ZS: Check permissions on a trait-by-trait basis for phenotype traits - if self.dataset.type == "Publish": - permissions = check_resource_availability(self.dataset, trait_dict['name']) - if "view" not in permissions['data']: - continue - - trait_dict['display_name'] = result[0] - if self.dataset.type == "Publish": - if result[10]: - trait_dict['display_name'] = str(result[10]) + "_" + str(result[0]) trait_dict['dataset'] = self.dataset.name - trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait_dict['name'], trait_dict['dataset'])) if self.dataset.type == "ProbeSet": + trait_dict['display_name'] = result[2] + trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait_dict['display_name'], trait_dict['dataset'])) trait_dict['symbol'] = "N/A" if result[3] is None else result[3].strip() description_text = "N/A" if result[4] is None or str(result[4]) == "" else trait_dict['symbol'] @@ -144,10 +133,21 @@ class SearchResultPage: trait_dict['lod_score'] = "N/A" if result[9] is None or result[9] == "" else f"{float(result[9]) / 4.61:.1f}" trait_dict['lrs_location'] = "N/A" if result[13] is None or result[13] == "" or result[14] is None else f"Chr{result[13]}: {float(result[14]):.6f}" elif self.dataset.type == "Geno": + trait_dict['display_name'] = str(result[0]) + trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait_dict['display_name'], trait_dict['dataset'])) trait_dict['location'] = "N/A" if (result[4] != "NULL" and result[4] != "") and (result[5] != 0): trait_dict['location'] = f"Chr{result[4]}: {float(result[5]):.6f}" elif self.dataset.type == "Publish": + # Check permissions on a trait-by-trait basis for phenotype traits + trait_dict['name'] = trait_dict['display_name'] = str(result[0]) + trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait_dict['name'], trait_dict['dataset'])) + permissions = check_resource_availability(self.dataset, trait_dict['display_name']) + if "view" not in permissions['data']: + continue + + if result[10]: + trait_dict['display_name'] = str(result[10]) + "_" + str(result[0]) trait_dict['description'] = "N/A" trait_dict['pubmed_id'] = "N/A" trait_dict['pubmed_link'] = "N/A" diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 5922ac75..95842316 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -206,7 +206,7 @@ 'width': "{{ max_widths.display_name * 8 }}px", 'targets': 2, 'render': function(data, type, row, meta) { - return '' + data.display_name + '' + return '' + data.display_name + '' } }, { @@ -361,6 +361,16 @@ 'targets': 9, 'orderSequence': [ "desc", "asc"] }{% elif dataset.type == 'Geno' %}, + { + 'title': "Record", + 'type': "natural-minus-na", + 'width': "{{ max_widths.display_name * 9 }}px", + 'data': null, + 'targets': 2, + 'render': function(data, type, row, meta) { + return '' + data.display_name + '' + } + }, { 'title': "
Location
", 'type': "natural-minus-na", -- cgit v1.2.3 From c16e361c402028c02421644c652c42c641931ddf Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 19 Oct 2021 15:48:33 +0000 Subject: Fixes issue with the bytes encoding being included in the string for ProbeSet trait descriptions on the search result page --- wqflask/wqflask/search_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index dcc8021a..5ca1f9ca 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -120,7 +120,7 @@ class SearchResultPage: trait_dict['symbol'] = "N/A" if result[3] is None else result[3].strip() description_text = "N/A" if result[4] is None or str(result[4]) == "" else trait_dict['symbol'] - target_string = result[5] + target_string = result[5].decode('utf-8') if result[5] else "" description_display = description_text if target_string is None or str(target_string) == "" else description_text + "; " + str(target_string).strip() trait_dict['description'] = description_display -- cgit v1.2.3 From 2d0c4fa72757674fa5e526696c66c5b7b6b6f983 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 22 Oct 2021 02:12:54 +0000 Subject: Possible fix for the weird inconsistent correlation key error for strain BXD073bxBXD077F1 --- wqflask/wqflask/correlation/correlation_gn3_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index a18bceaf..1e3a40f2 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -65,7 +65,7 @@ def process_samples(start_vars, sample_names, excluded_samples=None): excluded_samples = () sample_vals_dict = json.loads(start_vars["sample_vals"]) for sample in sample_names: - if sample not in excluded_samples: + if sample not in excluded_samples and sample in sample_vals_dict: val = sample_vals_dict[sample] if not val.strip().lower() == "x": sample_data[str(sample)] = float(val) -- cgit v1.2.3 From 5798e6a2161384cb6cf6b6e1e28eccf3a2ee8da3 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 22 Oct 2021 11:11:10 +0200 Subject: Updated funding and credits --- wqflask/wqflask/export_traits.py | 2 +- wqflask/wqflask/templates/base.html | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/wqflask/wqflask/export_traits.py b/wqflask/wqflask/export_traits.py index 5459dc31..f4e04a4b 100644 --- a/wqflask/wqflask/export_traits.py +++ b/wqflask/wqflask/export_traits.py @@ -48,7 +48,7 @@ def export_search_results_csv(targs): if targs['filter_term'] != "None": metadata.append(["Search Filter Terms: " + targs['filter_term']]) metadata.append(["Exported Row Number: " + str(len(table_rows))]) - metadata.append(["Funding for The GeneNetwork: NIGMS (R01 GM123489, 2017-2021), NIDA (P30 DA044223, 2017-2022), NIA (R01AG043930, 2013-2018), NIAAA (U01 AA016662, U01 AA013499, U24 AA013513, U01 AA014425, 2006-2017), NIDA/NIMH/NIAAA (P20-DA 21131, 2001-2012), NCI MMHCC (U01CA105417), NCRR/BIRN (U24 RR021760)"]) + metadata.append(["Funding for The GeneNetwork: NIGMS (R01 GM123489, 2017-2026), NIDA (P30 DA044223, 2017-2022), NIA (R01AG043930, 2013-2018), NIAAA (U01 AA016662, U01 AA013499, U24 AA013513, U01 AA014425, 2006-2017), NIDA/NIMH/NIAAA (P20-DA 21131, 2001-2012), NCI MMHCC (U01CA105417), NCRR/BIRN (U24 RR021760)"]) metadata.append([]) trait_list = [] diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index d30c575a..a878efa9 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -159,17 +159,16 @@ ; June 15, 2001 as WebQTL; and Jan 5, 2005 as GeneNetwork.

- This site is currently operated by + This site is currently operated by Rob Williams, - Pjotr Prins, + Pjotr Prins, Saunak Sen, Zachary Sloan, Arthur Centeno, - Christian Fischer and Bonface Munyoki.

-

Design and code by Pjotr Prins, Zach Sloan, Arthur Centeno, Christan Fischer, Bonface Munyoki, Danny Arends, Sam Ockman, Lei Yan, Xiaodong Zhou, Christian Fernandez, - Ning Liu, Rudi Alberts, Elissa Chesler, Sujoy Roy, Evan G. Williams, Alexander G. Williams, Kenneth Manly, Jintao Wang, Robert W. Williams, and +

Design and code by Pjotr Prins, Zach Sloan, Arthur Centeno, Christan Fischer, Bonface Munyoki, Danny Arends, Arun Isaac, Alex Mwangi, Fred Muriithi, Sam Ockman, Lei Yan, Xiaodong Zhou, Christian Fernandez, + Ning Liu, Rudi Alberts, Elissa Chesler, Sujoy Roy, Evan G. Williams, Alexander G. Williams, Kenneth Manly, Jintao Wang, Robert W. Williams, and colleagues.


@@ -182,7 +181,7 @@
  • NIGMS - Systems Genetics and Precision Medicine Project (R01 GM123489, 2017-2021) + Systems Genetics and Precision Medicine Project (R01 GM123489, 2017-2026)
  • NIDA -- cgit v1.2.3 From 29d0abcefb87cab9308522d30e5c2691108d83e9 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 22 Oct 2021 12:06:20 +0200 Subject: Add NSF award --- wqflask/wqflask/templates/base.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index a878efa9..e62f571d 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -183,6 +183,11 @@ NIGMS Systems Genetics and Precision Medicine Project (R01 GM123489, 2017-2026)
    • +
  • + NSF + Panorama: Integrated Rack-Scale Acceleration for Computational Pangenomics + (PPoSS 2118709, 2021-2016) +
  • NIDA NIDA Core Center of Excellence in Transcriptomics, Systems Genetics, and the Addictome (P30 DA044223, 2017-2022) -- cgit v1.2.3 From cc32cc4f1472ddaf63a5b9428e8a67b8ba139282 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 22 Oct 2021 18:36:45 +0000 Subject: Added proxy and local GN3 URLs in tools.py (which should be set in the settings file) --- wqflask/utility/tools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index e28abb48..1219993a 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -266,6 +266,8 @@ WEBSERVER_MODE = get_setting('WEBSERVER_MODE') GN2_BASE_URL = get_setting('GN2_BASE_URL') GN2_BRANCH_URL = get_setting('GN2_BRANCH_URL') GN_SERVER_URL = get_setting('GN_SERVER_URL') +GN_PROXY_URL = get_setting('GN_PROXY_URL') +GN3_LOCAL_URL = get_setting('GN_LOCAL_URL') SERVER_PORT = get_setting_int('SERVER_PORT') SQL_URI = get_setting('SQL_URI') LOG_LEVEL = get_setting('LOG_LEVEL') -- cgit v1.2.3 From f2cdd50ebd5b927d46c8fbf7f32c9ca4ea61686f Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 22 Oct 2021 18:42:06 +0000 Subject: Replace hardcoded GN3 RQTL URL --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 1dca1b1b..34b10fc5 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -12,14 +12,11 @@ import numpy as np from base.webqtlConfig import TMPDIR from base.trait import create_trait -from utility.tools import locate +from utility.tools import locate, GN3_LOCAL_URL import utility.logger logger = utility.logger.getLogger(__name__) -GN3_RQTL_URL = "http://localhost:8086/api/rqtl/compute" -GN3_TMP_PATH = "/export/local/home/zas1024/genenetwork3/tmp" - def run_rqtl(trait_name, vals, samples, dataset, mapping_scale, model, method, num_perm, perm_strata_list, do_control, control_marker, manhattan_plot, cofactors): """Run R/qtl by making a request to the GN3 endpoint and reading in the output file(s)""" @@ -49,7 +46,7 @@ def run_rqtl(trait_name, vals, samples, dataset, mapping_scale, model, method, n if perm_strata_list: post_data["pstrata"] = True - rqtl_output = requests.post(GN3_RQTL_URL, data=post_data).json() + rqtl_output = requests.post(GN3_LOCAL_URL + "api/rqtl/compute", data=post_data).json() if num_perm > 0: return rqtl_output['perm_results'], rqtl_output['suggestive'], rqtl_output['significant'], rqtl_output['results'] else: -- cgit v1.2.3 From a05d0e4eb9412d9495a0f96980df27acb1526a03 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 22 Oct 2021 18:45:10 +0000 Subject: Replace hardcoded GN proxy URLs with one pulled from settings --- wqflask/utility/authentication_tools.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py index 6802d689..afea69e1 100644 --- a/wqflask/utility/authentication_tools.py +++ b/wqflask/utility/authentication_tools.py @@ -4,11 +4,12 @@ import requests from flask import g from base import webqtlConfig - from utility.redis_tools import (get_redis_conn, get_resource_info, get_resource_id, add_resource) +from utility.tools import GN_PROXY_URL + Redis = get_redis_conn() def check_resource_availability(dataset, trait_id=None): @@ -24,19 +25,19 @@ def check_resource_availability(dataset, trait_id=None): if resource_id: resource_info = get_resource_info(resource_id) - # ZS: If resource isn't already in redis, add it with default + # If resource isn't already in redis, add it with default # privileges if not resource_info: resource_info = add_new_resource(dataset, trait_id) - # ZS: Check if super-user - we should probably come up with some + # Check if super-user - we should probably come up with some # way to integrate this into the proxy if g.user_session.user_id in Redis.smembers("super_users"): return webqtlConfig.SUPER_PRIVILEGES response = None - the_url = "http://localhost:8080/available?resource={}&user={}".format( + the_url = GN_PROXY_URL + "available?resource={}&user={}".format( resource_id, g.user_session.user_id) try: @@ -93,7 +94,7 @@ def get_group_code(dataset): def check_admin(resource_id=None): - the_url = "http://localhost:8080/available?resource={}&user={}".format( + the_url = GN_PROXY_URL + "available?resource={}&user={}".format( resource_id, g.user_session.user_id) try: response = json.loads(requests.get(the_url).content)['admin'] -- cgit v1.2.3 From a9ca9b0da80feaa5318343dbb7b814c78b5f41f7 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 22 Oct 2021 18:48:02 +0000 Subject: Replace hardcoded GN proxy URL in decorators.py with one pulled from settings --- wqflask/wqflask/decorators.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index 54aa6795..c19e1aef 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -3,6 +3,7 @@ from flask import g from typing import Dict from functools import wraps from utility.hmac import hmac_creation +from utility.tools import GN_PROXY_URL import json import requests @@ -25,11 +26,11 @@ def edit_access_required(f): _user_id = g.user_session.record.get(b"user_id", "").decode("utf-8") response = json.loads( - requests.get("http://localhost:8080/" - "available?resource=" - f"{resource_id}&user={_user_id}").content) + requests.get(GN_PROXY_URL + "available?resource=" + f"{resource_id}&user={_user_id}").content) except: response = {} + if "edit" not in response.get("data", []): return "You need to be admin", 401 return f(*args, **kwargs) -- cgit v1.2.3 From 6bdd48e506e7c05b89ab7669ff81040fdd862ebb Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 22 Oct 2021 18:51:23 +0000 Subject: Replace hardcoded GN proxy URLs in trait.py with one pulled from settings --- wqflask/base/trait.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 96a09302..f0749858 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -7,7 +7,7 @@ from base.webqtlCaseData import webqtlCaseData from base.data_set import create_dataset from utility import hmac from utility.authentication_tools import check_resource_availability -from utility.tools import GN2_BASE_URL +from utility.tools import GN2_BASE_URL, GN_PROXY_URL from utility.redis_tools import get_redis_conn, get_resource_id from utility.db_tools import escape @@ -361,10 +361,10 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): resource_id = get_resource_id(dataset, trait.name) if dataset.type == 'Publish': - the_url = "http://localhost:8080/run-action?resource={}&user={}&branch=data&action=view".format( + the_url = GN_PROXY_URL + "run-action?resource={}&user={}&branch=data&action=view".format( resource_id, g.user_session.user_id) else: - the_url = "http://localhost:8080/run-action?resource={}&user={}&branch=data&action=view&trait={}".format( + the_url = GN_PROXY_URL + "run-action?resource={}&user={}&branch=data&action=view&trait={}".format( resource_id, g.user_session.user_id, trait.name) try: -- cgit v1.2.3 From b8f2df1f08c423923aeb72a1b2c2316e6da232dc Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 22 Oct 2021 18:51:50 +0000 Subject: Fix line pulling GN3_LOCAL_URL from settings --- wqflask/utility/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 1219993a..0efe8ca9 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -267,7 +267,7 @@ GN2_BASE_URL = get_setting('GN2_BASE_URL') GN2_BRANCH_URL = get_setting('GN2_BRANCH_URL') GN_SERVER_URL = get_setting('GN_SERVER_URL') GN_PROXY_URL = get_setting('GN_PROXY_URL') -GN3_LOCAL_URL = get_setting('GN_LOCAL_URL') +GN3_LOCAL_URL = get_setting('GN3_LOCAL_URL') SERVER_PORT = get_setting_int('SERVER_PORT') SQL_URI = get_setting('SQL_URI') LOG_LEVEL = get_setting('LOG_LEVEL') -- cgit v1.2.3 From e8b6b4e99f7a4ea649dab0c5a7bf7695531e97d2 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 22 Oct 2021 18:52:39 +0000 Subject: Include the admin privilege for the 'editors' group, since it wasn't being set before (which caused some problems) --- scripts/authentication/resource.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py index 4996f34c..1a2bcd8a 100644 --- a/scripts/authentication/resource.py +++ b/scripts/authentication/resource.py @@ -97,7 +97,8 @@ if __name__ == "__main__": for resource_id, resource in RESOURCES.items(): _resource = json.loads(resource) # str -> dict conversion _resource["group_masks"] = {args.group_id: {"metadata": "edit", - "data": "edit"}} + "data": "edit", + "admin": "edit-admins"}} REDIS_CONN.hset("resources", resource_id, json.dumps(_resource)) -- cgit v1.2.3 From 205b5e7b1796898a5dd72482c6905f05bbaa44c7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sat, 9 Oct 2021 09:32:54 +0300 Subject: api: correlation: Delete unused logger --- wqflask/wqflask/api/correlation.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/wqflask/wqflask/api/correlation.py b/wqflask/wqflask/api/correlation.py index 870f3275..ff125d96 100644 --- a/wqflask/wqflask/api/correlation.py +++ b/wqflask/wqflask/api/correlation.py @@ -15,9 +15,6 @@ from wqflask.correlation import correlation_functions from utility import webqtlUtil, helper_functions, corr_result_helpers from utility.benchmark import Bench -import utility.logger -logger = utility.logger.getLogger(__name__) - def do_correlation(start_vars): assert('db' in start_vars) -- cgit v1.2.3 From a22ffd365e669b0d8d577a82ea6bcd9c5c188b2b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sat, 9 Oct 2021 09:34:09 +0300 Subject: api: correlation: Delete unused imports --- wqflask/wqflask/api/correlation.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/wqflask/wqflask/api/correlation.py b/wqflask/wqflask/api/correlation.py index ff125d96..a135ecc8 100644 --- a/wqflask/wqflask/api/correlation.py +++ b/wqflask/wqflask/api/correlation.py @@ -1,19 +1,13 @@ import collections - import scipy -from utility.db_tools import escape - -from flask import g - from base import data_set from base.trait import create_trait, retrieve_sample_data - -from wqflask.correlation.show_corr_results import generate_corr_json +from flask import g +from utility import corr_result_helpers +from utility.db_tools import escape from wqflask.correlation import correlation_functions -from utility import webqtlUtil, helper_functions, corr_result_helpers -from utility.benchmark import Bench def do_correlation(start_vars): -- cgit v1.2.3 From 0c6ad845e6f461e880b423bd8f79156754658f8a Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sat, 9 Oct 2021 09:43:40 +0300 Subject: Move markdown_routes to api/markdown All new API definitions should be migrated to "wqflask/ api" --- .../tests/unit/wqflask/api/test_markdown_routes.py | 54 ++++++ wqflask/tests/unit/wqflask/test_markdown_routes.py | 54 ------ wqflask/wqflask/__init__.py | 16 +- wqflask/wqflask/api/correlation.py | 6 - wqflask/wqflask/api/markdown.py | 186 +++++++++++++++++++++ wqflask/wqflask/markdown_routes.py | 186 --------------------- 6 files changed, 248 insertions(+), 254 deletions(-) create mode 100644 wqflask/tests/unit/wqflask/api/test_markdown_routes.py delete mode 100644 wqflask/tests/unit/wqflask/test_markdown_routes.py create mode 100644 wqflask/wqflask/api/markdown.py delete mode 100644 wqflask/wqflask/markdown_routes.py diff --git a/wqflask/tests/unit/wqflask/api/test_markdown_routes.py b/wqflask/tests/unit/wqflask/api/test_markdown_routes.py new file mode 100644 index 00000000..1c513ac5 --- /dev/null +++ b/wqflask/tests/unit/wqflask/api/test_markdown_routes.py @@ -0,0 +1,54 @@ +"""Test functions for wqflask/api/markdown.py""" + +import unittest +from unittest import mock + +from dataclasses import dataclass +from wqflask.api.markdown import render_markdown + + +@dataclass +class MockRequests404: + status_code: int = 404 + + +@dataclass +class MockRequests200: + status_code: int = 200 + content: str = b""" +# Glossary +This is some content + +## Sub-heading +This is another sub-heading""" + + +class TestMarkdownRoutesFunctions(unittest.TestCase): + """Test cases for functions in markdown""" + + @mock.patch('wqflask.api.markdown.requests.get') + def test_render_markdown_when_fetching_locally(self, requests_mock): + requests_mock.return_value = MockRequests404() + markdown_content = render_markdown("general/glossary/glossary.md") + requests_mock.assert_called_with( + "https://raw.githubusercontent.com" + "/genenetwork/gn-docs/" + "master/general/" + "glossary/glossary.md") + self.assertRegexpMatches(markdown_content, + "Content for general/glossary/glossary.md not available.") + + @mock.patch('wqflask.api.markdown.requests.get') + def test_render_markdown_when_fetching_remotely(self, requests_mock): + requests_mock.return_value = MockRequests200() + markdown_content = render_markdown("general/glossary/glossary.md") + requests_mock.assert_called_with( + "https://raw.githubusercontent.com" + "/genenetwork/gn-docs/" + "master/general/" + "glossary/glossary.md") + self.assertEqual("""

    Glossary

    +

    This is some content

    +

    Sub-heading

    +

    This is another sub-heading

    """, + markdown_content) diff --git a/wqflask/tests/unit/wqflask/test_markdown_routes.py b/wqflask/tests/unit/wqflask/test_markdown_routes.py deleted file mode 100644 index 90e0f17c..00000000 --- a/wqflask/tests/unit/wqflask/test_markdown_routes.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Test functions in markdown utils""" - -import unittest -from unittest import mock - -from dataclasses import dataclass -from wqflask.markdown_routes import render_markdown - - -@dataclass -class MockRequests404: - status_code: int = 404 - - -@dataclass -class MockRequests200: - status_code: int = 200 - content: str = b""" -# Glossary -This is some content - -## Sub-heading -This is another sub-heading""" - - -class TestMarkdownRoutesFunctions(unittest.TestCase): - """Test cases for functions in markdown_routes""" - - @mock.patch('wqflask.markdown_routes.requests.get') - def test_render_markdown_when_fetching_locally(self, requests_mock): - requests_mock.return_value = MockRequests404() - markdown_content = render_markdown("general/glossary/glossary.md") - requests_mock.assert_called_with( - "https://raw.githubusercontent.com" - "/genenetwork/gn-docs/" - "master/general/" - "glossary/glossary.md") - self.assertRegexpMatches(markdown_content, - "Content for general/glossary/glossary.md not available.") - - @mock.patch('wqflask.markdown_routes.requests.get') - def test_render_markdown_when_fetching_remotely(self, requests_mock): - requests_mock.return_value = MockRequests200() - markdown_content = render_markdown("general/glossary/glossary.md") - requests_mock.assert_called_with( - "https://raw.githubusercontent.com" - "/genenetwork/gn-docs/" - "master/general/" - "glossary/glossary.md") - self.assertEqual("""

    Glossary

    -

    This is some content

    -

    Sub-heading

    -

    This is another sub-heading

    """, - markdown_content) diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index 8b2055cd..5758b13e 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -8,14 +8,14 @@ from flask import Flask from typing import Tuple from urllib.parse import urlparse from utility import formatting -from wqflask.markdown_routes import glossary_blueprint -from wqflask.markdown_routes import references_blueprint -from wqflask.markdown_routes import links_blueprint -from wqflask.markdown_routes import policies_blueprint -from wqflask.markdown_routes import environments_blueprint -from wqflask.markdown_routes import facilities_blueprint -from wqflask.markdown_routes import blogs_blueprint -from wqflask.markdown_routes import news_blueprint +from wqflask.api.markdown import glossary_blueprint +from wqflask.api.markdown import references_blueprint +from wqflask.api.markdown import links_blueprint +from wqflask.api.markdown import policies_blueprint +from wqflask.api.markdown import environments_blueprint +from wqflask.api.markdown import facilities_blueprint +from wqflask.api.markdown import blogs_blueprint +from wqflask.api.markdown import news_blueprint app = Flask(__name__) diff --git a/wqflask/wqflask/api/correlation.py b/wqflask/wqflask/api/correlation.py index a135ecc8..9b875c99 100644 --- a/wqflask/wqflask/api/correlation.py +++ b/wqflask/wqflask/api/correlation.py @@ -9,7 +9,6 @@ from utility.db_tools import escape from wqflask.correlation import correlation_functions - def do_correlation(start_vars): assert('db' in start_vars) assert('target_db' in start_vars) @@ -26,7 +25,6 @@ def do_correlation(start_vars): corr_results = calculate_results( this_trait, this_dataset, target_dataset, corr_params) - #corr_results = collections.OrderedDict(sorted(corr_results.items(), key=lambda t: -abs(t[1][0]))) final_results = [] for _trait_counter, trait in enumerate(list(corr_results.keys())[:corr_params['return_count']]): @@ -54,11 +52,7 @@ def do_correlation(start_vars): "#_strains": num_overlap, "p_value": sample_p } - final_results.append(result_dict) - - # json_corr_results = generate_corr_json(final_corr_results, this_trait, this_dataset, target_dataset, for_api = True) - return final_results diff --git a/wqflask/wqflask/api/markdown.py b/wqflask/wqflask/api/markdown.py new file mode 100644 index 00000000..580b9ac0 --- /dev/null +++ b/wqflask/wqflask/api/markdown.py @@ -0,0 +1,186 @@ +"""Markdown routes + +Render pages from github, or if they are unavailable, look for it else where +""" + +import requests +import markdown +import os +import sys + +from bs4 import BeautifulSoup # type: ignore + +from flask import send_from_directory +from flask import Blueprint +from flask import render_template + +from typing import Dict +from typing import List + +glossary_blueprint = Blueprint('glossary_blueprint', __name__) +references_blueprint = Blueprint("references_blueprint", __name__) +environments_blueprint = Blueprint("environments_blueprint", __name__) +links_blueprint = Blueprint("links_blueprint", __name__) +policies_blueprint = Blueprint("policies_blueprint", __name__) +facilities_blueprint = Blueprint("facilities_blueprint", __name__) +news_blueprint = Blueprint("news_blueprint", __name__) + +blogs_blueprint = Blueprint("blogs_blueprint", __name__) + + +def render_markdown(file_name, is_remote_file=True): + """Try to fetch the file name from Github and if that fails, try to +look for it inside the file system """ + github_url = ("https://raw.githubusercontent.com/" + "genenetwork/gn-docs/master/") + + if not is_remote_file: + text = "" + with open(file_name, "r", encoding="utf-8") as input_file: + text = input_file.read() + return markdown.markdown(text, + extensions=['tables']) + + md_content = requests.get(f"{github_url}{file_name}") + + if md_content.status_code == 200: + return markdown.markdown(md_content.content.decode("utf-8"), + extensions=['tables']) + + return (f"\nContent for {file_name} not available. " + "Please check " + "(here to see where content exists)" + "[https://github.com/genenetwork/gn-docs]. " + "Please reach out to the gn2 team to have a look at this") + + +def get_file_from_python_search_path(pathname_suffix): + cands = [os.path.join(d, pathname_suffix) for d in sys.path] + try: + return list(filter(os.path.exists, cands))[0] + except IndexError: + return None + + +def get_blogs(user: str = "genenetwork", + repo_name: str = "gn-docs") -> dict: + + blogs: Dict[int, List] = {} + github_url = f"https://api.github.com/repos/{user}/{repo_name}/git/trees/master?recursive=1" + + repo_tree = requests.get(github_url).json()["tree"] + + for data in repo_tree: + path_name = data["path"] + if path_name.startswith("blog") and path_name.endswith(".md"): + split_path = path_name.split("/")[1:] + try: + year, title, file_name = split_path + except Exception as e: + year, file_name = split_path + title = "" + + subtitle = os.path.splitext(file_name)[0] + + blog = { + "title": title, + "subtitle": subtitle, + "full_path": path_name + } + + if year in blogs: + blogs[int(year)].append(blog) + else: + blogs[int(year)] = [blog] + + return dict(sorted(blogs.items(), key=lambda x: x[0], reverse=True)) + + +@glossary_blueprint.route('/') +def glossary(): + return render_template( + "glossary.html", + rendered_markdown=render_markdown("general/glossary/glossary.md")), 200 + + +@references_blueprint.route('/') +def references(): + return render_template( + "references.html", + rendered_markdown=render_markdown("general/references/references.md")), 200 + + +@news_blueprint.route('/') +def news(): + return render_template( + "news.html", + rendered_markdown=render_markdown("general/news/news.md")), 200 + + +@environments_blueprint.route("/") +def environments(): + + md_file = get_file_from_python_search_path("wqflask/DEPENDENCIES.md") + svg_file = get_file_from_python_search_path( + "wqflask/dependency-graph.html") + svg_data = None + if svg_file: + with open(svg_file, 'r') as f: + svg_data = "".join( + BeautifulSoup(f.read(), + 'lxml').body.script.contents) + + if md_file is not None: + return ( + render_template("environment.html", + svg_data=svg_data, + rendered_markdown=render_markdown( + md_file, + is_remote_file=False)), + 200 + ) + # Fallback: Fetch file from server + return (render_template( + "environment.html", + svg_data=None, + rendered_markdown=render_markdown( + "general/environments/environments.md")), + 200) + + +@environments_blueprint.route('/svg-dependency-graph') +def svg_graph(): + directory, file_name, _ = get_file_from_python_search_path( + "wqflask/dependency-graph.svg").partition("dependency-graph.svg") + return send_from_directory(directory, file_name) + + +@links_blueprint.route("/") +def links(): + return render_template( + "links.html", + rendered_markdown=render_markdown("general/links/links.md")), 200 + + +@policies_blueprint.route("/") +def policies(): + return render_template( + "policies.html", + rendered_markdown=render_markdown("general/policies/policies.md")), 200 + + +@facilities_blueprint.route("/") +def facilities(): + return render_template("facilities.html", rendered_markdown=render_markdown("general/help/facilities.md")), 200 + + +@blogs_blueprint.route("/") +def display_blog(blog_path): + return render_template("blogs.html", rendered_markdown=render_markdown(blog_path)) + + +@blogs_blueprint.route("/") +def blogs_list(): + blogs = get_blogs() + + return render_template("blogs_list.html", blogs=blogs) diff --git a/wqflask/wqflask/markdown_routes.py b/wqflask/wqflask/markdown_routes.py deleted file mode 100644 index 580b9ac0..00000000 --- a/wqflask/wqflask/markdown_routes.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Markdown routes - -Render pages from github, or if they are unavailable, look for it else where -""" - -import requests -import markdown -import os -import sys - -from bs4 import BeautifulSoup # type: ignore - -from flask import send_from_directory -from flask import Blueprint -from flask import render_template - -from typing import Dict -from typing import List - -glossary_blueprint = Blueprint('glossary_blueprint', __name__) -references_blueprint = Blueprint("references_blueprint", __name__) -environments_blueprint = Blueprint("environments_blueprint", __name__) -links_blueprint = Blueprint("links_blueprint", __name__) -policies_blueprint = Blueprint("policies_blueprint", __name__) -facilities_blueprint = Blueprint("facilities_blueprint", __name__) -news_blueprint = Blueprint("news_blueprint", __name__) - -blogs_blueprint = Blueprint("blogs_blueprint", __name__) - - -def render_markdown(file_name, is_remote_file=True): - """Try to fetch the file name from Github and if that fails, try to -look for it inside the file system """ - github_url = ("https://raw.githubusercontent.com/" - "genenetwork/gn-docs/master/") - - if not is_remote_file: - text = "" - with open(file_name, "r", encoding="utf-8") as input_file: - text = input_file.read() - return markdown.markdown(text, - extensions=['tables']) - - md_content = requests.get(f"{github_url}{file_name}") - - if md_content.status_code == 200: - return markdown.markdown(md_content.content.decode("utf-8"), - extensions=['tables']) - - return (f"\nContent for {file_name} not available. " - "Please check " - "(here to see where content exists)" - "[https://github.com/genenetwork/gn-docs]. " - "Please reach out to the gn2 team to have a look at this") - - -def get_file_from_python_search_path(pathname_suffix): - cands = [os.path.join(d, pathname_suffix) for d in sys.path] - try: - return list(filter(os.path.exists, cands))[0] - except IndexError: - return None - - -def get_blogs(user: str = "genenetwork", - repo_name: str = "gn-docs") -> dict: - - blogs: Dict[int, List] = {} - github_url = f"https://api.github.com/repos/{user}/{repo_name}/git/trees/master?recursive=1" - - repo_tree = requests.get(github_url).json()["tree"] - - for data in repo_tree: - path_name = data["path"] - if path_name.startswith("blog") and path_name.endswith(".md"): - split_path = path_name.split("/")[1:] - try: - year, title, file_name = split_path - except Exception as e: - year, file_name = split_path - title = "" - - subtitle = os.path.splitext(file_name)[0] - - blog = { - "title": title, - "subtitle": subtitle, - "full_path": path_name - } - - if year in blogs: - blogs[int(year)].append(blog) - else: - blogs[int(year)] = [blog] - - return dict(sorted(blogs.items(), key=lambda x: x[0], reverse=True)) - - -@glossary_blueprint.route('/') -def glossary(): - return render_template( - "glossary.html", - rendered_markdown=render_markdown("general/glossary/glossary.md")), 200 - - -@references_blueprint.route('/') -def references(): - return render_template( - "references.html", - rendered_markdown=render_markdown("general/references/references.md")), 200 - - -@news_blueprint.route('/') -def news(): - return render_template( - "news.html", - rendered_markdown=render_markdown("general/news/news.md")), 200 - - -@environments_blueprint.route("/") -def environments(): - - md_file = get_file_from_python_search_path("wqflask/DEPENDENCIES.md") - svg_file = get_file_from_python_search_path( - "wqflask/dependency-graph.html") - svg_data = None - if svg_file: - with open(svg_file, 'r') as f: - svg_data = "".join( - BeautifulSoup(f.read(), - 'lxml').body.script.contents) - - if md_file is not None: - return ( - render_template("environment.html", - svg_data=svg_data, - rendered_markdown=render_markdown( - md_file, - is_remote_file=False)), - 200 - ) - # Fallback: Fetch file from server - return (render_template( - "environment.html", - svg_data=None, - rendered_markdown=render_markdown( - "general/environments/environments.md")), - 200) - - -@environments_blueprint.route('/svg-dependency-graph') -def svg_graph(): - directory, file_name, _ = get_file_from_python_search_path( - "wqflask/dependency-graph.svg").partition("dependency-graph.svg") - return send_from_directory(directory, file_name) - - -@links_blueprint.route("/") -def links(): - return render_template( - "links.html", - rendered_markdown=render_markdown("general/links/links.md")), 200 - - -@policies_blueprint.route("/") -def policies(): - return render_template( - "policies.html", - rendered_markdown=render_markdown("general/policies/policies.md")), 200 - - -@facilities_blueprint.route("/") -def facilities(): - return render_template("facilities.html", rendered_markdown=render_markdown("general/help/facilities.md")), 200 - - -@blogs_blueprint.route("/") -def display_blog(blog_path): - return render_template("blogs.html", rendered_markdown=render_markdown(blog_path)) - - -@blogs_blueprint.route("/") -def blogs_list(): - blogs = get_blogs() - - return render_template("blogs_list.html", blogs=blogs) -- cgit v1.2.3 From 2f26b77c7be370dad03e9b8e2ce7f6040ccce528 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 12 Oct 2021 14:19:03 +0300 Subject: Add default setting for REDIS_URL --- etc/default_settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/default_settings.py b/etc/default_settings.py index 023aa53b..651cc55e 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -26,6 +26,9 @@ import sys GN_VERSION = open("../etc/VERSION", "r").read() +# Redis +REDIS_URL = "redis://:@localhost:6379/0" + # ---- MySQL SQL_URI = "mysql://gn2:mysql_password@localhost/db_webqtl_s" -- cgit v1.2.3 From eb2e43434c137a070db5943db21d3b92afdd9a91 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 12 Oct 2021 14:19:26 +0300 Subject: Mark `get_resource_info` as deprecated --- wqflask/utility/redis_tools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py index de9dde46..e68397ab 100644 --- a/wqflask/utility/redis_tools.py +++ b/wqflask/utility/redis_tools.py @@ -4,6 +4,7 @@ import datetime import redis # used for collections +from deprecated import deprecated from utility.hmac import hmac_creation from utility.logger import getLogger logger = getLogger(__name__) @@ -321,6 +322,7 @@ def get_resource_id(dataset, trait_id=None): return resource_id +@deprecated def get_resource_info(resource_id): resource_info = Redis.hget("resources", resource_id) if resource_info: -- cgit v1.2.3 From 212d19c29a42bd6966965b166cdbb4dd642e5eb4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 13 Oct 2021 13:05:26 +0300 Subject: Add test-cases for `get_user_membership` --- .../tests/unit/wqflask/test_resource_manager.py | 51 ++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 wqflask/tests/unit/wqflask/test_resource_manager.py diff --git a/wqflask/tests/unit/wqflask/test_resource_manager.py b/wqflask/tests/unit/wqflask/test_resource_manager.py new file mode 100644 index 00000000..a27f40e1 --- /dev/null +++ b/wqflask/tests/unit/wqflask/test_resource_manager.py @@ -0,0 +1,51 @@ +"""Test cases for wqflask/resource_manager.py""" +import unittest + +from unittest import mock +from wqflask.resource_manager import get_user_membership + + +class TestGetUserMembership(unittest.TestCase): + """Test cases for `get_user_membership`""" + + def setUp(self): + conn = mock.MagicMock() + conn.hgetall.return_value = { + '7fa95d07-0e2d-4bc5-b47c-448fdc1260b2': ( + '{"name": "editors", ' + '"admins": ["8ad942fe-490d-453e-bd37-56f252e41604", "rand"], ' + '"members": ["8ad942fe-490d-453e-bd37-56f252e41603", ' + '"rand"], ' + '"changed_timestamp": "Oct 06 2021 06:39PM", ' + '"created_timestamp": "Oct 06 2021 06:39PM"}')} + self.conn = conn + + def test_user_is_group_member_only(self): + """Test that a user is only a group member""" + self.assertEqual( + get_user_membership( + conn=self.conn, + user_id="8ad942fe-490d-453e-bd37-56f252e41603", + group_id="7fa95d07-0e2d-4bc5-b47c-448fdc1260b2"), + {"member": True, + "admin": False}) + + def test_user_is_group_admin_only(self): + """Test that a user is a group admin only""" + self.assertEqual( + get_user_membership( + conn=self.conn, + user_id="8ad942fe-490d-453e-bd37-56f252e41604", + group_id="7fa95d07-0e2d-4bc5-b47c-448fdc1260b2"), + {"member": False, + "admin": True}) + + def test_user_is_both_group_member_and_admin(self): + """Test that a user is both an admin and member of a group""" + self.assertEqual( + get_user_membership( + conn=self.conn, + user_id="rand", + group_id="7fa95d07-0e2d-4bc5-b47c-448fdc1260b2"), + {"member": True, + "admin": True}) -- cgit v1.2.3 From f42d0d5b04f2974e6d93155ef7b936653a9d6248 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 13 Oct 2021 13:07:48 +0300 Subject: resource_manager: Delete every route & helper method in this module This module is coupled to `utility.redis_tools` which needs to be deprecated. Also, the routes are created by importing `app` instead of creating a new blueprint. --- wqflask/wqflask/resource_manager.py | 130 ------------------------------------ 1 file changed, 130 deletions(-) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index c54dd0b3..0b1805d8 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -1,144 +1,14 @@ import json -from flask import (Flask, g, render_template, url_for, request, make_response, - redirect, flash) -from wqflask import app -from utility.authentication_tools import check_owner_or_admin -from utility.redis_tools import get_resource_info, get_group_info, get_groups_like_unique_column, get_user_id, get_user_by_unique_column, get_users_like_unique_column, add_access_mask, add_resource, change_resource_owner -@app.route("/resources/manage", methods=('GET', 'POST')) -def manage_resource(): - params = request.form if request.form else request.args - if 'resource_id' in request.args: - resource_id = request.args['resource_id'] - admin_status = check_owner_or_admin(resource_id=resource_id) - resource_info = get_resource_info(resource_id) - group_masks = resource_info['group_masks'] - group_masks_with_names = get_group_names(group_masks) - default_mask = resource_info['default_mask']['data'] - owner_id = resource_info['owner_id'] - owner_display_name = None - if owner_id != "none": - try: # ZS: User IDs are sometimes stored in Redis as bytes and sometimes as strings, so this is just to avoid any errors for the time being - owner_id = str.encode(owner_id) - except: - pass - owner_info = get_user_by_unique_column("user_id", owner_id) - if 'name' in owner_info: - owner_display_name = owner_info['full_name'] - elif 'user_name' in owner_info: - owner_display_name = owner_info['user_name'] - elif 'email_address' in owner_info: - owner_display_name = owner_info['email_address'] - return render_template("admin/manage_resource.html", owner_name=owner_display_name, resource_id=resource_id, resource_info=resource_info, default_mask=default_mask, group_masks=group_masks_with_names, admin_status=admin_status) -@app.route("/search_for_users", methods=('POST',)) -def search_for_user(): - params = request.form - user_list = [] - user_list += get_users_like_unique_column("full_name", params['user_name']) - user_list += get_users_like_unique_column( - "email_address", params['user_email']) - return json.dumps(user_list) - - -@app.route("/search_for_groups", methods=('POST',)) -def search_for_groups(): - params = request.form - group_list = [] - group_list += get_groups_like_unique_column("id", params['group_id']) - group_list += get_groups_like_unique_column("name", params['group_name']) - - user_list = [] - user_list += get_users_like_unique_column("full_name", params['user_name']) - user_list += get_users_like_unique_column( - "email_address", params['user_email']) - for user in user_list: - group_list += get_groups_like_unique_column("admins", user['user_id']) - group_list += get_groups_like_unique_column("members", user['user_id']) - - return json.dumps(group_list) - - -@app.route("/resources/change_owner", methods=('POST',)) -def change_owner(): - resource_id = request.form['resource_id'] - if 'new_owner' in request.form: - admin_status = check_owner_or_admin(resource_id=resource_id) - if admin_status == "owner": - new_owner_id = request.form['new_owner'] - change_resource_owner(resource_id, new_owner_id) - flash("The resource's owner has beeen changed.", "alert-info") - return redirect(url_for("manage_resource", resource_id=resource_id)) - else: - flash("You lack the permissions to make this change.", "error") - return redirect(url_for("manage_resource", resource_id=resource_id)) - else: - return render_template("admin/change_resource_owner.html", resource_id=resource_id) - - -@app.route("/resources/change_default_privileges", methods=('POST',)) -def change_default_privileges(): - resource_id = request.form['resource_id'] - admin_status = check_owner_or_admin(resource_id=resource_id) - if admin_status == "owner" or admin_status == "edit-admins": - resource_info = get_resource_info(resource_id) - default_mask = resource_info['default_mask'] - if request.form['open_to_public'] == "True": - default_mask['data'] = 'view' - else: - default_mask['data'] = 'no-access' - resource_info['default_mask'] = default_mask - add_resource(resource_info) - flash("Your changes have been saved.", "alert-info") - return redirect(url_for("manage_resource", resource_id=resource_id)) - else: - return redirect(url_for("no_access_page")) - - -@app.route("/resources/add_group", methods=('POST',)) -def add_group_to_resource(): - resource_id = request.form['resource_id'] admin_status = check_owner_or_admin(resource_id=resource_id) - if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access": - if 'selected_group' in request.form: - group_id = request.form['selected_group'] - resource_info = get_resource_info(resource_id) - default_privileges = resource_info['default_mask'] - return render_template("admin/set_group_privileges.html", resource_id=resource_id, group_id=group_id, default_privileges=default_privileges) - elif all(key in request.form for key in ('data_privilege', 'metadata_privilege', 'admin_privilege')): - group_id = request.form['group_id'] - group_name = get_group_info(group_id)['name'] - access_mask = { - 'data': request.form['data_privilege'], - 'metadata': request.form['metadata_privilege'], - 'admin': request.form['admin_privilege'] - } - add_access_mask(resource_id, group_id, access_mask) - flash("Privileges have been added for group {}.".format( - group_name), "alert-info") - return redirect(url_for("manage_resource", resource_id=resource_id)) - else: - return render_template("admin/search_for_groups.html", resource_id=resource_id) - else: - return redirect(url_for("no_access_page")) - - -def get_group_names(group_masks): - group_masks_with_names = {} - for group_id, group_mask in list(group_masks.items()): - this_mask = group_mask - group_name = get_group_info(group_id)['name'] - this_mask['name'] = group_name - group_masks_with_names[group_id] = this_mask - - return group_masks_with_names -- cgit v1.2.3 From 026a0f4e46a1909b16433bf1620f1b778fa6b913 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 13 Oct 2021 13:10:40 +0300 Subject: resource_manager: Add `get_user_membership` method --- wqflask/wqflask/resource_manager.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 0b1805d8..dfd853b2 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -1,8 +1,10 @@ +import redis import json +from typing import Dict @@ -11,4 +13,31 @@ import json +def get_user_membership(conn: redis.Redis, user_id: str, + group_id: str) -> Dict: + """Return a dictionary that indicates whether the `user_id` is a + member or admin of `group_id`. + + Args: + - conn: a Redis Connection with the responses decoded. + - user_id: a user's unique id + e.g. '8ad942fe-490d-453e-bd37-56f252e41603' + - group_id: a group's unique id + e.g. '7fa95d07-0e2d-4bc5-b47c-448fdc1260b2' + + Returns: + A dict indicating whether the user is an admin or a member of + the group: {"member": True, "admin": False} + + """ + results = {"member": False, "admin": False} + for key, value in conn.hgetall('groups').items(): + if key == group_id: + group_info = json.loads(value) + if user_id in group_info.get("admins"): + results["admin"] = True + if user_id in group_info.get("members"): + results["member"] = True + break + return results admin_status = check_owner_or_admin(resource_id=resource_id) -- cgit v1.2.3 From 6195ad475cddce2007b125b856829cbb41defd30 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 15 Oct 2021 08:49:23 +0300 Subject: resource_manager: Add OrderedEnum Borrowed from: --- wqflask/wqflask/resource_manager.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index dfd853b2..330600ed 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -1,13 +1,26 @@ import redis import json +import functools +from enum import Enum from typing import Dict +@functools.total_ordering +class OrderedEnum(Enum): + @classmethod + @functools.lru_cache(None) + def _member_list(cls): + return list(cls) + def __lt__(self, other): + if self.__class__ is other.__class__: + member_list = self.__class__._member_list() + return member_list.index(self) < member_list.index(other) + return NotImplemented -- cgit v1.2.3 From 3a1db0b113a6ab32926cf6070e7cc89459f83041 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 15 Oct 2021 08:50:26 +0300 Subject: Add immutable data structures for admin and data access roles --- wqflask/wqflask/resource_manager.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 330600ed..7f4718f0 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -2,7 +2,7 @@ import redis import json import functools -from enum import Enum +from enum import Enum, unique @@ -23,8 +23,18 @@ class OrderedEnum(Enum): return NotImplemented +@unique +class DataRole(OrderedEnum): + NO_ACCESS = "no-access" + VIEW = "view" + EDIT = "edit" +@unique +class AdminRole(OrderedEnum): + NOT_ADMIN = "not-admin" + EDIT_ACCESS = "edit-access" + EDIT_ADMINS = "edit-admins" def get_user_membership(conn: redis.Redis, user_id: str, group_id: str) -> Dict: -- cgit v1.2.3 From 610335cb3c3030cf39e91ad3232d468b388fc340 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 15 Oct 2021 08:51:46 +0300 Subject: Get a users access mask If a user has several access roles, select the highest role. --- .../tests/unit/wqflask/test_resource_manager.py | 59 ++++++++++++++++++++++ wqflask/wqflask/resource_manager.py | 48 ++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/wqflask/tests/unit/wqflask/test_resource_manager.py b/wqflask/tests/unit/wqflask/test_resource_manager.py index a27f40e1..b0b7e6a3 100644 --- a/wqflask/tests/unit/wqflask/test_resource_manager.py +++ b/wqflask/tests/unit/wqflask/test_resource_manager.py @@ -3,6 +3,9 @@ import unittest from unittest import mock from wqflask.resource_manager import get_user_membership +from wqflask.resource_manager import get_user_access_roles +from wqflask.resource_manager import DataRole +from wqflask.resource_manager import AdminRole class TestGetUserMembership(unittest.TestCase): @@ -49,3 +52,59 @@ class TestGetUserMembership(unittest.TestCase): group_id="7fa95d07-0e2d-4bc5-b47c-448fdc1260b2"), {"member": True, "admin": True}) + + +class TestCheckUserAccessRole(unittest.TestCase): + """Test cases for `get_user_access_roles`""" + + def setUp(self): + conn = mock.MagicMock() + conn.hget.return_value = ( + '{"owner_id": "8ad942fe-490d-453e-bd37", ' + '"default_mask": {"data": "no-access", ' + '"metadata": "no-access", ' + '"admin": "not-admin"}, ' + '"group_masks": ' + '{"7fa95d07-0e2d-4bc5-b47c-448fdc1260b2": ' + '{"metadata": "edit", "data": "edit"}}, ' + '"name": "_14329", "' + 'data": {"dataset": 1, "trait": 14329}, ' + '"type": "dataset-publish"}') + + conn.hgetall.return_value = { + '7fa95d07-0e2d-4bc5-b47c-448fdc1260b2': ( + '{"name": "editors", ' + '"admins": ["8ad942fe-490d-453e-bd37-56f252e41604", "rand"], ' + '"members": ["8ad942fe-490d-453e-bd37-56f252e41603", ' + '"rand"], ' + '"changed_timestamp": "Oct 06 2021 06:39PM", ' + '"created_timestamp": "Oct 06 2021 06:39PM"}')} + self.conn = conn + + def test_get_user_access_when_owner(self): + """Test that the right access roles are set""" + self.assertEqual(get_user_access_roles( + conn=self.conn, + resource_id="", # Can be anything + user_id="8ad942fe-490d-453e-bd37"), + {"data": DataRole.EDIT, + "metadata": DataRole.EDIT, + "admin": AdminRole.EDIT_ACCESS}) + + def test_get_user_access_default_mask(self): + self.assertEqual(get_user_access_roles( + conn=self.conn, + resource_id="", # Can be anything + user_id=""), + {"data": DataRole.NO_ACCESS, + "metadata": DataRole.NO_ACCESS, + "admin": AdminRole.NOT_ADMIN}) + + def test_get_user_access_group_mask(self): + self.assertEqual(get_user_access_roles( + conn=self.conn, + resource_id="", # Can be anything + user_id="8ad942fe-490d-453e-bd37-56f252e41603"), + {"data": DataRole.EDIT, + "metadata": DataRole.EDIT, + "admin": AdminRole.NOT_ADMIN}) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 7f4718f0..a3a94f9e 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -63,4 +63,52 @@ def get_user_membership(conn: redis.Redis, user_id: str, results["member"] = True break return results + + +def get_user_access_roles(conn: redis.Redis, + resource_id: str, + user_id: str) -> Dict: + """Get the highest access roles for a given user + + Args: + - conn: A redis connection with `decoded_responses == True`. + - resource_id: The unique id of a given resource. + + Returns: + A dict indicating the highest access role the user has. + """ + # This is the default access role + access_role = { + "data": [DataRole.NO_ACCESS], + "metadata": [DataRole.NO_ACCESS], + "admin": [AdminRole.NOT_ADMIN], + } + resource_info = json.loads(conn.hget('resources', resource_id)) + + # Check the resource's default mask + if default_mask := resource_info.get("default_mask"): + access_role["data"].append(DataRole(default_mask.get("data"))) + access_role["metadata"].append(DataRole(default_mask.get("metadata"))) + access_role["admin"].append(AdminRole(default_mask.get("admin"))) + + # Then check if the user is the owner Check with Zach and Rob if + # the owner, be default should, as the lowest access_roles, edit + # access + if resource_info.get("owner_id") == user_id: + access_role["data"].append(DataRole.EDIT) + access_role["metadata"].append(DataRole.EDIT) + access_role["admin"].append(AdminRole.EDIT_ACCESS) + + # Check the group mask. If the user is in that group mask, use the + # access roles for that group + if group_masks := resource_info.get("group_masks"): + for group_id, roles in group_masks.items(): + user_membership = get_user_membership(conn=conn, + user_id=user_id, + group_id=group_id) + if any(user_membership.values()): + access_role["data"].append(DataRole(roles.get("data"))) + access_role["metadata"].append( + DataRole(roles.get("metadata"))) + return {k: max(v) for k, v in access_role.items()} admin_status = check_owner_or_admin(resource_id=resource_id) -- cgit v1.2.3 From da86bc79798c05ec469d76f375741f306213e4d0 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 15 Oct 2021 13:01:38 +0300 Subject: Replace "resource_id" with "resource_info" dict This avoids calling Redis twice when fetching metadata about the resource. --- .../tests/unit/wqflask/test_resource_manager.py | 36 +++++++++++++--------- wqflask/wqflask/resource_manager.py | 10 +++--- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/wqflask/tests/unit/wqflask/test_resource_manager.py b/wqflask/tests/unit/wqflask/test_resource_manager.py index b0b7e6a3..9d5aaf0b 100644 --- a/wqflask/tests/unit/wqflask/test_resource_manager.py +++ b/wqflask/tests/unit/wqflask/test_resource_manager.py @@ -58,18 +58,26 @@ class TestCheckUserAccessRole(unittest.TestCase): """Test cases for `get_user_access_roles`""" def setUp(self): + self.resource_info = { + "owner_id": "8ad942fe-490d-453e-bd37", + "default_mask": { + "data": "no-access", + "metadata": "no-access", + "admin": "not-admin", + }, + "group_masks": { + "7fa95d07-0e2d-4bc5-b47c-448fdc1260b2": { + "metadata": "edit", + "data": "edit", + }}, + "name": "_14329", + "data": { + "dataset": 1, + "trait": 14329, + }, + "type": "dataset-publish", + } conn = mock.MagicMock() - conn.hget.return_value = ( - '{"owner_id": "8ad942fe-490d-453e-bd37", ' - '"default_mask": {"data": "no-access", ' - '"metadata": "no-access", ' - '"admin": "not-admin"}, ' - '"group_masks": ' - '{"7fa95d07-0e2d-4bc5-b47c-448fdc1260b2": ' - '{"metadata": "edit", "data": "edit"}}, ' - '"name": "_14329", "' - 'data": {"dataset": 1, "trait": 14329}, ' - '"type": "dataset-publish"}') conn.hgetall.return_value = { '7fa95d07-0e2d-4bc5-b47c-448fdc1260b2': ( @@ -85,7 +93,7 @@ class TestCheckUserAccessRole(unittest.TestCase): """Test that the right access roles are set""" self.assertEqual(get_user_access_roles( conn=self.conn, - resource_id="", # Can be anything + resource_info=self.resource_info, user_id="8ad942fe-490d-453e-bd37"), {"data": DataRole.EDIT, "metadata": DataRole.EDIT, @@ -94,7 +102,7 @@ class TestCheckUserAccessRole(unittest.TestCase): def test_get_user_access_default_mask(self): self.assertEqual(get_user_access_roles( conn=self.conn, - resource_id="", # Can be anything + resource_info=self.resource_info, user_id=""), {"data": DataRole.NO_ACCESS, "metadata": DataRole.NO_ACCESS, @@ -103,7 +111,7 @@ class TestCheckUserAccessRole(unittest.TestCase): def test_get_user_access_group_mask(self): self.assertEqual(get_user_access_roles( conn=self.conn, - resource_id="", # Can be anything + resource_info=self.resource_info, user_id="8ad942fe-490d-453e-bd37-56f252e41603"), {"data": DataRole.EDIT, "metadata": DataRole.EDIT, diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index a3a94f9e..57b99296 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -66,16 +66,19 @@ def get_user_membership(conn: redis.Redis, user_id: str, def get_user_access_roles(conn: redis.Redis, - resource_id: str, + resource_info: Dict, user_id: str) -> Dict: """Get the highest access roles for a given user Args: - - conn: A redis connection with `decoded_responses == True`. - - resource_id: The unique id of a given resource. + - conn: A redis connection with the responses decoded. + - resource_info: A dict containing details(metadata) about a + given resource. + - user_id: The unique id of a given user. Returns: A dict indicating the highest access role the user has. + """ # This is the default access role access_role = { @@ -83,7 +86,6 @@ def get_user_access_roles(conn: redis.Redis, "metadata": [DataRole.NO_ACCESS], "admin": [AdminRole.NOT_ADMIN], } - resource_info = json.loads(conn.hget('resources', resource_id)) # Check the resource's default mask if default_mask := resource_info.get("default_mask"): -- cgit v1.2.3 From 8dd3457b20b5ce96cf7e0f5029e3541d57ca116d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 15 Oct 2021 15:47:21 +0300 Subject: Remove "utility.hmac.hmac_creation" which causes circular imports Hacky but re-implement `hmac_creation` as `create_hmac` --- wqflask/wqflask/decorators.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index c19e1aef..5930e7ec 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -1,26 +1,34 @@ """This module contains gn2 decorators""" -from flask import g +import hashlib +import hmac +from flask import current_app, g from typing import Dict from functools import wraps -from utility.hmac import hmac_creation -from utility.tools import GN_PROXY_URL import json import requests +def create_hmac(data: str, secret: str) -> str: + return hmac.new(bytearray(secret, "latin-1"), + bytearray(data, "utf-8"), + hashlib.sha1).hexdigest[:20] def edit_access_required(f): """Use this for endpoints where admins are required""" @wraps(f) def wrap(*args, **kwargs): resource_id: str = "" if kwargs.get("inbredset_id"): # data type: dataset-publish - resource_id = hmac_creation("dataset-publish:" - f"{kwargs.get('inbredset_id')}:" - f"{kwargs.get('name')}") + resource_id = create_hmac( + data=("dataset-publish:" + f"{kwargs.get('inbredset_id')}:" + f"{kwargs.get('name')}"), + secret=current_app.config.get("SECRET_HMAC_CODE")) if kwargs.get("dataset_name"): # data type: dataset-probe - resource_id = hmac_creation("dataset-probeset:" - f"{kwargs.get('dataset_name')}") + resource_id = create_hmac( + data=("dataset-probeset:" + f"{kwargs.get('dataset_name')}"), + secret=current_app.config.get("SECRET_HMAC_CODE")) response: Dict = {} try: _user_id = g.user_session.record.get(b"user_id", -- cgit v1.2.3 From 712318f7ab5e676b229ba0d479be09e9f92b9568 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 15 Oct 2021 15:48:46 +0300 Subject: decorators: Add `@login_required` decorator --- wqflask/wqflask/decorators.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index 5930e7ec..cc69902e 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -1,6 +1,8 @@ """This module contains gn2 decorators""" import hashlib import hmac +import redis + from flask import current_app, g from typing import Dict from functools import wraps @@ -13,6 +15,23 @@ def create_hmac(data: str, secret: str) -> str: return hmac.new(bytearray(secret, "latin-1"), bytearray(data, "utf-8"), hashlib.sha1).hexdigest[:20] + + +def login_required(f): + """Use this for endpoints where login is required""" + @wraps(f) + def wrap(*args, **kwargs): + user_id = (g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) + redis_conn = redis.from_url(current_app.config["REDIS_URL"], + decode_responses=True) + if not redis_conn.hget("users", user_id): + return "You need to be logged in!", 401 + return f(*args, **kwargs) + return wrap + + def edit_access_required(f): """Use this for endpoints where admins are required""" @wraps(f) -- cgit v1.2.3 From 217fc72eee688aed4409120d964cb2140ce11961 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 15 Oct 2021 15:53:07 +0300 Subject: utility.hmac: Label "hmac_creation" as deprecated This function is coupled to "wqflask.app", therefore requiring it's import at the module level. This may lead circular importation issues when working with blueprints. --- wqflask/utility/hmac.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wqflask/utility/hmac.py b/wqflask/utility/hmac.py index 29891677..d6e515ed 100644 --- a/wqflask/utility/hmac.py +++ b/wqflask/utility/hmac.py @@ -1,11 +1,14 @@ import hmac import hashlib +from deprecated import deprecated from flask import url_for from wqflask import app +@deprecated("This function leads to circular imports. " + "If possible use wqflask.decorators.create_hmac instead.") def hmac_creation(stringy): """Helper function to create the actual hmac""" -- cgit v1.2.3 From 13c5727b90d2b5e20e69a334c1cb78fd258cf3e8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 18 Oct 2021 09:45:49 +0300 Subject: manage_resource.html: Remove "flash" --- wqflask/wqflask/templates/admin/manage_resource.html | 1 - 1 file changed, 1 deletion(-) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 33a37594..f69e6799 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -7,7 +7,6 @@ {% block content %}
    - {{ flash_me() }}

    Resource Manager

    {% if owner_name is not none %}Current Owner: {{ owner_name }}{% endif %} {% if admin_status == "owner" %}{% endif %}

    -- cgit v1.2.3 From 56e86ccd4fb4c88b0396fce9f926bafa4517766b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 18 Oct 2021 10:14:39 +0300 Subject: manage_resource.html: If owner owns a resource display their name --- .../wqflask/templates/admin/manage_resource.html | 23 +++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index f69e6799..0f4abec9 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -7,9 +7,30 @@ {% block content %}
    + {% set DATA_ACCESS = access_role.get('data') %} + {% set METADATA_ACCESS = access_role.get('metadata') %} + {% set ADMIN_STATUS = access_role.get('admin') %} +

    Resource Manager

    -

    {% if owner_name is not none %}Current Owner: {{ owner_name }}{% endif %} {% if admin_status == "owner" %}{% endif %}

    +

    + {% if resource_info.get('owner_id') != 'none'%} + {% set user_details = resource_info.get('owner_details') %} + Current Owner: {{ user_details.get('full_name') }} + {% if user_details.get('organization') %} +
    Organization {{ user_details.get('organization')}} + {% endif %} + {% endif %} + + {% if DATA_ACCESS > DataRole.VIEW and ADMIN_STATUS > AdminRole.NOT_ADMIN %} + + {% endif %} +

    -- cgit v1.2.3 From 46645a00902d78b7a0af867e70a1ee0fc1e46e97 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 18 Oct 2021 11:45:35 +0300 Subject: manage_resource.html: Display group name next to it's unique id --- wqflask/wqflask/templates/admin/manage_resource.html | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 0f4abec9..91e23e56 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -40,19 +40,19 @@
    - {{ resource_info.name }} + {{ resource_info.get('name') }}
    - {% if admin_status == "owner" %} + {% if DATA_ACCESS > DataRole.VIEW and ADMIN_STATUS > AdminRole.NOT_ADMIN %}
    @@ -67,17 +67,18 @@
    - {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} + {% if ADMIN_STATUS > AdminRole.NOT_ADMIN %}

    - {% if group_masks|length > 0 %} + {% if resource_info.get('group_masks', [])|length > 0 %}

    Current Group Permissions


    • Loading...
    + @@ -85,9 +86,10 @@ - {% for key, value in group_masks.items() %} + {% for key, value in resource_info.get('group_masks').items() %} + -- cgit v1.2.3 From 997fb64394a7fa49d154820bbfdafcdfcc05f32e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 18 Oct 2021 11:52:30 +0300 Subject: Remove dead line --- wqflask/wqflask/resource_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 57b99296..03b45822 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -113,4 +113,3 @@ def get_user_access_roles(conn: redis.Redis, access_role["metadata"].append( DataRole(roles.get("metadata"))) return {k: max(v) for k, v in access_role.items()} - admin_status = check_owner_or_admin(resource_id=resource_id) -- cgit v1.2.3 From 8eedaab544d5bd1ab667bdd88d055b671bbee1d0 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 18 Oct 2021 11:52:55 +0300 Subject: wqflask: resource_manager: Add `add_extra_resource_metadata` method --- wqflask/wqflask/resource_manager.py | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 03b45822..16bab250 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -113,3 +113,42 @@ def get_user_access_roles(conn: redis.Redis, access_role["metadata"].append( DataRole(roles.get("metadata"))) return {k: max(v) for k, v in access_role.items()} + + +def add_extra_resource_metadata(conn: redis.Redis, resource: Dict) -> Dict: + """If resource['owner_id'] exists, add metadata about that user. Also, +if the resource contains group masks, add the group name into the +resource dict. Note that resource['owner_id'] and the group masks are +unique identifiers so they aren't human readable names. + + Args: + - conn: A redis connection with the responses decoded. + - resource: A dict containing details(metadata) about a + given resource. + + Returns: + An embellished dictionary with the human readable names of the + group masks and the owner id if it was set. + + """ + # Embellish the resource information with owner details if the + # owner is set + if (owner_id := resource.get("owner_id", "none").lower()) == "none": + resource["owner_id"] = None + resource["owner_details"] = None + else: + user_details = json.loads(conn.hget("users", owner_id)) + resource["owner_details"] = { + "email_address": user_details.get("email_address"), + "full_name": user_details.get("full_name"), + "organization": user_details.get("organization"), + } + + # Embellish the resources information with the group name if the + # group masks are present + if groups := resource.get('group_masks', {}): + for group_id in groups.keys(): + resource['group_masks'][group_id]["group_name"] = ( + json.loads(conn.hget("groups", group_id)).get('name')) + return resource + -- cgit v1.2.3 From ff2813b8c0522d780ffc8bc44692a749de9f45e6 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 18 Oct 2021 11:54:37 +0300 Subject: Add "GET /resource-management/resources/" --- wqflask/wqflask/__init__.py | 5 +++++ wqflask/wqflask/resource_manager.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index 5758b13e..5b2d05d1 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -8,6 +8,9 @@ from flask import Flask from typing import Tuple from urllib.parse import urlparse from utility import formatting + +from wqflask.resource_manager import resource_management + from wqflask.api.markdown import glossary_blueprint from wqflask.api.markdown import references_blueprint from wqflask.api.markdown import links_blueprint @@ -55,6 +58,8 @@ app.register_blueprint(facilities_blueprint, url_prefix="/facilities") app.register_blueprint(blogs_blueprint, url_prefix="/blogs") app.register_blueprint(news_blueprint, url_prefix="/news") +app.register_blueprint(resource_management, url_prefix="/resource-management") + @app.before_request def before_request(): diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 16bab250..9665ebb0 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -4,10 +4,15 @@ import functools from enum import Enum, unique - +from flask import Blueprint +from flask import current_app +from flask import g +from flask import render_template from typing import Dict +from wqflask.decorators import login_required + @functools.total_ordering class OrderedEnum(Enum): @@ -36,6 +41,10 @@ class AdminRole(OrderedEnum): EDIT_ACCESS = "edit-access" EDIT_ADMINS = "edit-admins" + +resource_management = Blueprint('resource_management', __name__) + + def get_user_membership(conn: redis.Redis, user_id: str, group_id: str) -> Dict: """Return a dictionary that indicates whether the `user_id` is a @@ -152,3 +161,28 @@ unique identifiers so they aren't human readable names. json.loads(conn.hget("groups", group_id)).get('name')) return resource + +@resource_management.route("/resources/") +@login_required +def manage_resource(resource_id: str): + user_id = (g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) + redis_conn = redis.from_url( + current_app.config["REDIS_URL"], + decode_responses=True) + + # Abort early if the resource can't be found + if not (resource := redis_conn.hget("resources", resource_id)): + return f"Resource: {resource_id} Not Found!", 401 + + return render_template( + "admin/manage_resource.html", + resource_info=(embellished_resource:=add_extra_resource_metadata( + conn=redis_conn, + resource=json.loads(resource))), + access_role=get_user_access_roles( + conn=redis_conn, + resource_info=embellished_resource, + user_id=user_id), + DataRole=DataRole, AdminRole=AdminRole) -- cgit v1.2.3 From 8ba81937fec81c7e9790db422a52d99f4234a194 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 18 Oct 2021 12:02:51 +0300 Subject: authentication_tools: Mark `check_owner_or_admin` as deprecated Use the new auth proxy tools instead. --- wqflask/utility/authentication_tools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py index afea69e1..c4801c8c 100644 --- a/wqflask/utility/authentication_tools.py +++ b/wqflask/utility/authentication_tools.py @@ -1,6 +1,7 @@ import json import requests +from deprecated import deprecated from flask import g from base import webqtlConfig @@ -126,6 +127,7 @@ def check_owner(dataset=None, trait_id=None, resource_id=None): return False +@deprecated def check_owner_or_admin(dataset=None, trait_id=None, resource_id=None): if not resource_id: if dataset.type == "Temp": -- cgit v1.2.3 From 9f43e0231698f8fcd472ad736b4276c6ad6eeb6d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 18 Oct 2021 12:08:45 +0300 Subject: manage_resource.html: Remove unused stylesheets --- wqflask/wqflask/templates/admin/manage_resource.html | 4 ---- 1 file changed, 4 deletions(-) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 91e23e56..f4bcc163 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -1,9 +1,5 @@ {% extends "base.html" %} {% block title %}Resource Manager{% endblock %} -{% block css %} - - -{% endblock %} {% block content %}
    -- cgit v1.2.3 From 948a0b7e3572c39aefd1b18083afc174dcf4a164 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 18 Oct 2021 12:09:05 +0300 Subject: manage_resource: Add missing bootstrap "table" class --- wqflask/wqflask/templates/admin/manage_resource.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index f4bcc163..a970679d 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -71,7 +71,7 @@ {% if resource_info.get('group_masks', [])|length > 0 %}

    Current Group Permissions

    -
    Id Name Data Metadata
    {{ value.name }}{{ value.group_name}} {{ value.data }} {{ value.metadata }} {{ value.admin }}
    +
    -- cgit v1.2.3 From f3c17102d2d8fc6be35be237c299a22f0e16a920 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 18 Oct 2021 12:09:32 +0300 Subject: manage_resource: Remove column "Admin" from group permissions table A group does not have "admin" privileges. Instead, a user can have admin privileges over a particular group. --- wqflask/wqflask/templates/admin/manage_resource.html | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index a970679d..a2ed3538 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -78,17 +78,15 @@ - {% for key, value in resource_info.get('group_masks').items() %} - + - {% endfor %} @@ -101,12 +99,9 @@ - - {% endblock %} - {% block js %} -- cgit v1.2.3 From 198add36d970c005320700a0ed01e8632e8f6922 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 10:33:03 +0300 Subject: manage_resources.html: Clean up html --- .../wqflask/templates/admin/manage_resource.html | 33 +++++++++------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index a2ed3538..80dfc8ee 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -6,29 +6,24 @@ {% set DATA_ACCESS = access_role.get('data') %} {% set METADATA_ACCESS = access_role.get('metadata') %} {% set ADMIN_STATUS = access_role.get('admin') %} - -

    Resource Manager

    + {% if resource_info.get('owner_id') != 'none'%} + {% set user_details = resource_info.get('owner_details') %}

    - {% if resource_info.get('owner_id') != 'none'%} - {% set user_details = resource_info.get('owner_details') %} Current Owner: {{ user_details.get('full_name') }} - {% if user_details.get('organization') %} -
    Organization {{ user_details.get('organization')}} - {% endif %} - {% endif %} - - {% if DATA_ACCESS > DataRole.VIEW and ADMIN_STATUS > AdminRole.NOT_ADMIN %} - - {% endif %}

    -
    - + {% if user_details.get('organization') %} +

    + Organization: {{ user_details.get('organization')}} +

    + {% endif %} + {% if DATA_ACCESS > DataRole.VIEW and ADMIN_STATUS > AdminRole.NOT_ADMIN %} + + {% endif %} + {% endif %}
    -- cgit v1.2.3 From 150a527924f071a06bafc835e03b12eaa12b5768 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 11:05:17 +0300 Subject: wqflask: decorators: Fix type when calling hexdigest() --- wqflask/wqflask/decorators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index cc69902e..b1141fb7 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -14,7 +14,7 @@ import requests def create_hmac(data: str, secret: str) -> str: return hmac.new(bytearray(secret, "latin-1"), bytearray(data, "utf-8"), - hashlib.sha1).hexdigest[:20] + hashlib.sha1).hexdigest()[:20] def login_required(f): -- cgit v1.2.3 From 6bdf173ccb79e2e21cfc4e0d96377dae3bd1e6c1 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 14:55:03 +0300 Subject: scripts: group.py: Add shebang --- scripts/authentication/group.py | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 scripts/authentication/group.py diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py old mode 100644 new mode 100755 index c8c2caad..460a6329 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """A script for adding users to a specific group. Example: -- cgit v1.2.3 From 467723891d7c153c365322ea5812a714186a478d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 14:57:11 +0300 Subject: scripts: group.py: Add extra optional arg to specify uid --- scripts/authentication/group.py | 118 ++++++++++++++++++++++++---------------- 1 file changed, 71 insertions(+), 47 deletions(-) diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index 460a6329..08652454 100755 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -7,22 +7,40 @@ Assuming there are no groups and 'test@bonfacemunyoki.com' does not exist in Redis: .. code-block:: bash - python group.py -g "editors" -m "test@bonfacemunyoki.com" + python group.py -n "editors" \ + -m "me@bonfacemunyoki.com" results in:: - Successfully created the group: 'editors' - Data: '{"admins": [], "members": []}' + Successfully created the group: 'editors' + `HGET groups 0360449f-5a96-4940-8522-b22d62085da9`: {'name': 'editors', 'admins': [], 'members': ['8ad942fe-490d-453e-bd37-56f252e41603'], 'changed_timestamp': 'Oct 19 2021 09:34AM', 'created_timestamp': 'Oct 19 2021 09:34AM'} + -If 'me@bonfacemunyoki.com' exists in 'users' in Redis and we run: +Assuming we have a group's unique id: .. code-block:: bash - python group.py -g "editors" -m "me@bonfacemunyoki.com" + python group.py -n "editors" \ + -m "me@bonfacemunyoki.com" \ + -g "8ad942fe-490d-453e-bd37-56f252e41603" now results in:: - No new group was created. - Updated Data: {'admins': [], 'members': ['me@bonfacemunyoki.com']} + Successfully created the group: 'editors' + `HGET groups 8ad942fe-490d-453e-bd37-56f252e41603`: {'name': 'editors', 'admins': [], 'members': ['8ad942fe-490d-453e-bd37-56f252e41603'], 'changed_timestamp': 'Oct 19 2021 09:38AM', 'created_timestamp': 'Oct 19 2021 09:38AM'} + +If 'me@bonfacemunyoki.com' exists in 'users' in Redis for the above +command and we run: + +.. code-block:: bash + python group.py -n "editors" \ + -m "me@bonfacemunyoki.com" \ + -g "8ad942fe-490d-453e-bd37-56f252e41603" + +now results in:: + + No new group was created. + `HGET groups 8ad942fe-490d-453e-bd37-56f252e41603`: {'name': 'editors', 'admins': [], 'members': ['8ad942fe-490d-453e-bd37-56f252e41603'], 'changed_timestamp': 'Oct 19 2021 09:40AM'} + """ @@ -37,44 +55,48 @@ from typing import Dict, Optional, Set def create_group_data(users: Dict, target_group: str, members: Optional[str] = None, - admins: Optional[str] = None) -> Dict: - """Return a dictionary that contains the following keys: "key", - "field", and "value" that can be used in a redis hash as follows: - HSET key field value + admins: Optional[str] = None, + group_id: Optional[str] = None) -> Dict: + """Create group data which is isomorphic to a redis HSET i.e.: KEY, + FIELD and VALUE. If the group_id is not specified, a unique hash + will be generated. The "field" return value is a unique-id that is used to distinguish the groups. - Parameters: - - - `users`: a list of users for example: - - {'8ad942fe-490d-453e-bd37-56f252e41603': - '{"email_address": "me@test.com", - "full_name": "John Doe", - "organization": "Genenetwork", - "password": {"algorithm": "pbkdf2", - "hashfunc": "sha256", - "salt": "gJrd1HnPSSCmzB5veMPaVk2ozzDlS1Z7Ggcyl1+pciA=", - "iterations": 100000, "keylength": 32, - "created_timestamp": "2021-09-22T11:32:44.971912", - "password": "edcdaa60e84526c6"}, - "user_id": "8ad942fe", "confirmed": 1, - "registration_info": { - "timestamp": "2021-09-22T11:32:45.028833", - "ip_address": "127.0.0.1", - "user_agent": "Mozilla/5.0"}}'} - - - `target_group`: the group name that will be stored inside the - "groups" hash in Redis. - - - `members`: a comma-separated list of values that contain members - of the `target_group` e.g. "me@test1.com, me@test2.com, - me@test3.com" - - - `admins`: a comma-separated list of values that contain - administrators of the `target_group` e.g. "me@test1.com, - me@test2.com, me@test3.com" + Args: + - users: a list of users for example: + {'8ad942fe-490d-453e-bd37-56f252e41603': + '{"email_address": "me@test.com", + "full_name": "John Doe", + "organization": "Genenetwork", + "password": {"algorithm": "pbkdf2", + "hashfunc": "sha256", + "salt": "gJrd1HnPSSCmzB5veMPaVk2ozzDlS1Z7Ggcyl1+pciA=", + "iterations": 100000, "keylength": 32, + "created_timestamp": "2021-09-22T11:32:44.971912", + "password": "edcdaa60e84526c6"}, + "user_id": "8ad942fe", "confirmed": 1, + "registration_info": { + "timestamp": "2021-09-22T11:32:45.028833", + "ip_address": "127.0.0.1", + "user_agent": "Mozilla/5.0"}}'} + + - target_group: the group name that will be stored inside the + "groups" hash in Redis. + - members: an optional comma-separated list of values that + contain members of the `target_group` e.g. "me@test1.com, + me@test2.com, me@test3.com" + - admins: an optional comma-separated list of values that + contain administrators of the `target_group` + e.g. "me@test1.com, me@test2.com, me@test3.com" + - group_id: an optional unique identifier for a group. If not + set, a unique value will be auto-generated. + + Returns: + A dictionary that contains the following keys: "key", "field", + and "value" that can be used in a redis hash as follows: HSET key + field value """ # Emails @@ -96,7 +118,7 @@ def create_group_data(users: Dict, target_group: str, timestamp: str = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') return {"key": "groups", - "field": str(uuid.uuid4()), + "field": (group_id or str(uuid.uuid4())), "value": json.dumps({ "name": target_group, "admins": list(admin_ids), @@ -108,8 +130,10 @@ def create_group_data(users: Dict, target_group: str, if __name__ == "__main__": # Initialising the parser CLI arguments parser = argparse.ArgumentParser() - parser.add_argument("-g", "--group-name", + parser.add_argument("-n", "--group-name", help="This is the name of the GROUP mask") + parser.add_argument("-g", "--group-id", + help="[Optional] This is the name of the GROUP mask") parser.add_argument("-m", "--members", help="Members of the GROUP mask") parser.add_argument("-a", "--admins", @@ -133,7 +157,8 @@ if __name__ == "__main__": users=USERS, target_group=args.group_name, members=members, - admins=admins) + admins=admins, + group_id=args.group_id) if not REDIS_CONN.hget("groups", data.get("field")): updated_data = json.loads(data["value"]) @@ -144,11 +169,10 @@ if __name__ == "__main__": created_p = REDIS_CONN.hset(data.get("key", ""), data.get("field", ""), data.get("value", "")) - groups = json.loads(REDIS_CONN.hget("groups", data.get("field"))) # type: ignore if created_p: exit(f"\nSuccessfully created the group: '{args.group_name}'\n" - f"`HGETALL groups {args.group_name}`: {groups}\n") + f"`HGET groups {data.get('field')}`: {groups}\n") exit("\nNo new group was created.\n" - f"`HGETALL groups {args.group_name}`: {groups}\n") + f"`HGET groups {data.get('field')}`: {groups}\n") -- cgit v1.2.3 From 5d6559cdc0f9251a90b351bf440b930a000f8354 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 16:17:32 +0300 Subject: scripts: resource.py: Add shebang --- scripts/authentication/resource.py | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 scripts/authentication/resource.py diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py old mode 100644 new mode 100755 index 1a2bcd8a..1f5f31d8 --- a/scripts/authentication/resource.py +++ b/scripts/authentication/resource.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """A script that: - Optionally restores data from a json file. -- cgit v1.2.3 From 3c074638ca0b5e175e2dbb93aaf3bf21e8af8151 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 16:39:32 +0300 Subject: Move access_roles enums to it's own module --- wqflask/wqflask/access_roles.py | 30 ++++++++++++++++++++++++++++++ wqflask/wqflask/resource_manager.py | 32 ++------------------------------ 2 files changed, 32 insertions(+), 30 deletions(-) create mode 100644 wqflask/wqflask/access_roles.py diff --git a/wqflask/wqflask/access_roles.py b/wqflask/wqflask/access_roles.py new file mode 100644 index 00000000..6cffbc81 --- /dev/null +++ b/wqflask/wqflask/access_roles.py @@ -0,0 +1,30 @@ +import functools +from enum import Enum, unique + + +@functools.total_ordering +class OrderedEnum(Enum): + @classmethod + @functools.lru_cache(None) + def _member_list(cls): + return list(cls) + + def __lt__(self, other): + if self.__class__ is other.__class__: + member_list = self.__class__._member_list() + return member_list.index(self) < member_list.index(other) + return NotImplemented + + +@unique +class DataRole(OrderedEnum): + NO_ACCESS = "no-access" + VIEW = "view" + EDIT = "edit" + + +@unique +class AdminRole(OrderedEnum): + NOT_ADMIN = "not-admin" + EDIT_ACCESS = "edit-access" + EDIT_ADMINS = "edit-admins" diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 9665ebb0..4e160bb8 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -1,8 +1,6 @@ import redis import json -import functools -from enum import Enum, unique from flask import Blueprint from flask import current_app @@ -12,34 +10,8 @@ from flask import render_template from typing import Dict from wqflask.decorators import login_required - - -@functools.total_ordering -class OrderedEnum(Enum): - @classmethod - @functools.lru_cache(None) - def _member_list(cls): - return list(cls) - - def __lt__(self, other): - if self.__class__ is other.__class__: - member_list = self.__class__._member_list() - return member_list.index(self) < member_list.index(other) - return NotImplemented - - -@unique -class DataRole(OrderedEnum): - NO_ACCESS = "no-access" - VIEW = "view" - EDIT = "edit" - - -@unique -class AdminRole(OrderedEnum): - NOT_ADMIN = "not-admin" - EDIT_ACCESS = "edit-access" - EDIT_ADMINS = "edit-admins" +from wqflask.access_roles import AdminRole +from wqflask.access_roles import DataRole resource_management = Blueprint('resource_management', __name__) -- cgit v1.2.3 From c7899f6d55a3063a99e19e402ae753030a2ecff8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 16:40:57 +0300 Subject: wqflask: decorators: Make `edit_access_required` more generic This now supports passing the `resource_id` directly. --- wqflask/wqflask/decorators.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index b1141fb7..13867cda 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -6,6 +6,7 @@ import redis from flask import current_app, g from typing import Dict from functools import wraps +from wqflask.access_roles import DataRole import json import requests @@ -33,7 +34,7 @@ def login_required(f): def edit_access_required(f): - """Use this for endpoints where admins are required""" + """Use this for endpoints where people with admin or edit privileges are required""" @wraps(f) def wrap(*args, **kwargs): resource_id: str = "" @@ -48,6 +49,8 @@ def edit_access_required(f): data=("dataset-probeset:" f"{kwargs.get('dataset_name')}"), secret=current_app.config.get("SECRET_HMAC_CODE")) + if kwargs.get("resource_id"): # The resource_id is already provided + resource_id = kwargs.get("resource_id") response: Dict = {} try: _user_id = g.user_session.record.get(b"user_id", @@ -57,8 +60,8 @@ def edit_access_required(f): f"{resource_id}&user={_user_id}").content) except: response = {} - - if "edit" not in response.get("data", []): - return "You need to be admin", 401 + if max([DataRole(role) for role in response.get( + "data", ["no-access"])]) < DataRole.EDIT: + return "You need to have edit access", 401 return f(*args, **kwargs) return wrap -- cgit v1.2.3 From 400649f6bb8d0b11e51aa1fec00d82972a785723 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 16:42:59 +0300 Subject: manage_resource.html: Toggle the "is open to public" radio buttons --- wqflask/wqflask/templates/admin/manage_resource.html | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 80dfc8ee..1002ac45 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -24,8 +24,12 @@ {% endif %} {% endif %} - -
    + + +
    + + +
    @@ -35,15 +39,16 @@
    {% if DATA_ACCESS > DataRole.VIEW and ADMIN_STATUS > AdminRole.NOT_ADMIN %} + {% set is_open_to_public = DataRole(resource_info.get('default_mask').get('data')) > DataRole.NO_ACCESS %}
    @@ -51,7 +56,7 @@
    - +
    {% endif %} @@ -92,7 +97,7 @@
    {% endif %} -
    +
    -- cgit v1.2.3 From 803f4c47b070ac541e9d12c76101c8c1ac267640 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 16:43:46 +0300 Subject: Rename manage_resource -> view_resource --- wqflask/wqflask/resource_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 4e160bb8..e0d2105f 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -136,7 +136,7 @@ unique identifiers so they aren't human readable names. @resource_management.route("/resources/") @login_required -def manage_resource(resource_id: str): +def view_resource(resource_id: str): user_id = (g.user_session.record.get(b"user_id", b"").decode("utf-8") or g.user_session.record.get("user_id", "")) -- cgit v1.2.3 From a5d24f3885239c4fe15fd6d5dd070ec28403473d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 16:44:35 +0300 Subject: resource_manager: Use := to store embellished resource dict --- wqflask/wqflask/resource_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index e0d2105f..94c9ba4f 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -150,8 +150,9 @@ def view_resource(resource_id: str): return render_template( "admin/manage_resource.html", - resource_info=(embellished_resource:=add_extra_resource_metadata( + resource_info=(embellished_resource := add_extra_resource_metadata( conn=redis_conn, + resource_id=resource_id, resource=json.loads(resource))), access_role=get_user_access_roles( conn=redis_conn, -- cgit v1.2.3 From 6fed3eec407f9457260fb586067d76a35318445c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 16:45:26 +0300 Subject: wqflask: resource_manager: Add "resource_id" to resource dict --- wqflask/wqflask/resource_manager.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 94c9ba4f..94c351a4 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -96,7 +96,9 @@ def get_user_access_roles(conn: redis.Redis, return {k: max(v) for k, v in access_role.items()} -def add_extra_resource_metadata(conn: redis.Redis, resource: Dict) -> Dict: +def add_extra_resource_metadata(conn: redis.Redis, + resource_id: str, + resource: Dict) -> Dict: """If resource['owner_id'] exists, add metadata about that user. Also, if the resource contains group masks, add the group name into the resource dict. Note that resource['owner_id'] and the group masks are @@ -104,14 +106,17 @@ unique identifiers so they aren't human readable names. Args: - conn: A redis connection with the responses decoded. + - resource_id: The unique identifier of the resource. - resource: A dict containing details(metadata) about a given resource. Returns: - An embellished dictionary with the human readable names of the - group masks and the owner id if it was set. + An embellished dictionary with its resource id; the human + readable names of the group masks; and the owner id if it was set. """ + resource["resource_id"] = resource_id + # Embellish the resource information with owner details if the # owner is set if (owner_id := resource.get("owner_id", "none").lower()) == "none": -- cgit v1.2.3 From 666c0df4c6536b831a2c08ea61c87de8f37a696d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 16:46:09 +0300 Subject: Add `POST resource-management/resources//make-public` This endpoint either makes resources public or non-public by tweaking the access-masks. --- wqflask/wqflask/resource_manager.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 94c351a4..fe25902e 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -6,9 +6,13 @@ from flask import Blueprint from flask import current_app from flask import g from flask import render_template +from flask import redirect +from flask import request +from flask import url_for from typing import Dict +from wqflask.decorators import edit_access_required from wqflask.decorators import login_required from wqflask.access_roles import AdminRole from wqflask.access_roles import DataRole @@ -164,3 +168,33 @@ def view_resource(resource_id: str): resource_info=embellished_resource, user_id=user_id), DataRole=DataRole, AdminRole=AdminRole) + + +@resource_management.route("/resources//make-public", + methods=('POST',)) +@edit_access_required +@login_required +def update_resource_publicity(resource_id: str): + redis_conn = redis.from_url( + current_app.config["REDIS_URL"], + decode_responses=True) + resource_info = json.loads(redis_conn.hget("resources", resource_id)) + + if (is_open_to_public := request + .form + .to_dict() + .get("open_to_public")) == "True": + resource_info['default_mask'] = { + 'data': DataRole.VIEW.value, + 'admin': AdminRole.NOT_ADMIN.value, + 'metadata': DataRole.VIEW.value, + } + elif is_open_to_public == "False": + resource_info['default_mask'] = { + 'data': DataRole.NO_ACCESS.value, + 'admin': AdminRole.NOT_ADMIN.value, + 'metadata': DataRole.NO_ACCESS.value, + } + redis_conn.hset("resources", resource_id, json.dumps(resource_info)) + return redirect(url_for("resource_management.view_resource", + resource_id=resource_id)) -- cgit v1.2.3 From 182327254fe2964c3dd41aabe49ab99748800a64 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 16:47:38 +0300 Subject: scripts: resource: Add value for "admin" when updating resources --- scripts/authentication/resource.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py index 1f5f31d8..4b8d801a 100755 --- a/scripts/authentication/resource.py +++ b/scripts/authentication/resource.py @@ -38,14 +38,14 @@ from datetime import datetime def recover_hash(name: str, file_path: str, set_function) -> bool: """Recover back-ups using the `set_function` - Parameters: + Args: + - name: Redis hash where `file_path` will be restored + - file_path: File path where redis hash is sourced from + - set_function: Function used to do the Redis backup for + example: HSET - - `name`: Redis hash where `file_path` will be restored - - - `file_path`: File path where redis hash is sourced from - - - `set_function`: Function used to do the Redis backup for - example: HSET + Returns: + A boolean indicating whether the function ran successfully. """ try: @@ -99,7 +99,7 @@ if __name__ == "__main__": _resource = json.loads(resource) # str -> dict conversion _resource["group_masks"] = {args.group_id: {"metadata": "edit", "data": "edit", - "admin": "edit-admins"}} + "admin": "not-admin"}} REDIS_CONN.hset("resources", resource_id, json.dumps(_resource)) -- cgit v1.2.3 From cc3ae4707d2418712d13261d4bf9d5a509169c7e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 20 Oct 2021 12:28:44 +0300 Subject: Add "GN2_PROXY" as a configurable option --- etc/default_settings.py | 3 +++ wqflask/wqflask/decorators.py | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/etc/default_settings.py b/etc/default_settings.py index 651cc55e..8636f4db 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -29,6 +29,9 @@ GN_VERSION = open("../etc/VERSION", "r").read() # Redis REDIS_URL = "redis://:@localhost:6379/0" +# gn2-proxy +GN2_PROXY = "http://localhost:8080" + # ---- MySQL SQL_URI = "mysql://gn2:mysql_password@localhost/db_webqtl_s" diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index 13867cda..edbea90f 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -5,6 +5,7 @@ import redis from flask import current_app, g from typing import Dict +from urllib.parse import urljoin from functools import wraps from wqflask.access_roles import DataRole @@ -56,8 +57,10 @@ def edit_access_required(f): _user_id = g.user_session.record.get(b"user_id", "").decode("utf-8") response = json.loads( - requests.get(GN_PROXY_URL + "available?resource=" - f"{resource_id}&user={_user_id}").content) + requests.get(urljoin( + current_app.config.get("GN2_PROXY"), + ("available?resource=" + f"{resource_id}&user={_user_id}"))).content) except: response = {} if max([DataRole(role) for role in response.get( -- cgit v1.2.3 From 050391c297f35fa4073d3360de47b889a39f0829 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 20 Oct 2021 12:29:08 +0300 Subject: Add `edit_admins_required` decorator --- wqflask/wqflask/decorators.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index edbea90f..cd06aee7 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -7,6 +7,7 @@ from flask import current_app, g from typing import Dict from urllib.parse import urljoin from functools import wraps +from wqflask.access_roles import AdminRole from wqflask.access_roles import DataRole import json @@ -68,3 +69,27 @@ def edit_access_required(f): return "You need to have edit access", 401 return f(*args, **kwargs) return wrap + + +def edit_admins_access_required(f): + """Use this for endpoints where ownership of a resource is required""" + @wraps(f) + def wrap(*args, **kwargs): + resource_id: str = kwargs.get("resource_id", "") + response: Dict = {} + try: + _user_id = g.user_session.record.get(b"user_id", + "").decode("utf-8") + response = json.loads( + requests.get(urljoin( + current_app.config.get("GN2_PROXY"), + ("available?resource=" + f"{resource_id}&user={_user_id}"))).content) + except: + response = {} + if max([AdminRole(role) for role in response.get( + "data", ["not-admin"])]) >= AdminRole.EDIT_ADMINS: + return "You need to have edit-admins access", 401 + return f(*args, **kwargs) + return wrap + -- cgit v1.2.3 From c76971e2eaa556a97481974c73f964c2b341e723 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 20 Oct 2021 13:43:05 +0300 Subject: Use gn-proxy to fetch access-roles instead of looking at redis --- wqflask/wqflask/resource_manager.py | 71 +++++++++++++------------------------ 1 file changed, 25 insertions(+), 46 deletions(-) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index fe25902e..74941341 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -1,6 +1,6 @@ import redis import json - +import requests from flask import Blueprint from flask import current_app @@ -10,6 +10,7 @@ from flask import redirect from flask import request from flask import url_for +from urllib.parse import urljoin from typing import Dict from wqflask.decorators import edit_access_required @@ -50,54 +51,33 @@ def get_user_membership(conn: redis.Redis, user_id: str, return results -def get_user_access_roles(conn: redis.Redis, - resource_info: Dict, - user_id: str) -> Dict: +def get_user_access_roles( + resource_id: str, + user_id: str, + gn_proxy_url: str="http://localhost:8080") -> Dict: """Get the highest access roles for a given user Args: - - conn: A redis connection with the responses decoded. - - resource_info: A dict containing details(metadata) about a - given resource. + - resource_id: The unique id of a given resource. - user_id: The unique id of a given user. + - gn_proxy_url: The URL where gn-proxy is running. Returns: A dict indicating the highest access role the user has. """ - # This is the default access role - access_role = { - "data": [DataRole.NO_ACCESS], - "metadata": [DataRole.NO_ACCESS], - "admin": [AdminRole.NOT_ADMIN], - } - - # Check the resource's default mask - if default_mask := resource_info.get("default_mask"): - access_role["data"].append(DataRole(default_mask.get("data"))) - access_role["metadata"].append(DataRole(default_mask.get("metadata"))) - access_role["admin"].append(AdminRole(default_mask.get("admin"))) - - # Then check if the user is the owner Check with Zach and Rob if - # the owner, be default should, as the lowest access_roles, edit - # access - if resource_info.get("owner_id") == user_id: - access_role["data"].append(DataRole.EDIT) - access_role["metadata"].append(DataRole.EDIT) - access_role["admin"].append(AdminRole.EDIT_ACCESS) - - # Check the group mask. If the user is in that group mask, use the - # access roles for that group - if group_masks := resource_info.get("group_masks"): - for group_id, roles in group_masks.items(): - user_membership = get_user_membership(conn=conn, - user_id=user_id, - group_id=group_id) - if any(user_membership.values()): - access_role["data"].append(DataRole(roles.get("data"))) - access_role["metadata"].append( - DataRole(roles.get("metadata"))) - return {k: max(v) for k, v in access_role.items()} + role_mapping = {} + for x, y in zip(DataRole, AdminRole): + role_mapping.update({x.value: x, }) + role_mapping.update({y.value: y, }) + access_role = {} + for key, value in json.loads( + requests.get(urljoin( + gn_proxy_url, + ("available?resource=" + f"{resource_id}&user={user_id}"))).content).items(): + access_role[key] = max(map(lambda x: role_mapping[x], value)) + return access_role def add_extra_resource_metadata(conn: redis.Redis, @@ -152,21 +132,20 @@ def view_resource(resource_id: str): redis_conn = redis.from_url( current_app.config["REDIS_URL"], decode_responses=True) - # Abort early if the resource can't be found if not (resource := redis_conn.hget("resources", resource_id)): return f"Resource: {resource_id} Not Found!", 401 return render_template( "admin/manage_resource.html", - resource_info=(embellished_resource := add_extra_resource_metadata( + resource_info=(add_extra_resource_metadata( conn=redis_conn, resource_id=resource_id, resource=json.loads(resource))), access_role=get_user_access_roles( - conn=redis_conn, - resource_info=embellished_resource, - user_id=user_id), + resource_id=resource_id, + user_id=user_id, + gn_proxy_url=current_app.config.get("GN2_PROXY")), DataRole=DataRole, AdminRole=AdminRole) @@ -183,7 +162,7 @@ def update_resource_publicity(resource_id: str): if (is_open_to_public := request .form .to_dict() - .get("open_to_public")) == "True": + .get("open_to_public")) == "True": resource_info['default_mask'] = { 'data': DataRole.VIEW.value, 'admin': AdminRole.NOT_ADMIN.value, -- cgit v1.2.3 From fbffb2e4f7538cc34332f37064d4b67cd52588fb Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 20 Oct 2021 13:43:57 +0300 Subject: Update failing tests when testing `get_user_access_roles` --- .../tests/unit/wqflask/test_resource_manager.py | 82 ++++++++-------------- 1 file changed, 29 insertions(+), 53 deletions(-) diff --git a/wqflask/tests/unit/wqflask/test_resource_manager.py b/wqflask/tests/unit/wqflask/test_resource_manager.py index 9d5aaf0b..01fb2021 100644 --- a/wqflask/tests/unit/wqflask/test_resource_manager.py +++ b/wqflask/tests/unit/wqflask/test_resource_manager.py @@ -1,4 +1,5 @@ """Test cases for wqflask/resource_manager.py""" +import json import unittest from unittest import mock @@ -57,62 +58,37 @@ class TestGetUserMembership(unittest.TestCase): class TestCheckUserAccessRole(unittest.TestCase): """Test cases for `get_user_access_roles`""" - def setUp(self): - self.resource_info = { - "owner_id": "8ad942fe-490d-453e-bd37", - "default_mask": { - "data": "no-access", - "metadata": "no-access", - "admin": "not-admin", - }, - "group_masks": { - "7fa95d07-0e2d-4bc5-b47c-448fdc1260b2": { - "metadata": "edit", - "data": "edit", - }}, - "name": "_14329", - "data": { - "dataset": 1, - "trait": 14329, - }, - "type": "dataset-publish", - } - conn = mock.MagicMock() - - conn.hgetall.return_value = { - '7fa95d07-0e2d-4bc5-b47c-448fdc1260b2': ( - '{"name": "editors", ' - '"admins": ["8ad942fe-490d-453e-bd37-56f252e41604", "rand"], ' - '"members": ["8ad942fe-490d-453e-bd37-56f252e41603", ' - '"rand"], ' - '"changed_timestamp": "Oct 06 2021 06:39PM", ' - '"created_timestamp": "Oct 06 2021 06:39PM"}')} - self.conn = conn - - def test_get_user_access_when_owner(self): + @mock.patch("wqflask.resource_manager.requests.get") + def test_edit_access(self, requests_mock): """Test that the right access roles are set""" + response = mock.PropertyMock(return_value=json.dumps( + { + 'data': ['no-access', 'view', 'edit', ], + 'metadata': ['no-access', 'view', 'edit', ], + 'admin': ['not-admin', 'edit-access', ], + } + )) + type(requests_mock.return_value).content = response self.assertEqual(get_user_access_roles( - conn=self.conn, - resource_info=self.resource_info, + resource_id="0196d92e1665091f202f", user_id="8ad942fe-490d-453e-bd37"), - {"data": DataRole.EDIT, - "metadata": DataRole.EDIT, - "admin": AdminRole.EDIT_ACCESS}) + {"data": DataRole.EDIT, + "metadata": DataRole.EDIT, + "admin": AdminRole.EDIT_ACCESS}) - def test_get_user_access_default_mask(self): + @mock.patch("wqflask.resource_manager.requests.get") + def test_no_access(self, requests_mock): + response = mock.PropertyMock(return_value=json.dumps( + { + 'data': ['no-access', ], + 'metadata': ['no-access', ], + 'admin': ['not-admin', ], + } + )) + type(requests_mock.return_value).content = response self.assertEqual(get_user_access_roles( - conn=self.conn, - resource_info=self.resource_info, + resource_id="0196d92e1665091f202f", user_id=""), - {"data": DataRole.NO_ACCESS, - "metadata": DataRole.NO_ACCESS, - "admin": AdminRole.NOT_ADMIN}) - - def test_get_user_access_group_mask(self): - self.assertEqual(get_user_access_roles( - conn=self.conn, - resource_info=self.resource_info, - user_id="8ad942fe-490d-453e-bd37-56f252e41603"), - {"data": DataRole.EDIT, - "metadata": DataRole.EDIT, - "admin": AdminRole.NOT_ADMIN}) + {"data": DataRole.NO_ACCESS, + "metadata": DataRole.NO_ACCESS, + "admin": AdminRole.NOT_ADMIN}) -- cgit v1.2.3 From d32f0e0f56827c653902456b50ed71911a795a3c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 20 Oct 2021 14:11:16 +0300 Subject: decorators: Use the correct logic to check for edit_admins_access --- wqflask/wqflask/decorators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index cd06aee7..843539ee 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -88,7 +88,7 @@ def edit_admins_access_required(f): except: response = {} if max([AdminRole(role) for role in response.get( - "data", ["not-admin"])]) >= AdminRole.EDIT_ADMINS: + "admin", ["not-admin"])]) < AdminRole.EDIT_ADMINS: return "You need to have edit-admins access", 401 return f(*args, **kwargs) return wrap -- cgit v1.2.3 From af1d704504d8f03135b805d5b4be100c5c0ad6f8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 20 Oct 2021 14:12:53 +0300 Subject: manage_resource.html: Update the URL for updating the owner --- wqflask/wqflask/templates/admin/manage_resource.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 1002ac45..af4d4908 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -19,7 +19,7 @@ {% endif %} {% if DATA_ACCESS > DataRole.VIEW and ADMIN_STATUS > AdminRole.NOT_ADMIN %} {% endif %} -- cgit v1.2.3 From 935270b1cc1e265b785958cf5805bf155d8ae859 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 20 Oct 2021 14:38:11 +0300 Subject: utility: redis_tools: Remove dead functions --- wqflask/utility/redis_tools.py | 78 ------------------------------------------ 1 file changed, 78 deletions(-) diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py index e68397ab..c2a3b057 100644 --- a/wqflask/utility/redis_tools.py +++ b/wqflask/utility/redis_tools.py @@ -58,30 +58,6 @@ def get_user_by_unique_column(column_name, column_value): return item_details -def get_users_like_unique_column(column_name, column_value): - """Like previous function, but this only checks if the input is a - subset of a field and can return multiple results - - """ - matched_users = [] - - if column_value != "": - user_list = Redis.hgetall("users") - if column_name != "user_id": - for key in user_list: - user_ob = json.loads(user_list[key]) - if "user_id" not in user_ob: - set_user_attribute(key, "user_id", key) - user_ob["user_id"] = key - if column_name in user_ob: - if column_value in user_ob[column_name]: - matched_users.append(user_ob) - else: - matched_users.append(load_json_from_redis(user_list, column_value)) - - return matched_users - - def set_user_attribute(user_id, column_name, column_value): user_info = json.loads(Redis.hget("users", user_id)) user_info[column_name] = column_value @@ -166,52 +142,6 @@ def get_group_info(group_id): return group_info -def get_group_by_unique_column(column_name, column_value): - """ Get group by column; not sure if there's a faster way to do this """ - - matched_groups = [] - - all_group_list = Redis.hgetall("groups") - for key in all_group_list: - group_info = json.loads(all_group_list[key]) - # ZS: Since these fields are lists, search in the list - if column_name == "admins" or column_name == "members": - if column_value in group_info[column_name]: - matched_groups.append(group_info) - else: - if group_info[column_name] == column_value: - matched_groups.append(group_info) - - return matched_groups - - -def get_groups_like_unique_column(column_name, column_value): - """Like previous function, but this only checks if the input is a - subset of a field and can return multiple results - - """ - matched_groups = [] - - if column_value != "": - group_list = Redis.hgetall("groups") - if column_name != "group_id": - for key in group_list: - group_info = json.loads(group_list[key]) - # ZS: Since these fields are lists, search in the list - if column_name == "admins" or column_name == "members": - if column_value in group_info[column_name]: - matched_groups.append(group_info) - else: - if column_name in group_info: - if column_value in group_info[column_name]: - matched_groups.append(group_info) - else: - matched_groups.append( - load_json_from_redis(group_list, column_value)) - - return matched_groups - - def create_group(admin_user_ids, member_user_ids=[], group_name="Default Group Name"): group_id = str(uuid.uuid4()) @@ -354,11 +284,3 @@ def add_access_mask(resource_id, group_id, access_mask): Redis.hset("resources", resource_id, json.dumps(the_resource)) return the_resource - - -def change_resource_owner(resource_id, new_owner_id): - the_resource = get_resource_info(resource_id) - the_resource['owner_id'] = new_owner_id - - Redis.delete("resource") - Redis.hset("resources", resource_id, json.dumps(the_resource)) -- cgit v1.2.3 From be0e73c1db12806580fa5c541eceec2a54c10ad3 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 25 Oct 2021 12:05:57 +0300 Subject: Add "GET /resources//change-owner" endpoint --- wqflask/wqflask/resource_manager.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 74941341..9487bab5 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -14,6 +14,7 @@ from urllib.parse import urljoin from typing import Dict from wqflask.decorators import edit_access_required +from wqflask.decorators import edit_admins_access_required from wqflask.decorators import login_required from wqflask.access_roles import AdminRole from wqflask.access_roles import DataRole @@ -177,3 +178,12 @@ def update_resource_publicity(resource_id: str): redis_conn.hset("resources", resource_id, json.dumps(resource_info)) return redirect(url_for("resource_management.view_resource", resource_id=resource_id)) + + +@resource_management.route("/resources//change-owner") +@edit_admins_access_required +@login_required +def view_resource_owner(resource_id: str): + return render_template( + "admin/change_resource_owner.html", + resource_id=resource_id) -- cgit v1.2.3 From 696a6c6dc758e16b582e20419819acd5b17a75f1 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 25 Oct 2021 12:07:05 +0300 Subject: Add "POST /resource-management//users/search" --- wqflask/wqflask/resource_manager.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 9487bab5..58c0b5cb 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -11,7 +11,7 @@ from flask import request from flask import url_for from urllib.parse import urljoin -from typing import Dict +from typing import Dict, Tuple from wqflask.decorators import edit_access_required from wqflask.decorators import edit_admins_access_required @@ -187,3 +187,21 @@ def view_resource_owner(resource_id: str): return render_template( "admin/change_resource_owner.html", resource_id=resource_id) + + +@resource_management.route("/users/search", methods=('POST',)) +@edit_admins_access_required +@login_required +def search_user(resource_id: str): + results = {} + for user in (users := redis.from_url( + current_app.config["REDIS_URL"], + decode_responses=True).hgetall("users")): + user = json.loads(users[user]) + for q in (request.form.get("user_name"), + request.form.get("user_email")): + if q and (q in user.get("email_address") or + q in user.get("full_name")): + results[user.get("user_id", "")] = user + return json.dumps(tuple(results.values())) + -- cgit v1.2.3 From 6296b5ff43b0a6209443679350bf15b399406699 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 25 Oct 2021 12:07:56 +0300 Subject: change_resource_owner.html: Update endpoint for user-search This link has the {{ resource_id }} set because it's a required argument for the "@edit_admins_access_required" decorator --- wqflask/wqflask/templates/admin/change_resource_owner.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/admin/change_resource_owner.html b/wqflask/wqflask/templates/admin/change_resource_owner.html index ae9409b0..b88d680d 100644 --- a/wqflask/wqflask/templates/admin/change_resource_owner.html +++ b/wqflask/wqflask/templates/admin/change_resource_owner.html @@ -57,7 +57,7 @@ $('#find_users').click(function() { $.ajax({ method: "POST", - url: "/search_for_users", + url: "/resource-management/{{ resource_id }}/users/search", data: { user_name: $('input[name=user_name]').val(), user_email: $('input[name=user_email]').val() -- cgit v1.2.3 From 57cd071f98fddf1bbd23a0ea1b42005698fc4a81 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 25 Oct 2021 12:10:20 +0300 Subject: change_resource_owner.html: Replace "var" with "let" "let" is lexically scoped. --- wqflask/wqflask/templates/admin/change_resource_owner.html | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/admin/change_resource_owner.html b/wqflask/wqflask/templates/admin/change_resource_owner.html index b88d680d..951df207 100644 --- a/wqflask/wqflask/templates/admin/change_resource_owner.html +++ b/wqflask/wqflask/templates/admin/change_resource_owner.html @@ -67,9 +67,8 @@ }) populate_users = function(json_user_list){ - var user_list = JSON.parse(json_user_list) - - var the_html = "" + let user_list = JSON.parse(json_user_list) + let the_html = "" if (user_list.length > 0){ the_html += "
    IdName Data MetadataAdmin
    {{ value.name }}{{ key }} {{ value.group_name}} {{ value.data }} {{ value.metadata }}{{ value.admin }}
    "; the_html += ""; -- cgit v1.2.3 From 7337cc6b141933f7b85f6e4928c6a88b73c3cbba Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 25 Oct 2021 12:12:11 +0300 Subject: change_resource_owner.html: Replace "the_html" with searchResultHtml --- .../templates/admin/change_resource_owner.html | 36 +++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/wqflask/wqflask/templates/admin/change_resource_owner.html b/wqflask/wqflask/templates/admin/change_resource_owner.html index 951df207..74ee1e45 100644 --- a/wqflask/wqflask/templates/admin/change_resource_owner.html +++ b/wqflask/wqflask/templates/admin/change_resource_owner.html @@ -68,40 +68,40 @@ populate_users = function(json_user_list){ let user_list = JSON.parse(json_user_list) - let the_html = "" + let searchResultsHtml = "" if (user_list.length > 0){ - the_html += "
    IndexNameE-mail AddressOrganization
    "; - the_html += ""; - the_html += ""; + searchResultsHtml += "
    IndexNameE-mail AddressOrganization
    "; + searchResultsHtml += ""; + searchResultsHtml += ""; for (_i = 0, _len = user_list.length; _i < _len; _i++) { this_user = user_list[_i] - the_html += ""; - the_html += ""; - the_html += "" + searchResultsHtml += ""; + searchResultsHtml += ""; + searchResultsHtml += "" if ("full_name" in this_user) { - the_html += ""; + searchResultsHtml += ""; } else { - the_html += "" + searchResultsHtml += "" } if ("email_address" in this_user) { - the_html += ""; + searchResultsHtml += ""; } else { - the_html += "" + searchResultsHtml += "" } if ("organization" in this_user) { - the_html += ""; + searchResultsHtml += ""; } else { - the_html += "" + searchResultsHtml += "" } - the_html += "" + searchResultsHtml += "" } - the_html += ""; - the_html += "
    IndexNameE-mail AddressOrganization
    " + (_i + 1).toString() + "
    " + (_i + 1).toString() + "" + this_user.full_name + "" + this_user.full_name + "N/AN/A" + this_user.email_address + "" + this_user.email_address + "N/AN/A" + this_user.organization + "" + this_user.organization + "N/AN/A
    "; + searchResultsHtml += ""; + searchResultsHtml += ""; } else { - the_html = "No users were found matching the entered criteria." + searchResultsHtml = "No users were found matching the entered criteria." } - $('#user_results').html(the_html) + $('#user_results').html(searchResultsHtml) if (user_list.length > 0){ $('#users_table').dataTable({ 'order': [[1, "asc" ]], -- cgit v1.2.3 From 18e57bad1b44fece0f7b1f98bd73e5556691a3cd Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 25 Oct 2021 12:42:11 +0300 Subject: Add "POST /resources//change-owner" endpoint --- wqflask/wqflask/resource_manager.py | 31 +++++++++++++++++----- .../templates/admin/change_resource_owner.html | 3 +-- .../wqflask/templates/admin/manage_resource.html | 3 ++- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 58c0b5cb..3371e59d 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -1,23 +1,24 @@ -import redis import json +import redis import requests from flask import Blueprint from flask import current_app +from flask import flash from flask import g -from flask import render_template from flask import redirect +from flask import render_template from flask import request from flask import url_for -from urllib.parse import urljoin from typing import Dict, Tuple +from urllib.parse import urljoin +from wqflask.access_roles import AdminRole +from wqflask.access_roles import DataRole from wqflask.decorators import edit_access_required from wqflask.decorators import edit_admins_access_required from wqflask.decorators import login_required -from wqflask.access_roles import AdminRole -from wqflask.access_roles import DataRole resource_management = Blueprint('resource_management', __name__) @@ -55,7 +56,7 @@ def get_user_membership(conn: redis.Redis, user_id: str, def get_user_access_roles( resource_id: str, user_id: str, - gn_proxy_url: str="http://localhost:8080") -> Dict: + gn_proxy_url: str = "http://localhost:8080") -> Dict: """Get the highest access roles for a given user Args: @@ -189,6 +190,23 @@ def view_resource_owner(resource_id: str): resource_id=resource_id) +@resource_management.route("/resources//change-owner", + methods=('POST',)) +@edit_admins_access_required +@login_required +def change_owner(resource_id: str): + if user_id := request.form.get("new_owner"): + redis_conn = redis.from_url( + current_app.config["REDIS_URL"], + decode_responses=True) + resource = json.loads(redis_conn.hget("resources", resource_id)) + resource["owner_id"] = user_id + redis_conn.hset("resources", resource_id, json.dumps(resource)) + flash("The resource's owner has been changed.", "alert-info") + return redirect(url_for("resource_management.view_resource", + resource_id=resource_id)) + + @resource_management.route("/users/search", methods=('POST',)) @edit_admins_access_required @login_required @@ -204,4 +222,3 @@ def search_user(resource_id: str): q in user.get("full_name")): results[user.get("user_id", "")] = user return json.dumps(tuple(results.values())) - diff --git a/wqflask/wqflask/templates/admin/change_resource_owner.html b/wqflask/wqflask/templates/admin/change_resource_owner.html index 74ee1e45..7fd84387 100644 --- a/wqflask/wqflask/templates/admin/change_resource_owner.html +++ b/wqflask/wqflask/templates/admin/change_resource_owner.html @@ -10,8 +10,7 @@

    Search for user to assign ownership to:

    -
    - +
    diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index af4d4908..a84bd77f 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -2,7 +2,8 @@ {% block title %}Resource Manager{% endblock %} {% block content %} -
    +
    + {{ flash_me() }} {% set DATA_ACCESS = access_role.get('data') %} {% set METADATA_ACCESS = access_role.get('metadata') %} {% set ADMIN_STATUS = access_role.get('admin') %} -- cgit v1.2.3 From 700802303e5e8221a9d591ba985d6607aa61e1ce Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 25 Oct 2021 12:43:01 +0300 Subject: manage_resource.html: Replace button with link --- wqflask/wqflask/templates/admin/manage_resource.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index a84bd77f..613aa70e 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -19,10 +19,10 @@ {% endif %} {% if DATA_ACCESS > DataRole.VIEW and ADMIN_STATUS > AdminRole.NOT_ADMIN %} - + {% endif %} {% endif %} -- cgit v1.2.3 From 6151faa9ea67af4bf4ea95fb681a9dc4319474b6 Mon Sep 17 00:00:00 2001 From: Arthur Centeno Date: Mon, 25 Oct 2021 20:51:16 +0000 Subject: Updated version of tutorials by ACenteno on 10-25-21 --- doc/joss/2016/2020.12.23.424047v1.full.pdf | Bin 0 -> 3804818 bytes wqflask/wqflask/templates/tutorials.html | 256 +++++++++++++++++++++++++++-- 2 files changed, 245 insertions(+), 11 deletions(-) create mode 100644 doc/joss/2016/2020.12.23.424047v1.full.pdf diff --git a/doc/joss/2016/2020.12.23.424047v1.full.pdf b/doc/joss/2016/2020.12.23.424047v1.full.pdf new file mode 100644 index 00000000..491dddf3 Binary files /dev/null and b/doc/joss/2016/2020.12.23.424047v1.full.pdf differ diff --git a/wqflask/wqflask/templates/tutorials.html b/wqflask/wqflask/templates/tutorials.html index 3e6ef01c..dbda8d6f 100644 --- a/wqflask/wqflask/templates/tutorials.html +++ b/wqflask/wqflask/templates/tutorials.html @@ -2,16 +2,250 @@ {% block title %}Tutorials/Primers{% endblock %} {% block content %} - - - -
    -

    Tutorials/Primers

    - -

    -
    + + GeneNetwork Webinar Series, Tutorials and Short Video Tours + + + + + + + + + + + + + + + + + + + + + + + +
    + +
    + + + +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + +
    Title/DescriptionPresentation

    Webinar #01 - Introduction to Quantitative Trait Loci (QTL) Analysis

    +

    Friday, May 8th, 2020
    + 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT

    +

    Goals of this webinar (trait variance to QTL):

    +
      +
    • Define quantitative trait locus (QTL)
      • +
    • Explain how genome scans can help find QTL
      • +
    +

    Presented by:
    +Dr. Saunak Sen
    +Professor and Chief of Biostatistics
    +Department of Preventative Medicine
    +University of Tennessee Health Science Center +

    +

    Link to course material +

    		 +

    Webinar #02 - Mapping Addiction and Behavioral Traits and Getting at Causal Gene Variants with GeneNetwork

    +

    Friday, May 22nd. 2020 + 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT +

    +

    Goals of this webinar (QTL to gene variant):

    +
      +
    • Demonstrate mapping a quantitative trait using GeneNetwork (GN)
      • +
    • Explore GN tools to identify genes and genetics variants related to a QTL
      • +
    +

    Presented by:
    +Dr. Rob Williams
    +Professor and Chair
    +Department of Genetics, Genomics, and Informatics
    +University of Tennessee Health Science Center +

    Link to course material +

    Data structure, disease risk, GXE, and causal modeling

    +

    Friday, November 20th at 9am PDT/ 11pm CDT/ 12pm EDT
    + 1-hour presentation followed by 30 minutes of discussion     + +

    Human disease is mainly due to complex interactions between genetic and environmental factors (GXE). We need to acquire the right "smart" data types—coherent and multiplicative data—required to make accurate predictions about risk and outcome for n = 1 individuals—a daunting task. We have developed large families of fully sequenced mice that mirror the genetic complexity of humans. We are using these Reference Populations to generate multiplicatively useful data and to build and test causal quantitative models of disease mechanisms with a special focus on diseases of aging, addiction, and neurological and psychiatric disease. + +

    Speaker Bio: Robert (Rob) W. Williams received a BA in neuroscience from UC Santa Cruz (1975) and a Ph.D. in system physiology at UC Davis with Leo M. Chalupa (1983). He did postdoctoral work in developmental neurobiology at Yale School of Medicine with Pasko Rakic where he developed novel stereological methods to estimate cell populations in brain. In 2013 Williams established the Department of Genetics, Genomics and Informatics at UTHSC. He holds the UT Oak Ridge National Laboratory Governor’s Chair in Computational Genomics. Williams is director of the Complex Trait Community (www.complextrait.org) and editor-in-chief of Frontiers in Neurogenomics. One of Williams’ more notable contributions is in the field of systems neurogenetics and experimental precision medicine. He and his research collaborators have built GeneNetwork (www.genenetwork.org), an online resource of data and analysis code that is used as a platform for experimental precision medicine.

    + +

    Presented by:
    +Dr. Rob Williams
    +Professor and Chair
    +Department of Genetics, Genomics, and Informatics
    +University of Tennessee Health Science Center +

    + + +    		 +
    + + +
    +
    + + + + + + + + + + + + + + + + + + + + + + + +
    Title/DescriptionPresentation

    #01- Introduction to Gene Network

    +

    Please note that this tutorial is based on GeneNetwork v1 + +

    GeneNetwork is a group of linked data sets and tools used to study complex networks of genes, molecules, and higher order gene function and phenotypes. GeneNetwork combines more than 25 years of legacy data generated by hundreds of scientists together with sequence data (SNPs) and massive transcriptome data sets (expression genetic or eQTL data sets). The quantitative trait locus (QTL) mapping module that is built into GN is optimized for fast on-line analysis of traits that are controlled by combinations of gene variants and environmental factors. GeneNetwork can be used to study humans, mice (BXD, AXB, LXS, etc.), rats (HXB), Drosophila, and plant species (barley and Arabidopsis). Most of these population data sets are linked with dense genetic maps (genotypes) that can be used to locate the genetic modifiers that cause differences in expression and phenotypes, including disease susceptibility. + +

    Users are welcome to enter their own private data directly into GeneNetwork to exploit the full range of analytic tools and to map modulators in a powerful environment. This combination of data and fast analytic functions enable users to study relations between sequence variants, molecular networks, and function.

    + +

    Presented by:
    +Dr. Rob Williams
    +Professor and Chair
    +Department of Genetics, Genomics, and Informatics
    +University of Tennessee Health Science Center +

    + + +    		 +

    #02 - How to search in GeneNetwork

    +    		 +
    + + +
    +
    + + + + + + + + + + + + + + + + +
    TitleSpeakerVideo link
    Diallel Crosses, Artificial Intelligence, and Mouse Models of Alzheimer’s DiseaseDavid G. Ashbrook
    Assistant Professor
    University of Tennessee Health Science Center    		YouTube link
    + +
    + + +
    + +
    +
    + +
    + +
    + + + +
    + + + + + + + + + + {% endblock %} + -- cgit v1.2.3 From 61fb2dbba2b98debd5e56d49a5517df18dea171b Mon Sep 17 00:00:00 2001 From: Arthur Centeno Date: Mon, 25 Oct 2021 22:13:17 +0000 Subject: Updated version tutorial on 10-25-21 by ACenteno --- wqflask/wqflask/templates/tutorials.html | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/wqflask/wqflask/templates/tutorials.html b/wqflask/wqflask/templates/tutorials.html index dbda8d6f..81fb5f5a 100644 --- a/wqflask/wqflask/templates/tutorials.html +++ b/wqflask/wqflask/templates/tutorials.html @@ -169,6 +169,15 @@ Registration: https://bit.ly/osga_2020- + + + + + +

    #03 - GeneNetwork.org: genetic analysis for all neuroscientists


    Presented by David G. Ashbrook Assistant Professor University of Tennessee Health Science Center + + + -- cgit v1.2.3 From 3181bd6261b09a5e9b027256057c21c49792bd32 Mon Sep 17 00:00:00 2001 From: jgart Date: Fri, 10 Sep 2021 00:29:28 -0400 Subject: Remove unnecessary git pull commands from installation instructions --- doc/README.org | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/README.org b/doc/README.org index 1236016e..8839aefc 100644 --- a/doc/README.org +++ b/doc/README.org @@ -81,14 +81,12 @@ GeneNetwork2 with : source ~/opt/guix-pull/etc/profile : git clone https://git.genenetwork.org/guix-bioinformatics/guix-bioinformatics.git ~/guix-bioinformatics : cd ~/guix-bioinformatics -: git pull : env GUIX_PACKAGE_PATH=$HOME/guix-bioinformatics guix package -i genenetwork2 -p ~/opt/genenetwork2 you probably also need guix-past (the upstream channel for older packages): : git clone https://gitlab.inria.fr/guix-hpc/guix-past.git ~/guix-past : cd ~/guix-past -: git pull : env GUIX_PACKAGE_PATH=$HOME/guix-bioinformatics:$HOME/guix-past/modules ~/opt/guix-pull/bin/guix package -i genenetwork2 -p ~/opt/genenetwork2 ignore the warnings. Guix should install the software without trying -- cgit v1.2.3 From ca23d4ed6943d25c14ffac767b64fd60bded515e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 17:10:33 +0300 Subject: Move endpoints for viewing metadata to own module --- wqflask/wqflask/__init__.py | 3 + wqflask/wqflask/metadata_edits.py | 147 ++++++++++++++++++++++++++++++++++++++ wqflask/wqflask/views.py | 115 +---------------------------- 3 files changed, 153 insertions(+), 112 deletions(-) create mode 100644 wqflask/wqflask/metadata_edits.py diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index 5b2d05d1..a5097287 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -11,6 +11,8 @@ from utility import formatting from wqflask.resource_manager import resource_management +from wqflask.metadata_edits import metadata_edit + from wqflask.api.markdown import glossary_blueprint from wqflask.api.markdown import references_blueprint from wqflask.api.markdown import links_blueprint @@ -60,6 +62,7 @@ app.register_blueprint(news_blueprint, url_prefix="/news") app.register_blueprint(resource_management, url_prefix="/resource-management") +app.register_blueprint(metadata_edit, url_prefix="/datasets/") @app.before_request def before_request(): diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py new file mode 100644 index 00000000..94e2710b --- /dev/null +++ b/wqflask/wqflask/metadata_edits.py @@ -0,0 +1,147 @@ +import MySQLdb +import os +import json +import difflib + + +from collections import namedtuple +from flask import Blueprint, current_app, render_template, request +from itertools import groupby + +from wqflask.decorators import edit_access_required + +from gn3.db import diff_from_dict +from gn3.db import fetchall +from gn3.db import fetchone +from gn3.db import insert +from gn3.db import update +from gn3.db.metadata_audit import MetadataAudit +from gn3.db.phenotypes import Phenotype +from gn3.db.phenotypes import Probeset +from gn3.db.phenotypes import Publication +from gn3.db.phenotypes import PublishXRef +from gn3.db.phenotypes import probeset_mapping + + +metadata_edit = Blueprint('metadata_edit', __name__) + + +def edit_phenotype(conn, name, dataset_id): + publish_xref = fetchone( + conn=conn, + table="PublishXRef", + where=PublishXRef(id_=name, + inbred_set_id=dataset_id)) + phenotype_ = fetchone( + conn=conn, + table="Phenotype", + where=Phenotype(id_=publish_xref.phenotype_id)) + publication_ = fetchone( + conn=conn, + table="Publication", + where=Publication(id_=publish_xref.publication_id)) + json_data = fetchall( + conn, + "metadata_audit", + where=MetadataAudit(dataset_id=publish_xref.id_)) + Edit = namedtuple("Edit", ["field", "old", "new", "diff"]) + Diff = namedtuple("Diff", ["author", "diff", "timestamp"]) + diff_data = [] + for data in json_data: + json_ = json.loads(data.json_data) + timestamp = json_.get("timestamp") + author = json_.get("author") + for key, value in json_.items(): + if isinstance(value, dict): + for field, data_ in value.items(): + diff_data.append( + Diff(author=author, + diff=Edit(field, + data_.get("old"), + data_.get("new"), + "\n".join(difflib.ndiff( + [data_.get("old")], + [data_.get("new")]))), + timestamp=timestamp)) + diff_data_ = None + if len(diff_data) > 0: + diff_data_ = groupby(diff_data, lambda x: x.timestamp) + return { + "diff": diff_data_, + "publish_xref": publish_xref, + "phenotype": phenotype_, + "publication": publication_, + } + + +def edit_probeset(conn, name): + probeset_ = fetchone(conn=conn, + table="ProbeSet", + columns=list(probeset_mapping.values()), + where=Probeset(name=name)) + json_data = fetchall( + conn, + "metadata_audit", + where=MetadataAudit(dataset_id=probeset_.id_)) + Edit = namedtuple("Edit", ["field", "old", "new", "diff"]) + Diff = namedtuple("Diff", ["author", "diff", "timestamp"]) + diff_data = [] + for data in json_data: + json_ = json.loads(data.json_data) + timestamp = json_.get("timestamp") + author = json_.get("author") + for key, value in json_.items(): + if isinstance(value, dict): + for field, data_ in value.items(): + diff_data.append( + Diff(author=author, + diff=Edit(field, + data_.get("old"), + data_.get("new"), + "\n".join(difflib.ndiff( + [data_.get("old")], + [data_.get("new")]))), + timestamp=timestamp)) + diff_data_ = None + if len(diff_data) > 0: + diff_data_ = groupby(diff_data, lambda x: x.timestamp) + return { + "diff": diff_data_, + "probeset": probeset_, + } + + +@metadata_edit.route("//traits//edit") +@edit_access_required +def display_phenotype_metadata(dataset_id: str, name: str): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + _d = edit_phenotype(conn=conn, name=name, dataset_id=dataset_id) + return render_template( + "edit_phenotype.html", + diff=_d.get("diff"), + publish_xref=_d.get("publish_xref"), + phenotype=_d.get("phenotype"), + publication=_d.get("publication"), + resource_id=request.args.get("resource-id"), + version=os.environ.get("GN_VERSION"), + ) + + +@metadata_edit.route("/traits//edit") +@edit_access_required +def display_probeset_metadata(name: str): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + _d = edit_probeset(conn=conn, name=name) + return render_template( + "edit_probeset.html", + diff=_d.get("diff"), + probeset=_d.get("probeset"), + resource_id=request.args.get("resource-id"), + version=os.environ.get("GN_VERSION"), + ) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index b0da1f21..463b7c3a 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -4,7 +4,6 @@ import MySQLdb import array import base64 import csv -import difflib import datetime import flask import io # Todo: Use cStringIO? @@ -20,8 +19,6 @@ import traceback import uuid import xlsxwriter -from itertools import groupby -from collections import namedtuple from zipfile import ZipFile from zipfile import ZIP_DEFLATED @@ -30,19 +27,12 @@ from wqflask import app from gn3.commands import run_cmd from gn3.computations.gemma import generate_hash_of_string from gn3.db import diff_from_dict -from gn3.db import fetchall -from gn3.db import fetchone from gn3.db import insert from gn3.db import update from gn3.db.metadata_audit import MetadataAudit from gn3.db.phenotypes import Phenotype from gn3.db.phenotypes import Probeset from gn3.db.phenotypes import Publication -from gn3.db.phenotypes import PublishXRef -from gn3.db.phenotypes import probeset_mapping -# from gn3.db.traits import get_trait_csv_sample_data -# from gn3.db.traits import update_sample_data - from flask import current_app from flask import g @@ -426,106 +416,6 @@ def submit_trait_form(): version=GN_VERSION) -@app.route("/trait//edit/inbredset-id/") -@edit_access_required -def edit_phenotype(name, inbredset_id): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - publish_xref = fetchone( - conn=conn, - table="PublishXRef", - where=PublishXRef(id_=name, - inbred_set_id=inbredset_id)) - phenotype_ = fetchone( - conn=conn, - table="Phenotype", - where=Phenotype(id_=publish_xref.phenotype_id)) - publication_ = fetchone( - conn=conn, - table="Publication", - where=Publication(id_=publish_xref.publication_id)) - json_data = fetchall( - conn, - "metadata_audit", - where=MetadataAudit(dataset_id=publish_xref.id_)) - - Edit = namedtuple("Edit", ["field", "old", "new", "diff"]) - Diff = namedtuple("Diff", ["author", "diff", "timestamp"]) - diff_data = [] - for data in json_data: - json_ = json.loads(data.json_data) - timestamp = json_.get("timestamp") - author = json_.get("author") - for key, value in json_.items(): - if isinstance(value, dict): - for field, data_ in value.items(): - diff_data.append( - Diff(author=author, - diff=Edit(field, - data_.get("old"), - data_.get("new"), - "\n".join(difflib.ndiff( - [data_.get("old")], - [data_.get("new")]))), - timestamp=timestamp)) - diff_data_ = None - if len(diff_data) > 0: - diff_data_ = groupby(diff_data, lambda x: x.timestamp) - return render_template( - "edit_phenotype.html", - diff=diff_data_, - publish_xref=publish_xref, - phenotype=phenotype_, - publication=publication_, - version=GN_VERSION, - ) - - -@app.route("/trait/edit/probeset-name/") -@edit_access_required -def edit_probeset(dataset_name): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - probeset_ = fetchone(conn=conn, - table="ProbeSet", - columns=list(probeset_mapping.values()), - where=Probeset(name=dataset_name)) - json_data = fetchall( - conn, - "metadata_audit", - where=MetadataAudit(dataset_id=probeset_.id_)) - Edit = namedtuple("Edit", ["field", "old", "new", "diff"]) - Diff = namedtuple("Diff", ["author", "diff", "timestamp"]) - diff_data = [] - for data in json_data: - json_ = json.loads(data.json_data) - timestamp = json_.get("timestamp") - author = json_.get("author") - for key, value in json_.items(): - if isinstance(value, dict): - for field, data_ in value.items(): - diff_data.append( - Diff(author=author, - diff=Edit(field, - data_.get("old"), - data_.get("new"), - "\n".join(difflib.ndiff( - [data_.get("old")], - [data_.get("new")]))), - timestamp=timestamp)) - diff_data_ = None - if len(diff_data) > 0: - diff_data_ = groupby(diff_data, lambda x: x.timestamp) - return render_template( - "edit_probeset.html", - diff=diff_data_, - probeset=probeset_) - - @app.route("/trait/update", methods=["POST"]) @edit_access_required def update_phenotype(): @@ -653,7 +543,6 @@ def update_phenotype(): @app.route("/probeset/update", methods=["POST"]) -@edit_access_required def update_probeset(): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), @@ -691,7 +580,9 @@ def update_probeset(): where=Probeset(id_=data_.get("id"))) diff_data = {} - author = g.user_session.record.get(b'user_name') + author = (g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) if updated_probeset: diff_data.update({"Probeset": diff_from_dict(old={ k: data_.get(f"old_{k}") for k, v in probeset_.items() -- cgit v1.2.3 From 433f62500408d84a49153628384f4b4c3e9a7b2e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 17:10:50 +0300 Subject: Get "resource-id" from query parameters instead of computing it --- wqflask/wqflask/decorators.py | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index 843539ee..a4ff7ce3 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -1,9 +1,7 @@ """This module contains gn2 decorators""" -import hashlib -import hmac import redis -from flask import current_app, g +from flask import current_app, g, request from typing import Dict from urllib.parse import urljoin from functools import wraps @@ -14,18 +12,12 @@ import json import requests -def create_hmac(data: str, secret: str) -> str: - return hmac.new(bytearray(secret, "latin-1"), - bytearray(data, "utf-8"), - hashlib.sha1).hexdigest()[:20] - - def login_required(f): """Use this for endpoints where login is required""" @wraps(f) def wrap(*args, **kwargs): user_id = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or + b"").decode("utf-8") or g.user_session.record.get("user_id", "")) redis_conn = redis.from_url(current_app.config["REDIS_URL"], decode_responses=True) @@ -40,28 +32,21 @@ def edit_access_required(f): @wraps(f) def wrap(*args, **kwargs): resource_id: str = "" - if kwargs.get("inbredset_id"): # data type: dataset-publish - resource_id = create_hmac( - data=("dataset-publish:" - f"{kwargs.get('inbredset_id')}:" - f"{kwargs.get('name')}"), - secret=current_app.config.get("SECRET_HMAC_CODE")) - if kwargs.get("dataset_name"): # data type: dataset-probe - resource_id = create_hmac( - data=("dataset-probeset:" - f"{kwargs.get('dataset_name')}"), - secret=current_app.config.get("SECRET_HMAC_CODE")) - if kwargs.get("resource_id"): # The resource_id is already provided + if request.args.get("resource-id"): + resource_id = request.args.get("resource-id") + elif kwargs.get("resource_id"): resource_id = kwargs.get("resource_id") response: Dict = {} try: - _user_id = g.user_session.record.get(b"user_id", - "").decode("utf-8") + _user_id = (g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) response = json.loads( requests.get(urljoin( current_app.config.get("GN2_PROXY"), ("available?resource=" f"{resource_id}&user={_user_id}"))).content) + except: response = {} if max([DataRole(role) for role in response.get( @@ -78,8 +63,9 @@ def edit_admins_access_required(f): resource_id: str = kwargs.get("resource_id", "") response: Dict = {} try: - _user_id = g.user_session.record.get(b"user_id", - "").decode("utf-8") + _user_id = (g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) response = json.loads( requests.get(urljoin( current_app.config.get("GN2_PROXY"), @@ -92,4 +78,3 @@ def edit_admins_access_required(f): return "You need to have edit-admins access", 401 return f(*args, **kwargs) return wrap - -- cgit v1.2.3 From 50553aa93b6fb0df15f30235c70cff7f08d93ef4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 17:21:21 +0300 Subject: Remove debug statement --- wqflask/wqflask/show_trait/show_trait.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index c4d1ae1c..d6355921 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -525,10 +525,6 @@ class ShowTrait: sample_group_type='primary', header="%s Only" % (self.dataset.group.name)) self.sample_groups = (primary_samples,) - print("\nttttttttttttttttttttttttttttttttttttttttttttt\n") - print(self.sample_groups) - print("\nttttttttttttttttttttttttttttttttttttttttttttt\n") - self.primary_sample_names = primary_sample_names self.dataset.group.allsamples = all_samples_ordered -- cgit v1.2.3 From a2a8f5da64a52d52354eded0b1b3adf149a234b3 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 20:51:19 +0300 Subject: Use new "get_user_access_roles" when showing a trait * wqflask/wqflask/show_trait/show_trait.py (ShowTrait): Replace `check_owner_or_admin` with `get_user_access_roles` * wqflask/wqflask/views.py (show_temp_trait): Pass `user_id` as extra arg to ShowTrait. (show_trait_page): Ditto. --- wqflask/wqflask/show_trait/show_trait.py | 22 +++++++++++----------- wqflask/wqflask/views.py | 12 ++++++++++-- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index d6355921..777efd02 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -20,9 +20,13 @@ from base import data_set from utility import helper_functions from utility.authentication_tools import check_owner_or_admin from utility.tools import locate_ignore_error +from utility.tools import GN_PROXY_URL from utility.redis_tools import get_redis_conn, get_resource_id from utility.logger import getLogger +from wqflask.access_roles import AdminRole +from wqflask.access_roles import DataRole +from wqflask.resource_manager import get_user_access_roles Redis = get_redis_conn() ONE_YEAR = 60 * 60 * 24 * 365 @@ -38,14 +42,11 @@ logger = getLogger(__name__) class ShowTrait: - def __init__(self, kw): + def __init__(self, user_id, kw): if 'trait_id' in kw and kw['dataset'] != "Temp": self.temp_trait = False self.trait_id = kw['trait_id'] helper_functions.get_species_dataset_trait(self, kw) - self.resource_id = get_resource_id(self.dataset, self.trait_id) - self.admin_status = check_owner_or_admin( - resource_id=self.resource_id) elif 'group' in kw: self.temp_trait = True self.trait_id = "Temp_" + kw['species'] + "_" + kw['group'] + \ @@ -62,9 +63,6 @@ class ShowTrait: self.this_trait = create_trait(dataset=self.dataset, name=self.trait_id, cellid=None) - - self.admin_status = check_owner_or_admin( - dataset=self.dataset, trait_id=self.trait_id) else: self.temp_trait = True self.trait_id = kw['trait_id'] @@ -75,11 +73,13 @@ class ShowTrait: self.this_trait = create_trait(dataset=self.dataset, name=self.trait_id, cellid=None) - self.trait_vals = Redis.get(self.trait_id).split() - self.admin_status = check_owner_or_admin( - dataset=self.dataset, trait_id=self.trait_id) - + self.resource_id = get_resource_id(self.dataset, + self.trait_id) + self.admin_status = get_user_access_roles( + user_id=user_id, + resource_id=(self.resource_id or ""), + gn_proxy_url=GN_PROXY_URL) # ZS: Get verify/rna-seq link URLs try: blatsequence = self.this_trait.blatseq diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 463b7c3a..6b00bf34 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -735,7 +735,11 @@ def export_perm_data(): @app.route("/show_temp_trait", methods=('POST',)) def show_temp_trait_page(): logger.info(request.url) - template_vars = show_trait.ShowTrait(request.form) + user_id = (g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) + template_vars = show_trait.ShowTrait(user_id=user_id, + kw=request.form) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, indent=" ") @@ -745,7 +749,11 @@ def show_temp_trait_page(): @app.route("/show_trait") def show_trait_page(): logger.info(request.url) - template_vars = show_trait.ShowTrait(request.args) + user_id = (g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) + template_vars = show_trait.ShowTrait(user_id=user_id, + kw=request.args) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, indent=" ") -- cgit v1.2.3 From 2b82f284efe60767c0e3f30e095094cee9c10a81 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 20:56:11 +0300 Subject: Delete noisy logging --- wqflask/wqflask/show_trait/show_trait.py | 4 ---- wqflask/wqflask/views.py | 2 -- 2 files changed, 6 deletions(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 777efd02..fa1206c9 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -22,7 +22,6 @@ from utility.authentication_tools import check_owner_or_admin from utility.tools import locate_ignore_error from utility.tools import GN_PROXY_URL from utility.redis_tools import get_redis_conn, get_resource_id -from utility.logger import getLogger from wqflask.access_roles import AdminRole from wqflask.access_roles import DataRole @@ -31,8 +30,6 @@ from wqflask.resource_manager import get_user_access_roles Redis = get_redis_conn() ONE_YEAR = 60 * 60 * 24 * 365 -logger = getLogger(__name__) - ############################################### # # Todo: Put in security to ensure that user has permission to access @@ -610,7 +607,6 @@ def get_nearest_marker(this_trait, this_db): GenoFreeze.Id = GenoXRef.GenoFreezeId AND GenoFreeze.Name = '{}' ORDER BY ABS( Geno.Mb - {}) LIMIT 1""".format(this_chr, this_db.group.name + "Geno", this_mb) - logger.sql(query) result = g.db.execute(query).fetchall() if result == []: diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 6b00bf34..08c88a25 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -734,7 +734,6 @@ def export_perm_data(): @app.route("/show_temp_trait", methods=('POST',)) def show_temp_trait_page(): - logger.info(request.url) user_id = (g.user_session.record.get(b"user_id", b"").decode("utf-8") or g.user_session.record.get("user_id", "")) @@ -748,7 +747,6 @@ def show_temp_trait_page(): @app.route("/show_trait") def show_trait_page(): - logger.info(request.url) user_id = (g.user_session.record.get(b"user_id", b"").decode("utf-8") or g.user_session.record.get("user_id", "")) -- cgit v1.2.3 From 46a9f5e522b96346034044b946ff85ac71197699 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 20:57:01 +0300 Subject: Make DataRole and AdminRole available to all jinja templates --- wqflask/wqflask/__init__.py | 11 +++++++++++ wqflask/wqflask/resource_manager.py | 3 +-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index a5097287..169192c7 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -9,6 +9,7 @@ from typing import Tuple from urllib.parse import urlparse from utility import formatting +from wqflask.access_roles import DataRole, AdminRole from wqflask.resource_manager import resource_management from wqflask.metadata_edits import metadata_edit @@ -70,6 +71,16 @@ def before_request(): g.request_time = lambda: "%.5fs" % (time.time() - g.request_start_time) +@app.context_processor +def include_admin_role_class(): + return {'AdminRole': AdminRole} + + +@app.context_processor +def include_data_role_class(): + return {'DataRole': DataRole} + + from wqflask.api import router from wqflask import group_manager from wqflask import resource_manager diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index 3371e59d..e338a22d 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -147,8 +147,7 @@ def view_resource(resource_id: str): access_role=get_user_access_roles( resource_id=resource_id, user_id=user_id, - gn_proxy_url=current_app.config.get("GN2_PROXY")), - DataRole=DataRole, AdminRole=AdminRole) + gn_proxy_url=current_app.config.get("GN2_PROXY"))) @resource_management.route("/resources//make-public", -- cgit v1.2.3 From 3f0f6dfd00fe2cf302a7dac2c7507d82f02e8a35 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 20:58:31 +0300 Subject: show_trait_details.html: Update html links for editing metadata --- wqflask/wqflask/templates/show_trait_details.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 2a21dd24..f824d6e2 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -236,11 +236,12 @@ {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} {% if this_trait.dataset.type == 'Publish' %} - + {{ this_trait }} + {% endif %} {% if this_trait.dataset.type == 'ProbeSet' %} - + {% endif %} {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} -- cgit v1.2.3 From 5e3f63c45baba9ba8967144dc3bc4fbe0f798b07 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 21:01:05 +0300 Subject: show_trait_details.html: Use new link for managing privileges --- wqflask/wqflask/templates/show_trait_details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index f824d6e2..183c4c61 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -244,7 +244,7 @@ {% endif %} {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} - + {% endif %} {% endif %}
    -- cgit v1.2.3 From 8a3faf5a975895a05d0bf4c9c3a77079b1b6bf5a Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 21:01:05 +0300 Subject: show_trait_details.html: Use new acccess roles in template --- wqflask/wqflask/templates/show_trait_details.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 183c4c61..b0639879 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -234,7 +234,7 @@ {% endif %} {% endif %} - {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} + {% if admin_status.get('metadata', Datarole.VIEW) > DataRole.VIEW %} {% if this_trait.dataset.type == 'Publish' %} {{ this_trait }} @@ -243,7 +243,7 @@ {% if this_trait.dataset.type == 'ProbeSet' %} {% endif %} - {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} + {% if admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} {% endif %} {% endif %} -- cgit v1.2.3 From 0e0da16c9bd5b0ea6366201cb34de6547dbe4080 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 22:37:04 +0300 Subject: metadata_edits: Add "POST //traits//update" --- wqflask/wqflask/metadata_edits.py | 136 +++++++++++++++++++++++++++++++++++++- wqflask/wqflask/views.py | 126 ----------------------------------- 2 files changed, 135 insertions(+), 127 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 94e2710b..bab1fa71 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -1,11 +1,13 @@ import MySQLdb import os import json +import datetime import difflib from collections import namedtuple -from flask import Blueprint, current_app, render_template, request +from flask import (Blueprint, current_app, redirect, + flash, g, render_template, request) from itertools import groupby from wqflask.decorators import edit_access_required @@ -21,6 +23,9 @@ from gn3.db.phenotypes import Probeset from gn3.db.phenotypes import Publication from gn3.db.phenotypes import PublishXRef from gn3.db.phenotypes import probeset_mapping +from gn3.commands import run_cmd +from gn3.db.traits import get_trait_csv_sample_data +from gn3.db.traits import update_sample_data metadata_edit = Blueprint('metadata_edit', __name__) @@ -125,6 +130,7 @@ def display_phenotype_metadata(dataset_id: str, name: str): publish_xref=_d.get("publish_xref"), phenotype=_d.get("phenotype"), publication=_d.get("publication"), + dataset_id=dataset_id, resource_id=request.args.get("resource-id"), version=os.environ.get("GN_VERSION"), ) @@ -145,3 +151,131 @@ def display_probeset_metadata(name: str): resource_id=request.args.get("resource-id"), version=os.environ.get("GN_VERSION"), ) + + +@metadata_edit.route("//traits//update", methods=("POST",)) +@edit_access_required +def update_phenotype(dataset_id: str, name: str): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + data_ = request.form.to_dict() + TMPDIR = current_app.config.get("TMPDIR") + author = (g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) + phenotype_id = str(data_.get('phenotype-id')) + if 'file' not in request.files: + flash("No sample-data has been uploaded", "warning") + else: + file_ = request.files['file'] + SAMPLE_DATADIR = os.path.join(TMPDIR, "sample-data") + if not os.path.exists(SAMPLE_DATADIR): + os.makedirs(SAMPLE_DATADIR) + if not os.path.exists(os.path.join(SAMPLE_DATADIR, + "diffs")): + os.makedirs(os.path.join(SAMPLE_DATADIR, + "diffs")) + if not os.path.exists(os.path.join(SAMPLE_DATADIR, + "updated")): + os.makedirs(os.path.join(SAMPLE_DATADIR, + "updated")) + current_time = str(datetime.datetime.now().isoformat()) + new_file_name = (os.path.join(TMPDIR, + "sample-data/updated/", + (f"{author}." + f"{name}.{phenotype_id}." + f"{current_time}.csv"))) + uploaded_file_name = (os.path.join( + TMPDIR, + "sample-data/updated/", + (f"updated.{author}." + f"{request.args.get('resource-id')}." + f"{current_time}.csv"))) + file_.save(new_file_name) + publishdata_id = "" + lines = [] + with open(new_file_name, "r") as f: + lines = f.read() + first_line = lines.split('\n', 1)[0] + publishdata_id = first_line.split("Id:")[-1].strip() + with open(new_file_name, "w") as f: + f.write(lines.split("\n\n")[-1]) + csv_ = get_trait_csv_sample_data(conn=conn, + trait_name=str(name), + phenotype_id=str(phenotype_id)) + with open(uploaded_file_name, "w") as f_: + f_.write(csv_.split("\n\n")[-1]) + r = run_cmd(cmd=("csvdiff " + f"'{uploaded_file_name}' '{new_file_name}' " + "--format json")) + diff_output = (f"{TMPDIR}/sample-data/diffs/" + f"{author}.{request.args.get('resource-id')}." + f"{current_time}.json") + with open(diff_output, "w") as f: + dict_ = json.loads(r.get("output")) + dict_.update({ + "author": author, + "publishdata_id": publishdata_id, + "dataset_id": data_.get("dataset-name"), + "timestamp": datetime.datetime.now().strftime( + "%Y-%m-%d %H:%M:%S") + }) + f.write(json.dumps(dict_)) + flash("Sample-data has been successfully uploaded", "success") + # Run updates: + phenotype_ = { + "pre_pub_description": data_.get("pre-pub-desc"), + "post_pub_description": data_.get("post-pub-desc"), + "original_description": data_.get("orig-desc"), + "units": data_.get("units"), + "pre_pub_abbreviation": data_.get("pre-pub-abbrev"), + "post_pub_abbreviation": data_.get("post-pub-abbrev"), + "lab_code": data_.get("labcode"), + "submitter": data_.get("submitter"), + "owner": data_.get("owner"), + "authorized_users": data_.get("authorized-users"), + } + updated_phenotypes = update( + conn, "Phenotype", + data=Phenotype(**phenotype_), + where=Phenotype(id_=data_.get("phenotype-id"))) + diff_data = {} + if updated_phenotypes: + diff_data.update({"Phenotype": diff_from_dict(old={ + k: data_.get(f"old_{k}") for k, v in phenotype_.items() + if v is not None}, new=phenotype_)}) + publication_ = { + "abstract": data_.get("abstract"), + "authors": data_.get("authors"), + "title": data_.get("title"), + "journal": data_.get("journal"), + "volume": data_.get("volume"), + "pages": data_.get("pages"), + "month": data_.get("month"), + "year": data_.get("year") + } + updated_publications = update( + conn, "Publication", + data=Publication(**publication_), + where=Publication(id_=data_.get("pubmed-id", + data_.get("old_id_")))) + if updated_publications: + diff_data.update({"Publication": diff_from_dict(old={ + k: data_.get(f"old_{k}") for k, v in publication_.items() + if v is not None}, new=publication_)}) + if diff_data: + diff_data.update({"dataset_id": name}) + diff_data.update({"resource_id": request.args.get('resource-id')}) + diff_data.update({"author": author}) + diff_data.update({"timestamp": datetime.datetime.now().strftime( + "%Y-%m-%d %H:%M:%S")}) + insert(conn, + table="metadata_audit", + data=MetadataAudit(dataset_id=name, + editor=author, + json_data=json.dumps(diff_data))) + flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success") + return redirect(f"/datasets/{dataset_id}/traits/{name}/edit" + f"?resource-id={request.args.get('resource-id')}") diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 08c88a25..963e48b7 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -416,132 +416,6 @@ def submit_trait_form(): version=GN_VERSION) -@app.route("/trait/update", methods=["POST"]) -@edit_access_required -def update_phenotype(): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - data_ = request.form.to_dict() - TMPDIR = current_app.config.get("TMPDIR") - author = g.user_session.record.get(b'user_name') - if 'file' not in request.files: - flash("No sample-data has been uploaded", "warning") - else: - file_ = request.files['file'] - trait_name = str(data_.get('dataset-name')) - phenotype_id = str(data_.get('phenotype-id', 35)) - SAMPLE_DATADIR = os.path.join(TMPDIR, "sample-data") - if not os.path.exists(SAMPLE_DATADIR): - os.makedirs(SAMPLE_DATADIR) - if not os.path.exists(os.path.join(SAMPLE_DATADIR, - "diffs")): - os.makedirs(os.path.join(SAMPLE_DATADIR, - "diffs")) - if not os.path.exists(os.path.join(SAMPLE_DATADIR, - "updated")): - os.makedirs(os.path.join(SAMPLE_DATADIR, - "updated")) - current_time = str(datetime.datetime.now().isoformat()) - new_file_name = (os.path.join(TMPDIR, - "sample-data/updated/", - (f"{author.decode('utf-8')}." - f"{trait_name}.{phenotype_id}." - f"{current_time}.csv"))) - uploaded_file_name = (os.path.join( - TMPDIR, - "sample-data/updated/", - (f"updated.{author.decode('utf-8')}." - f"{trait_name}.{phenotype_id}." - f"{current_time}.csv"))) - file_.save(new_file_name) - publishdata_id = "" - lines = [] - with open(new_file_name, "r") as f: - lines = f.read() - first_line = lines.split('\n', 1)[0] - publishdata_id = first_line.split("Id:")[-1].strip() - with open(new_file_name, "w") as f: - f.write(lines.split("\n\n")[-1]) - csv_ = get_trait_csv_sample_data(conn=conn, - trait_name=str(trait_name), - phenotype_id=str(phenotype_id)) - with open(uploaded_file_name, "w") as f_: - f_.write(csv_.split("\n\n")[-1]) - r = run_cmd(cmd=("csvdiff " - f"'{uploaded_file_name}' '{new_file_name}' " - "--format json")) - diff_output = (f"{TMPDIR}/sample-data/diffs/" - f"{trait_name}.{author.decode('utf-8')}." - f"{phenotype_id}.{current_time}.json") - with open(diff_output, "w") as f: - dict_ = json.loads(r.get("output")) - dict_.update({ - "author": author.decode('utf-8'), - "publishdata_id": publishdata_id, - "dataset_id": data_.get("dataset-name"), - "timestamp": datetime.datetime.now().strftime( - "%Y-%m-%d %H:%M:%S") - }) - f.write(json.dumps(dict_)) - flash("Sample-data has been successfully uploaded", "success") - # Run updates: - phenotype_ = { - "pre_pub_description": data_.get("pre-pub-desc"), - "post_pub_description": data_.get("post-pub-desc"), - "original_description": data_.get("orig-desc"), - "units": data_.get("units"), - "pre_pub_abbreviation": data_.get("pre-pub-abbrev"), - "post_pub_abbreviation": data_.get("post-pub-abbrev"), - "lab_code": data_.get("labcode"), - "submitter": data_.get("submitter"), - "owner": data_.get("owner"), - "authorized_users": data_.get("authorized-users"), - } - updated_phenotypes = update( - conn, "Phenotype", - data=Phenotype(**phenotype_), - where=Phenotype(id_=data_.get("phenotype-id"))) - diff_data = {} - if updated_phenotypes: - diff_data.update({"Phenotype": diff_from_dict(old={ - k: data_.get(f"old_{k}") for k, v in phenotype_.items() - if v is not None}, new=phenotype_)}) - publication_ = { - "abstract": data_.get("abstract"), - "authors": data_.get("authors"), - "title": data_.get("title"), - "journal": data_.get("journal"), - "volume": data_.get("volume"), - "pages": data_.get("pages"), - "month": data_.get("month"), - "year": data_.get("year") - } - updated_publications = update( - conn, "Publication", - data=Publication(**publication_), - where=Publication(id_=data_.get("pubmed-id", - data_.get("old_id_")))) - if updated_publications: - diff_data.update({"Publication": diff_from_dict(old={ - k: data_.get(f"old_{k}") for k, v in publication_.items() - if v is not None}, new=publication_)}) - if diff_data: - diff_data.update({"dataset_id": data_.get("dataset-name")}) - diff_data.update({"author": author.decode('utf-8')}) - diff_data.update({"timestamp": datetime.datetime.now().strftime( - "%Y-%m-%d %H:%M:%S")}) - insert(conn, - table="metadata_audit", - data=MetadataAudit(dataset_id=data_.get("dataset-name"), - editor=author.decode("utf-8"), - json_data=json.dumps(diff_data))) - flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success") - return redirect(f"/trait/{data_.get('dataset-name')}" - f"/edit/inbredset-id/{data_.get('inbred-set-id')}") - - @app.route("/probeset/update", methods=["POST"]) def update_probeset(): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), -- cgit v1.2.3 From e640c85bdb5174d86f2ba5adf112bbf9259fdc94 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 26 Oct 2021 22:38:05 +0300 Subject: edit_phenotype.html: Update form element for submitting POST data --- wqflask/wqflask/templates/edit_phenotype.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 7a841793..5166a903 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -62,8 +62,7 @@
    {% endif %} - - +

    Trait Information:

    @@ -226,7 +225,6 @@
    - -- cgit v1.2.3 From 0618b9f78d6ca2517f6265a9cfe7db64b6ae12ef Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:20:20 +0300 Subject: Default to an empty string if a session_id is empty "user_id" is jumbled up; it's either bytes or plain strings. This correctly parses it. --- wqflask/wqflask/decorators.py | 23 +++++++++++------------ wqflask/wqflask/metadata_edits.py | 5 ++--- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index a4ff7ce3..0c3c2a89 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -16,9 +16,9 @@ def login_required(f): """Use this for endpoints where login is required""" @wraps(f) def wrap(*args, **kwargs): - user_id = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or - g.user_session.record.get("user_id", "")) + user_id = ((g.user_session.record.get(b"user_id") or + b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") redis_conn = redis.from_url(current_app.config["REDIS_URL"], decode_responses=True) if not redis_conn.hget("users", user_id): @@ -38,15 +38,14 @@ def edit_access_required(f): resource_id = kwargs.get("resource_id") response: Dict = {} try: - _user_id = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or - g.user_session.record.get("user_id", "")) + user_id = ((g.user_session.record.get(b"user_id") or + b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") response = json.loads( requests.get(urljoin( current_app.config.get("GN2_PROXY"), ("available?resource=" - f"{resource_id}&user={_user_id}"))).content) - + f"{resource_id}&user={user_id}"))).content) except: response = {} if max([DataRole(role) for role in response.get( @@ -63,14 +62,14 @@ def edit_admins_access_required(f): resource_id: str = kwargs.get("resource_id", "") response: Dict = {} try: - _user_id = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or - g.user_session.record.get("user_id", "")) + user_id = ((g.user_session.record.get(b"user_id") or + b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") response = json.loads( requests.get(urljoin( current_app.config.get("GN2_PROXY"), ("available?resource=" - f"{resource_id}&user={_user_id}"))).content) + f"{resource_id}&user={user_id}"))).content) except: response = {} if max([AdminRole(role) for role in response.get( diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index bab1fa71..7e237976 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -162,9 +162,8 @@ def update_phenotype(dataset_id: str, name: str): host=current_app.config.get("DB_HOST")) data_ = request.form.to_dict() TMPDIR = current_app.config.get("TMPDIR") - author = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or - g.user_session.record.get("user_id", "")) + author = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") phenotype_id = str(data_.get('phenotype-id')) if 'file' not in request.files: flash("No sample-data has been uploaded", "warning") -- cgit v1.2.3 From fb7456a0e850290961c0f8be59c09fdd425f15f1 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:22:53 +0300 Subject: Rename metadata edit endpoints --- wqflask/wqflask/metadata_edits.py | 8 ++++---- wqflask/wqflask/templates/edit_phenotype.html | 2 +- wqflask/wqflask/templates/edit_probeset.html | 2 +- wqflask/wqflask/templates/show_trait_details.html | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 7e237976..08ff4704 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -116,7 +116,7 @@ def edit_probeset(conn, name): } -@metadata_edit.route("//traits//edit") +@metadata_edit.route("//traits/") @edit_access_required def display_phenotype_metadata(dataset_id: str, name: str): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), @@ -136,7 +136,7 @@ def display_phenotype_metadata(dataset_id: str, name: str): ) -@metadata_edit.route("/traits//edit") +@metadata_edit.route("/traits/") @edit_access_required def display_probeset_metadata(name: str): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), @@ -153,7 +153,7 @@ def display_probeset_metadata(name: str): ) -@metadata_edit.route("//traits//update", methods=("POST",)) +@metadata_edit.route("//traits/", methods=("POST",)) @edit_access_required def update_phenotype(dataset_id: str, name: str): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), @@ -276,5 +276,5 @@ def update_phenotype(dataset_id: str, name: str): editor=author, json_data=json.dumps(diff_data))) flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success") - return redirect(f"/datasets/{dataset_id}/traits/{name}/edit" + return redirect(f"/datasets/{dataset_id}/traits/{name}" f"?resource-id={request.args.get('resource-id')}") diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 5166a903..6b08163e 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -62,7 +62,7 @@
    {% endif %} - +

    Trait Information:

    diff --git a/wqflask/wqflask/templates/edit_probeset.html b/wqflask/wqflask/templates/edit_probeset.html index 85d49561..cd7b1379 100644 --- a/wqflask/wqflask/templates/edit_probeset.html +++ b/wqflask/wqflask/templates/edit_probeset.html @@ -53,7 +53,7 @@ Submit Trait | Reset {% endif %} - +Probeset Information:
    diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index b0639879..7a09f52e 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -237,11 +237,11 @@ {% if admin_status.get('metadata', Datarole.VIEW) > DataRole.VIEW %} {% if this_trait.dataset.type == 'Publish' %} {{ this_trait }} - + {% endif %} {% if this_trait.dataset.type == 'ProbeSet' %} - + {% endif %} {% if admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} -- cgit v1.2.3 From d15d381fd9018e9883d17046496250321327763b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:23:15 +0300 Subject: Fix typo --- wqflask/wqflask/templates/show_trait_details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 7a09f52e..b4b1860d 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -234,7 +234,7 @@ {% endif %} {% endif %} - {% if admin_status.get('metadata', Datarole.VIEW) > DataRole.VIEW %} + {% if admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} {% if this_trait.dataset.type == 'Publish' %} {{ this_trait }} -- cgit v1.2.3 From c125dfbbc34a78dda7b5b9a62763aacbf52f98ff Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:23:24 +0300 Subject: Use correct predicate to see if a resource belongs to anyone --- wqflask/wqflask/templates/admin/manage_resource.html | 2 +- wqflask/wqflask/views.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 613aa70e..9ca75906 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -8,7 +8,7 @@ {% set METADATA_ACCESS = access_role.get('metadata') %} {% set ADMIN_STATUS = access_role.get('admin') %}

    Resource Manager

    - {% if resource_info.get('owner_id') != 'none'%} + {% if resource_info.get('owner_id') %} {% set user_details = resource_info.get('owner_details') %}

    Current Owner: {{ user_details.get('full_name') }} diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 963e48b7..665aabb0 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -608,9 +608,8 @@ def export_perm_data(): @app.route("/show_temp_trait", methods=('POST',)) def show_temp_trait_page(): - user_id = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or - g.user_session.record.get("user_id", "")) + user_id = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") template_vars = show_trait.ShowTrait(user_id=user_id, kw=request.form) template_vars.js_data = json.dumps(template_vars.js_data, @@ -621,9 +620,8 @@ def show_temp_trait_page(): @app.route("/show_trait") def show_trait_page(): - user_id = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or - g.user_session.record.get("user_id", "")) + user_id = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") template_vars = show_trait.ShowTrait(user_id=user_id, kw=request.args) template_vars.js_data = json.dumps(template_vars.js_data, -- cgit v1.2.3 From c8938c9b3d9318f257083857aa9a0f79e3fbe220 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:25:26 +0300 Subject: manage_resource.html: Add missing opening tag --- wqflask/wqflask/templates/admin/manage_resource.html | 1 + 1 file changed, 1 insertion(+) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 9ca75906..5f6140f0 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -3,6 +3,7 @@ {% block content %}
    +
    {{ flash_me() }} {% set DATA_ACCESS = access_role.get('data') %} {% set METADATA_ACCESS = access_role.get('metadata') %} -- cgit v1.2.3 From 106a6a4932151347318fd22ccfe87c2fe2ac70d6 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:26:04 +0300 Subject: manage_resource.html: Set ADMIN_STATUS variable --- wqflask/wqflask/templates/admin/manage_resource.html | 1 + 1 file changed, 1 insertion(+) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 5f6140f0..46753f01 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -27,6 +27,7 @@ {% endif %} {% endif %}
    + {% set ADMIN_STATUS = access_role.get('admin') %}
    -- cgit v1.2.3 From ce864679a5724ce55caccdf9c520a81da52d8f75 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:26:24 +0300 Subject: show_trait_details.html: Delete debug variable --- wqflask/wqflask/templates/show_trait_details.html | 1 - 1 file changed, 1 deletion(-) diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index b4b1860d..fa892853 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -236,7 +236,6 @@ {% if admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} {% if this_trait.dataset.type == 'Publish' %} - {{ this_trait }} {% endif %} -- cgit v1.2.3 From d49536e8ec945d852a713d36e1b6db51d951c295 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:26:56 +0300 Subject: manage_resource.html: Apply indentation --- .../wqflask/templates/admin/manage_resource.html | 76 +++++++++++----------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 46753f01..64d4b6eb 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -5,29 +5,29 @@
    {{ flash_me() }} - {% set DATA_ACCESS = access_role.get('data') %} - {% set METADATA_ACCESS = access_role.get('metadata') %} - {% set ADMIN_STATUS = access_role.get('admin') %} -

    Resource Manager

    - {% if resource_info.get('owner_id') %} - {% set user_details = resource_info.get('owner_details') %} -

    - Current Owner: {{ user_details.get('full_name') }} -

    - {% if user_details.get('organization') %} -

    - Organization: {{ user_details.get('organization')}} -

    - {% endif %} - {% if DATA_ACCESS > DataRole.VIEW and ADMIN_STATUS > AdminRole.NOT_ADMIN %} -     - Change Owner - - {% endif %} - {% endif %} -
    + {% set DATA_ACCESS = access_role.get('data') %} + {% set METADATA_ACCESS = access_role.get('metadata') %} {% set ADMIN_STATUS = access_role.get('admin') %} + {% set ADMIN_STATUS = access_role.get('admin') %} +

    Resource Manager

    + {% if resource_info.get('owner_id') %} + {% set user_details = resource_info.get('owner_details') %} +

    + Current Owner: {{ user_details.get('full_name') }} +

    + {% if user_details.get('organization') %} +

    + Organization: {{ user_details.get('organization')}} +

    + {% endif %} + {% if DATA_ACCESS > DataRole.VIEW and ADMIN_STATUS > AdminRole.NOT_ADMIN %} + + Change Owner + + {% endif %} + {% endif %} +
    @@ -53,7 +53,7 @@ +

    @@ -100,25 +100,25 @@
    {% endif %} - + - + -{% endblock %} -{% block js %} + {% endblock %} + {% block js %} -{% endblock %} + {% endblock %} -- cgit v1.2.3 From 7547801f579ded3b8f94d98bc954a530d5275682 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:27:28 +0300 Subject: decorators.py: Auto-pep8 file --- wqflask/wqflask/decorators.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index 0c3c2a89..1ef8c188 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -28,7 +28,8 @@ def login_required(f): def edit_access_required(f): - """Use this for endpoints where people with admin or edit privileges are required""" + """Use this for endpoints where people with admin or edit privileges +are required""" @wraps(f) def wrap(*args, **kwargs): resource_id: str = "" -- cgit v1.2.3 From 4bc8c4d2b5fcfc164026e9a924b3b784af8957ea Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:28:12 +0300 Subject: Move endpoint for updating probeset to metadata_edits.py as a bp --- wqflask/wqflask/metadata_edits.py | 59 +++++++++++++++++++++++++++++++++++++++ wqflask/wqflask/views.py | 58 -------------------------------------- 2 files changed, 59 insertions(+), 58 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 08ff4704..ef9dc127 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -278,3 +278,62 @@ def update_phenotype(dataset_id: str, name: str): flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success") return redirect(f"/datasets/{dataset_id}/traits/{name}" f"?resource-id={request.args.get('resource-id')}") + + +@metadata_edit.route("/traits/", methods=["POST"]) +@edit_access_required +def update_probeset(name: str): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + data_ = request.form.to_dict() + probeset_ = { + "id_": data_.get("id"), + "symbol": data_.get("symbol"), + "description": data_.get("description"), + "probe_target_description": data_.get("probe_target_description"), + "chr_": data_.get("chr"), + "mb": data_.get("mb"), + "alias": data_.get("alias"), + "geneid": data_.get("geneid"), + "homologeneid": data_.get("homologeneid"), + "unigeneid": data_.get("unigeneid"), + "omim": data_.get("OMIM"), + "refseq_transcriptid": data_.get("refseq_transcriptid"), + "blatseq": data_.get("blatseq"), + "targetseq": data_.get("targetseq"), + "strand_probe": data_.get("Strand_Probe"), + "probe_set_target_region": data_.get("probe_set_target_region"), + "probe_set_specificity": data_.get("probe_set_specificity"), + "probe_set_blat_score": data_.get("probe_set_blat_score"), + "probe_set_blat_mb_start": data_.get("probe_set_blat_mb_start"), + "probe_set_blat_mb_end": data_.get("probe_set_blat_mb_end"), + "probe_set_strand": data_.get("probe_set_strand"), + "probe_set_note_by_rw": data_.get("probe_set_note_by_rw"), + "flag": data_.get("flag") + } + diff_data = {} + author = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") + if (updated_probeset := update( + conn, "ProbeSet", + data=Probeset(**probeset_), + where=Probeset(id_=data_.get("id")))): + diff_data.update({"Probeset": diff_from_dict(old={ + k: data_.get(f"old_{k}") for k, v in probeset_.items() + if v is not None}, new=probeset_)}) + if diff_data: + diff_data.update({"probeset_name": data_.get("probeset_name")}) + diff_data.update({"author": author}) + diff_data.update({"resource_id": request.args.get('resource-id')}) + diff_data.update({"timestamp": datetime.datetime.now().strftime( + "%Y-%m-%d %H:%M:%S")}) + insert(conn, + table="metadata_audit", + data=MetadataAudit(dataset_id=data_.get("id"), + editor=author.decode("utf-8"), + json_data=json.dumps(diff_data))) + return redirect(f"/datasets/traits/{name}" + f"?resource-id={request.args.get('resource-id')}") + diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 665aabb0..220d9b87 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -416,64 +416,6 @@ def submit_trait_form(): version=GN_VERSION) -@app.route("/probeset/update", methods=["POST"]) -def update_probeset(): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - data_ = request.form.to_dict() - probeset_ = { - "id_": data_.get("id"), - "symbol": data_.get("symbol"), - "description": data_.get("description"), - "probe_target_description": data_.get("probe_target_description"), - "chr_": data_.get("chr"), - "mb": data_.get("mb"), - "alias": data_.get("alias"), - "geneid": data_.get("geneid"), - "homologeneid": data_.get("homologeneid"), - "unigeneid": data_.get("unigeneid"), - "omim": data_.get("OMIM"), - "refseq_transcriptid": data_.get("refseq_transcriptid"), - "blatseq": data_.get("blatseq"), - "targetseq": data_.get("targetseq"), - "strand_probe": data_.get("Strand_Probe"), - "probe_set_target_region": data_.get("probe_set_target_region"), - "probe_set_specificity": data_.get("probe_set_specificity"), - "probe_set_blat_score": data_.get("probe_set_blat_score"), - "probe_set_blat_mb_start": data_.get("probe_set_blat_mb_start"), - "probe_set_blat_mb_end": data_.get("probe_set_blat_mb_end"), - "probe_set_strand": data_.get("probe_set_strand"), - "probe_set_note_by_rw": data_.get("probe_set_note_by_rw"), - "flag": data_.get("flag") - } - updated_probeset = update( - conn, "ProbeSet", - data=Probeset(**probeset_), - where=Probeset(id_=data_.get("id"))) - - diff_data = {} - author = (g.user_session.record.get(b"user_id", - b"").decode("utf-8") or - g.user_session.record.get("user_id", "")) - if updated_probeset: - diff_data.update({"Probeset": diff_from_dict(old={ - k: data_.get(f"old_{k}") for k, v in probeset_.items() - if v is not None}, new=probeset_)}) - if diff_data: - diff_data.update({"probeset_name": data_.get("probeset_name")}) - diff_data.update({"author": author.decode('utf-8')}) - diff_data.update({"timestamp": datetime.datetime.now().strftime( - "%Y-%m-%d %H:%M:%S")}) - insert(conn, - table="metadata_audit", - data=MetadataAudit(dataset_id=data_.get("id"), - editor=author.decode("utf-8"), - json_data=json.dumps(diff_data))) - return redirect(f"/trait/edit/probeset-name/{data_.get('probeset_name')}") - - @app.route("/create_temp_trait", methods=('POST',)) def create_temp_trait(): logger.info(request.url) -- cgit v1.2.3 From 77865c2ffbc921abb2fe04a64d03e51d47b2ecce Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 14:34:11 +0300 Subject: show_trait_details.html: Fix broken "edit-phenotype" link --- wqflask/wqflask/templates/show_trait_details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index fa892853..3e59a3ee 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -240,7 +240,7 @@ {% endif %} {% if this_trait.dataset.type == 'ProbeSet' %} - + {% endif %} {% if admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} -- cgit v1.2.3 From f7e18594b5a60ea966a38fc3424c32d104ce7a77 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 15:04:02 +0300 Subject: edit_probeset.html: Apply indentation --- wqflask/wqflask/templates/edit_probeset.html | 70 ++++++++++++++-------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/wqflask/wqflask/templates/edit_probeset.html b/wqflask/wqflask/templates/edit_probeset.html index cd7b1379..ab91b701 100644 --- a/wqflask/wqflask/templates/edit_probeset.html +++ b/wqflask/wqflask/templates/edit_probeset.html @@ -9,52 +9,52 @@ Submit Trait | Reset
    - -

    Update History

    -     - - - - - - - - - {% set ns = namespace(display_cell=True) %} + +

    Update History

    +     +
    TimestampEditorFieldDiff
    + + + + + + + + {% set ns = namespace(display_cell=True) %} - {% for timestamp, group in diff %} - {% set ns.display_cell = True %} - {% for i in group %} - - {% if ns.display_cell and i.timestamp == timestamp %} + {% for timestamp, group in diff %} + {% set ns.display_cell = True %} + {% for i in group %} + + {% if ns.display_cell and i.timestamp == timestamp %} - {% set author = i.author %} - {% set timestamp_ = i.timestamp %} + {% set author = i.author %} + {% set timestamp_ = i.timestamp %} - {% else %} + {% else %} - {% set author = "" %} - {% set timestamp_ = "" %} + {% set author = "" %} + {% set timestamp_ = "" %} - {% endif %} - - - - - {% set ns.display_cell = False %} - - {% endfor %} - {% endfor %} - -
    TimestampEditorFieldDiff
    {{ timestamp_ }}{{ author }}{{ i.diff.field }}
    {{ i.diff.diff }}
    + {% endif %} + {{ timestamp_ }} + {{ author }} + {{ i.diff.field }} + 
    {{ i.diff.diff }}
    + {% set ns.display_cell = False %} + + {% endfor %} + {% endfor %} + +
    {% endif %} -
    Probeset Information: + +

    Probeset Information:

    -- cgit v1.2.3 From b0946c079d0d4b953285c3f5781482a3d7a9f87d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 15:04:42 +0300 Subject: Fix html --- wqflask/wqflask/templates/edit_phenotype.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 6b08163e..c3cde391 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -62,7 +62,7 @@
    {% endif %} - +

    Trait Information:

    -- cgit v1.2.3 From 178aae274982b8d7bc7720310e53c60dc35b9701 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 15:04:57 +0300 Subject: Remove decoding when parsing author variable --- wqflask/wqflask/metadata_edits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index ef9dc127..b03c999e 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -332,7 +332,7 @@ def update_probeset(name: str): insert(conn, table="metadata_audit", data=MetadataAudit(dataset_id=data_.get("id"), - editor=author.decode("utf-8"), + editor=author, json_data=json.dumps(diff_data))) return redirect(f"/datasets/traits/{name}" f"?resource-id={request.args.get('resource-id')}") -- cgit v1.2.3 From bf71e24e7e471c99b650c5250d0ee9e692e4a6fd Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 15:05:18 +0300 Subject: Pass "name" to "edit_probeset.html" template --- wqflask/wqflask/metadata_edits.py | 1 + 1 file changed, 1 insertion(+) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index b03c999e..690f04c2 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -148,6 +148,7 @@ def display_probeset_metadata(name: str): "edit_probeset.html", diff=_d.get("diff"), probeset=_d.get("probeset"), + name=name, resource_id=request.args.get("resource-id"), version=os.environ.get("GN_VERSION"), ) -- cgit v1.2.3 From ea1de9bffd36dbd7417ff24b9a0caab1e3449081 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 15:05:47 +0300 Subject: Prefer tuples over lists --- wqflask/wqflask/metadata_edits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 690f04c2..d232b32b 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -281,7 +281,7 @@ def update_phenotype(dataset_id: str, name: str): f"?resource-id={request.args.get('resource-id')}") -@metadata_edit.route("/traits/", methods=["POST"]) +@metadata_edit.route("/traits/", methods=("POST",)) @edit_access_required def update_probeset(name: str): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), -- cgit v1.2.3 From 06f7358bbade1760cd7ac552ecc95ff896e46378 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 17:05:34 +0300 Subject: workflows: main.yml: Add GN_PROXY_URL and GN3_LOCAL_URL to CI --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8e2c7966..0cf4557f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,6 +44,8 @@ jobs: /gn2-profile/bin/screen -dm bash -c "env GN2_PROFILE=/gn2-profile \ TMPDIR=/tmp SERVER_PORT=5004 \ WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \ + GN_PROXY_URL='http://localhost:8080' \ + GN3_LOCAL_URL='http://localhost:8081' \ GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \ etc/default_settings.py" @@ -52,6 +54,8 @@ jobs: env GN2_PROFILE=/gn2-profile \ TMPDIR=/tmp SERVER_PORT=5004 \ WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \ + GN_PROXY_URL='http://localhost:8080' \ + GN3_LOCAL_URL='http://localhost:8081' \ GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \ etc/default_settings.py -c -m unittest discover -v -- cgit v1.2.3 From 67d71753259e3466b5672c759f22a105c8b653ef Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 27 Oct 2021 17:09:23 +0300 Subject: README.md: Add GN_PROXY_URL and GN3_LOCAL_URL to documentation --- README.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 003cfd7b..6921d299 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,11 @@ genenetwork2 A quick example is ```sh -env GN2_PROFILE=~/opt/gn-latest SERVER_PORT=5300 GENENETWORK_FILES=~/data/gn2_data/ ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev +env GN2_PROFILE=~/opt/gn-latest SERVER_PORT=5300 \ + GENENETWORK_FILES=~/data/gn2_data/ \ + GN_PROXY_URL="http://localhost:8080"\ + GN3_LOCAL_URL="http://localhost:8081"\ + ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev ``` For full examples (you may need to set a number of environment @@ -59,7 +63,12 @@ We are building 'Mechanical Rob' automated testing using Python which can be run with: ```sh -env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py -a http://localhost:5003 +env GN2_PROFILE=~/opt/gn-latest \ + ./bin/genenetwork2 \ + GN_PROXY_URL="http://localhost:8080" \ + GN3_LOCAL_URL="http://localhost:8081 "\ + ./etc/default_settings.py -c \ + ../test/requests/test-website.py -a http://localhost:5003 ``` The GN2_PROFILE is the Guix profile that contains all @@ -87,9 +96,9 @@ runcmd coverage html The `runcmd` and `runpython` are shell aliases defined in the following way: ```sh -alias runpython="env GN2_PROFILE=~/opt/gn-latest TMPDIR=/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ ./bin/genenetwork2 +alias runpython="env GN2_PROFILE=~/opt/gn-latest TMPDIR=/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 -alias runcmd="time env GN2_PROFILE=~/opt/gn-latest TMPDIR=//tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ ./bin/genenetwork2 ./etc/default_settings.py -cli" +alias runcmd="time env GN2_PROFILE=~/opt/gn-latest TMPDIR=//tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 ./etc/default_settings.py -cli" ``` Replace some of the env variables as per your use case. -- cgit v1.2.3 From 230184bf1484ed672bf66c29110bcb47e556f72f Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 28 Oct 2021 10:57:17 +0300 Subject: Remove the use of '@deprecated' This causes noisy logging. --- wqflask/utility/authentication_tools.py | 2 -- wqflask/utility/hmac.py | 3 --- wqflask/utility/redis_tools.py | 2 -- 3 files changed, 7 deletions(-) diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py index c4801c8c..afea69e1 100644 --- a/wqflask/utility/authentication_tools.py +++ b/wqflask/utility/authentication_tools.py @@ -1,7 +1,6 @@ import json import requests -from deprecated import deprecated from flask import g from base import webqtlConfig @@ -127,7 +126,6 @@ def check_owner(dataset=None, trait_id=None, resource_id=None): return False -@deprecated def check_owner_or_admin(dataset=None, trait_id=None, resource_id=None): if not resource_id: if dataset.type == "Temp": diff --git a/wqflask/utility/hmac.py b/wqflask/utility/hmac.py index d6e515ed..29891677 100644 --- a/wqflask/utility/hmac.py +++ b/wqflask/utility/hmac.py @@ -1,14 +1,11 @@ import hmac import hashlib -from deprecated import deprecated from flask import url_for from wqflask import app -@deprecated("This function leads to circular imports. " - "If possible use wqflask.decorators.create_hmac instead.") def hmac_creation(stringy): """Helper function to create the actual hmac""" diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py index c2a3b057..a6c5875f 100644 --- a/wqflask/utility/redis_tools.py +++ b/wqflask/utility/redis_tools.py @@ -4,7 +4,6 @@ import datetime import redis # used for collections -from deprecated import deprecated from utility.hmac import hmac_creation from utility.logger import getLogger logger = getLogger(__name__) @@ -252,7 +251,6 @@ def get_resource_id(dataset, trait_id=None): return resource_id -@deprecated def get_resource_info(resource_id): resource_info = Redis.hget("resources", resource_id) if resource_info: -- cgit v1.2.3 From 03caa57ad209f3bdd135be9d6516b94261c9b8de Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 28 Oct 2021 11:05:19 +0300 Subject: Remove all elasticsearch references in gn2 --- bin/genenetwork2 | 7 - doc/elasticsearch.org | 247 ------------------------------ test/requests/parametrized_test.py | 32 ---- test/requests/test-website.py | 1 - test/requests/test_forgot_password.py | 29 +--- test/requests/test_registration.py | 36 ++--- wqflask/maintenance/quantile_normalize.py | 18 --- wqflask/utility/elasticsearch_tools.py | 121 --------------- wqflask/utility/tools.py | 5 +- wqflask/wqflask/user_session.py | 1 - 10 files changed, 23 insertions(+), 474 deletions(-) delete mode 100644 doc/elasticsearch.org delete mode 100644 test/requests/parametrized_test.py delete mode 100644 wqflask/utility/elasticsearch_tools.py diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 2b94b2a2..5f714d2e 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -101,13 +101,6 @@ fi export GN2_SETTINGS=$settings # Python echo GN2_SETTINGS=$settings -# This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead -# if [ -z $ELASTICSEARCH_PROFILE ]; then -# echo -e "WARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE"; -# else -# PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python3.8/site-packages" -# fi - if [ -z $GN2_PROFILE ] ; then echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) diff --git a/doc/elasticsearch.org b/doc/elasticsearch.org deleted file mode 100644 index 864a8363..00000000 --- a/doc/elasticsearch.org +++ /dev/null @@ -1,247 +0,0 @@ -* Elasticsearch - -** Introduction - -GeneNetwork uses elasticsearch (ES) for all things considered -'state'. One example is user collections, another is user management. - -** Example - -To get the right environment, first you can get a python REPL with something like - -: env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ../etc/default_settings.py -cli python - -(make sure to use the correct GN2_PROFILE!) - -Next try - -#+BEGIN_SRC python - -from elasticsearch import Elasticsearch, TransportError - -es = Elasticsearch([{ "host": 'localhost', "port": '9200' }]) - -# Dump all data - -es.search("*") - -# To fetch an E-mail record from the users index - -record = es.search( - index = 'users', doc_type = 'local', body = { - "query": { "match": { "email_address": "myname@email.com" } } - }) - -# It is also possible to do wild card matching - -q = { "query": { "wildcard" : { "full_name" : "pjot*" } }} -es.search(index = 'users', doc_type = 'local', body = q) - -# To get elements from that record: - -record['hits']['hits'][0][u'_source']['full_name'] -u'Pjotr' - -record['hits']['hits'][0][u'_source']['email_address'] -u"myname@email.com" - -#+END_SRC - -** Health - -ES provides support for checking its health: - -: curl -XGET http://localhost:9200/_cluster/health?pretty=true - -#+BEGIN_SRC json - - - { - "cluster_name" : "asgard", - "status" : "yellow", - "timed_out" : false, - "number_of_nodes" : 1, - "number_of_data_nodes" : 1, - "active_primary_shards" : 5, - "active_shards" : 5, - "relocating_shards" : 0, - "initializing_shards" : 0, - "unassigned_shards" : 5 - } - -#+END_SRC - -Yellow means just one instance is running (no worries). - -To get full cluster info - -: curl -XGET "localhost:9200/_cluster/stats?human&pretty" - -#+BEGIN_SRC json -{ - "_nodes" : { - "total" : 1, - "successful" : 1, - "failed" : 0 - }, - "cluster_name" : "elasticsearch", - "timestamp" : 1529050366452, - "status" : "yellow", - "indices" : { - "count" : 3, - "shards" : { - "total" : 15, - "primaries" : 15, - "replication" : 0.0, - "index" : { - "shards" : { - "min" : 5, - "max" : 5, - "avg" : 5.0 - }, - "primaries" : { - "min" : 5, - "max" : 5, - "avg" : 5.0 - }, - "replication" : { - "min" : 0.0, - "max" : 0.0, - "avg" : 0.0 - } - } - }, - "docs" : { - "count" : 14579, - "deleted" : 0 - }, - "store" : { - "size" : "44.7mb", - "size_in_bytes" : 46892794 - }, - "fielddata" : { - "memory_size" : "0b", - "memory_size_in_bytes" : 0, - "evictions" : 0 - }, - "query_cache" : { - "memory_size" : "0b", - "memory_size_in_bytes" : 0, - "total_count" : 0, - "hit_count" : 0, - "miss_count" : 0, - "cache_size" : 0, - "cache_count" : 0, - "evictions" : 0 - }, - "completion" : { - "size" : "0b", - "size_in_bytes" : 0 - }, - "segments" : { - "count" : 24, - "memory" : "157.3kb", - "memory_in_bytes" : 161112, - "terms_memory" : "122.6kb", - "terms_memory_in_bytes" : 125569, - "stored_fields_memory" : "15.3kb", - "stored_fields_memory_in_bytes" : 15728, - "term_vectors_memory" : "0b", - "term_vectors_memory_in_bytes" : 0, - "norms_memory" : "10.8kb", - "norms_memory_in_bytes" : 11136, - "points_memory" : "111b", - "points_memory_in_bytes" : 111, - "doc_values_memory" : "8.3kb", - "doc_values_memory_in_bytes" : 8568, - "index_writer_memory" : "0b", - "index_writer_memory_in_bytes" : 0, - "version_map_memory" : "0b", - "version_map_memory_in_bytes" : 0, - "fixed_bit_set" : "0b", - "fixed_bit_set_memory_in_bytes" : 0, - "max_unsafe_auto_id_timestamp" : -1, - "file_sizes" : { } - } - }, - "nodes" : { - "count" : { - "total" : 1, - "data" : 1, - "coordinating_only" : 0, - "master" : 1, - "ingest" : 1 - }, - "versions" : [ - "6.2.1" - ], - "os" : { - "available_processors" : 16, - "allocated_processors" : 16, - "names" : [ - { - "name" : "Linux", - "count" : 1 - } - ], - "mem" : { - "total" : "125.9gb", - "total_in_bytes" : 135189286912, - "free" : "48.3gb", - "free_in_bytes" : 51922628608, - "used" : "77.5gb", - "used_in_bytes" : 83266658304, - "free_percent" : 38, - "used_percent" : 62 - } - }, - "process" : { - "cpu" : { - "percent" : 0 - }, - "open_file_descriptors" : { - "min" : 415, - "max" : 415, - "avg" : 415 - } - }, - "jvm" : { - "max_uptime" : "1.9d", - "max_uptime_in_millis" : 165800616, - "versions" : [ - { - "version" : "9.0.4", - "vm_name" : "OpenJDK 64-Bit Server VM", - "vm_version" : "9.0.4+11", - "vm_vendor" : "Oracle Corporation", - "count" : 1 - } - ], - "mem" : { - "heap_used" : "1.1gb", - "heap_used_in_bytes" : 1214872032, - "heap_max" : "23.8gb", - "heap_max_in_bytes" : 25656426496 - }, - "threads" : 110 - }, - "fs" : { - "total" : "786.4gb", - "total_in_bytes" : 844400918528, - "free" : "246.5gb", - "free_in_bytes" : 264688160768, - "available" : "206.5gb", - "available_in_bytes" : 221771468800 - }, - "plugins" : [ ], - "network_types" : { - "transport_types" : { - "netty4" : 1 - }, - "http_types" : { - "netty4" : 1 - } - } - } -} -#+BEGIN_SRC json diff --git a/test/requests/parametrized_test.py b/test/requests/parametrized_test.py deleted file mode 100644 index 50003850..00000000 --- a/test/requests/parametrized_test.py +++ /dev/null @@ -1,32 +0,0 @@ -import logging -import unittest -from wqflask import app -from utility.elasticsearch_tools import get_elasticsearch_connection, get_user_by_unique_column -from elasticsearch import Elasticsearch, TransportError - -class ParametrizedTest(unittest.TestCase): - - def __init__(self, methodName='runTest', gn2_url="http://localhost:5003", es_url="localhost:9200"): - super(ParametrizedTest, self).__init__(methodName=methodName) - self.gn2_url = gn2_url - self.es_url = es_url - - def setUp(self): - self.es = get_elasticsearch_connection() - self.es_cleanup = [] - - es_logger = logging.getLogger("elasticsearch") - es_logger.setLevel(app.config.get("LOG_LEVEL")) - es_logger.addHandler( - logging.FileHandler("/tmp/es_TestRegistrationInfo.log")) - es_trace_logger = logging.getLogger("elasticsearch.trace") - es_trace_logger.addHandler( - logging.FileHandler("/tmp/es_TestRegistrationTrace.log")) - - def tearDown(self): - from time import sleep - self.es.delete_by_query( - index="users" - , doc_type="local" - , body={"query":{"match":{"email_address":"test@user.com"}}}) - sleep(1) diff --git a/test/requests/test-website.py b/test/requests/test-website.py index 8bfb47c2..d619a7d5 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -43,7 +43,6 @@ def dummy(args_obj, parser): def integration_tests(args_obj, parser): gn2_url = args_obj.host - es_url = app.config.get("ELASTICSEARCH_HOST")+":"+str(app.config.get("ELASTICSEARCH_PORT")) run_integration_tests(gn2_url, es_url) def initTest(klass, gn2_url, es_url): diff --git a/test/requests/test_forgot_password.py b/test/requests/test_forgot_password.py index 346524bc..65b061f8 100644 --- a/test/requests/test_forgot_password.py +++ b/test/requests/test_forgot_password.py @@ -1,25 +1,22 @@ import requests -from utility.elasticsearch_tools import get_user_by_unique_column from parameterized import parameterized from parametrized_test import ParametrizedTest passwork_reset_link = '' forgot_password_page = None -class TestForgotPassword(ParametrizedTest): +class TestForgotPassword(ParametrizedTest): def setUp(self): super(TestForgotPassword, self).setUp() self.forgot_password_url = self.gn2_url+"/n/forgot_password_submit" + def send_email(to_addr, msg, fromaddr="no-reply@genenetwork.org"): print("CALLING: send_email_mock()") email_data = { - "to_addr": to_addr - , "msg": msg - , "fromaddr": from_addr} + "to_addr": to_addr, "msg": msg, "fromaddr": from_addr} data = { - "es_connection": self.es, "email_address": "test@user.com", "full_name": "Test User", "organization": "Test Organisation", @@ -27,24 +24,12 @@ class TestForgotPassword(ParametrizedTest): "password_confirm": "test_password" } - def testWithoutEmail(self): data = {"email_address": ""} - error_notification = '
    You MUST provide an email
    ' + error_notification = ('
    ' + 'You MUST provide an email
    ') result = requests.post(self.forgot_password_url, data=data) self.assertEqual(result.url, self.gn2_url+"/n/forgot_password") self.assertTrue( - result.content.find(error_notification) >= 0 - , "Error message should be displayed but was not") - - def testWithNonExistingEmail(self): - # Monkey patching doesn't work, so simply test that getting by email - # returns the correct data - user = get_user_by_unique_column(self.es, "email_address", "non-existent@domain.com") - self.assertTrue(user is None, "Should not find non-existent user") - - def testWithExistingEmail(self): - # Monkey patching doesn't work, so simply test that getting by email - # returns the correct data - user = get_user_by_unique_column(self.es, "email_address", "test@user.com") - self.assertTrue(user is not None, "Should find user") + result.content.find(error_notification) >= 0, + "Error message should be displayed but was not") diff --git a/test/requests/test_registration.py b/test/requests/test_registration.py index 0047e8a6..5d08bf58 100644 --- a/test/requests/test_registration.py +++ b/test/requests/test_registration.py @@ -1,31 +1,25 @@ import sys import requests -from parametrized_test import ParametrizedTest class TestRegistration(ParametrizedTest): - def tearDown(self): - for item in self.es_cleanup: - self.es.delete(index="users", doc_type="local", id=item["_id"]) def testRegistrationPage(self): - if self.es.ping(): - data = { - "email_address": "test@user.com", - "full_name": "Test User", - "organization": "Test Organisation", - "password": "test_password", - "password_confirm": "test_password" - } - requests.post(self.gn2_url+"/n/register", data) - response = self.es.search( - index="users" - , doc_type="local" - , body={ - "query": {"match": {"email_address": "test@user.com"}}}) - self.assertEqual(len(response["hits"]["hits"]), 1) - else: - self.skipTest("The elasticsearch server is down") + data = { + "email_address": "test@user.com", + "full_name": "Test User", + "organization": "Test Organisation", + "password": "test_password", + "password_confirm": "test_password" + } + requests.post(self.gn2_url+"/n/register", data) + response = self.es.search( + index="users" + , doc_type="local" + , body={ + "query": {"match": {"email_address": "test@user.com"}}}) + self.assertEqual(len(response["hits"]["hits"]), 1) + def main(gn2, es): import unittest diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py index 0cc963e5..32780ca6 100644 --- a/wqflask/maintenance/quantile_normalize.py +++ b/wqflask/maintenance/quantile_normalize.py @@ -5,14 +5,10 @@ import urllib.parse import numpy as np import pandas as pd -from elasticsearch import Elasticsearch, TransportError -from elasticsearch.helpers import bulk from flask import Flask, g, request from wqflask import app -from utility.elasticsearch_tools import get_elasticsearch_connection -from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT, SQL_URI def parse_db_uri(): @@ -106,20 +102,6 @@ if __name__ == '__main__': Conn = MySQLdb.Connect(**parse_db_uri()) Cursor = Conn.cursor() - # es = Elasticsearch([{ - # "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT - # }], timeout=60) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None - - es = get_elasticsearch_connection(for_user=False) - - #input_filename = "/home/zas1024/cfw_data/" + sys.argv[1] + ".txt" - #input_df = create_dataframe(input_filename) - #output_df = quantileNormalize(input_df) - - #output_df.to_csv('quant_norm.csv', sep='\t') - - #out_filename = sys.argv[1][:-4] + '_quantnorm.txt' - success, _ = bulk(es, set_data(sys.argv[1])) response = es.search( diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py deleted file mode 100644 index eae3ba03..00000000 --- a/wqflask/utility/elasticsearch_tools.py +++ /dev/null @@ -1,121 +0,0 @@ -# Elasticsearch support -# -# Some helpful commands to view the database: -# -# You can test the server being up with -# -# curl -H 'Content-Type: application/json' http://localhost:9200 -# -# List all indices -# -# curl -H 'Content-Type: application/json' 'localhost:9200/_cat/indices?v' -# -# To see the users index 'table' -# -# curl http://localhost:9200/users -# -# To list all user ids -# -# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d ' -# { -# "query" : { -# "match_all" : {} -# }, -# "stored_fields": [] -# }' -# -# To view a record -# -# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d ' -# { -# "query" : { -# "match" : { "email_address": "pjotr2017@thebird.nl"} -# } -# }' -# -# -# To delete the users index and data (dangerous!) -# -# curl -XDELETE -H 'Content-Type: application/json' 'localhost:9200/users' - - -from elasticsearch import Elasticsearch, TransportError -import logging - -from utility.logger import getLogger -logger = getLogger(__name__) - -from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT - - -def test_elasticsearch_connection(): - es = Elasticsearch(['http://' + ELASTICSEARCH_HOST + \ - ":" + str(ELASTICSEARCH_PORT) + '/'], verify_certs=True) - if not es.ping(): - logger.warning("Elasticsearch is DOWN") - - -def get_elasticsearch_connection(for_user=True): - """Return a connection to ES. Returns None on failure""" - logger.info("get_elasticsearch_connection") - es = None - try: - assert(ELASTICSEARCH_HOST) - assert(ELASTICSEARCH_PORT) - logger.info("ES HOST", ELASTICSEARCH_HOST) - - es = Elasticsearch([{ - "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT - }], timeout=30, retry_on_timeout=True) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None - - if for_user: - setup_users_index(es) - - es_logger = logging.getLogger("elasticsearch") - es_logger.setLevel(logging.INFO) - es_logger.addHandler(logging.NullHandler()) - except Exception as e: - logger.error("Failed to get elasticsearch connection", e) - es = None - - return es - - -def setup_users_index(es_connection): - if es_connection: - index_settings = { - "properties": { - "email_address": { - "type": "keyword"}}} - - es_connection.indices.create(index='users', ignore=400) - es_connection.indices.put_mapping( - body=index_settings, index="users", doc_type="local") - - -def get_user_by_unique_column(es, column_name, column_value, index="users", doc_type="local"): - return get_item_by_unique_column(es, column_name, column_value, index=index, doc_type=doc_type) - - -def save_user(es, user, user_id): - es_save_data(es, "users", "local", user, user_id) - - -def get_item_by_unique_column(es, column_name, column_value, index, doc_type): - item_details = None - try: - response = es.search( - index=index, doc_type=doc_type, body={ - "query": {"match": {column_name: column_value}} - }) - if len(response["hits"]["hits"]) > 0: - item_details = response["hits"]["hits"][0]["_source"] - except TransportError as te: - pass - return item_details - - -def es_save_data(es, index, doc_type, data_item, data_id,): - from time import sleep - es.create(index, doc_type, body=data_item, id=data_id) - sleep(1) # Delay 1 second to allow indexing diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 0efe8ca9..f28961ec 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -287,6 +287,7 @@ JS_GN_PATH = get_setting('JS_GN_PATH') GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') +GITHUB_AUTH_URL = "" if GITHUB_CLIENT_ID != 'UNKNOWN' and GITHUB_CLIENT_SECRET: GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id=" + \ GITHUB_CLIENT_ID + "&client_secret=" + GITHUB_CLIENT_SECRET @@ -301,10 +302,6 @@ if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET: "&redirect_uri=" + GN2_BRANCH_URL + "n/login/orcid_oauth2" ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') -ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') -ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') -# import utility.elasticsearch_tools as es -# es.test_elasticsearch_connection() SMTP_CONNECT = get_setting('SMTP_CONNECT') SMTP_USERNAME = get_setting('SMTP_USERNAME') diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py index 67e2e158..d3c4a62f 100644 --- a/wqflask/wqflask/user_session.py +++ b/wqflask/wqflask/user_session.py @@ -10,7 +10,6 @@ from flask import (Flask, g, render_template, url_for, request, make_response, from wqflask import app from utility import hmac -#from utility.elasticsearch_tools import get_elasticsearch_connection from utility.redis_tools import get_redis_conn, get_user_id, get_user_by_unique_column, set_user_attribute, get_user_collections, save_collections Redis = get_redis_conn() -- cgit v1.2.3 From 44bf8ee8463fe7182282ec10e259d4c9eb7526b8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 28 Oct 2021 11:14:42 +0300 Subject: Remove tests that refer to deleted "ParametrizedTest" "ParametrizedTest" references ElasticSearch. --- test/requests/test_forgot_password.py | 35 --------------------- test/requests/test_login_github.py | 47 ----------------------------- test/requests/test_login_local.py | 57 ----------------------------------- test/requests/test_login_orcid.py | 47 ----------------------------- 4 files changed, 186 deletions(-) delete mode 100644 test/requests/test_forgot_password.py delete mode 100644 test/requests/test_login_github.py delete mode 100644 test/requests/test_login_local.py delete mode 100644 test/requests/test_login_orcid.py diff --git a/test/requests/test_forgot_password.py b/test/requests/test_forgot_password.py deleted file mode 100644 index 65b061f8..00000000 --- a/test/requests/test_forgot_password.py +++ /dev/null @@ -1,35 +0,0 @@ -import requests -from parameterized import parameterized -from parametrized_test import ParametrizedTest - -passwork_reset_link = '' -forgot_password_page = None - - -class TestForgotPassword(ParametrizedTest): - def setUp(self): - super(TestForgotPassword, self).setUp() - self.forgot_password_url = self.gn2_url+"/n/forgot_password_submit" - - def send_email(to_addr, msg, fromaddr="no-reply@genenetwork.org"): - print("CALLING: send_email_mock()") - email_data = { - "to_addr": to_addr, "msg": msg, "fromaddr": from_addr} - - data = { - "email_address": "test@user.com", - "full_name": "Test User", - "organization": "Test Organisation", - "password": "test_password", - "password_confirm": "test_password" - } - - def testWithoutEmail(self): - data = {"email_address": ""} - error_notification = ('
    ' - 'You MUST provide an email
    ') - result = requests.post(self.forgot_password_url, data=data) - self.assertEqual(result.url, self.gn2_url+"/n/forgot_password") - self.assertTrue( - result.content.find(error_notification) >= 0, - "Error message should be displayed but was not") diff --git a/test/requests/test_login_github.py b/test/requests/test_login_github.py deleted file mode 100644 index 1bf4f695..00000000 --- a/test/requests/test_login_github.py +++ /dev/null @@ -1,47 +0,0 @@ -import uuid -import requests -from time import sleep -from wqflask import app -from parameterized import parameterized -from parametrized_test import ParametrizedTest - -login_link_text = 'Sign in' -logout_link_text = 'Sign out' -uid = str(uuid.uuid4()) - -class TestLoginGithub(ParametrizedTest): - - def setUp(self): - super(TestLoginGithub, self).setUp() - data = { - "user_id": uid - , "name": "A. T. Est User" - , "github_id": 693024 - , "user_url": "https://fake-github.com/atestuser" - , "login_type": "github" - , "organization": "" - , "active": 1 - , "confirmed": 1 - } - self.es.create(index="users", doc_type="local", body=data, id=uid) - sleep(1) - - def tearDown(self): - super(TestLoginGithub, self).tearDown() - self.es.delete(index="users", doc_type="local", id=uid) - - def testLoginUrl(self): - login_button_text = 'Login with Github' - result = requests.get(self.gn2_url+"/n/login") - index = result.content.find(login_button_text) - self.assertTrue(index >= 0, "Should have found `Login with Github` button") - - @parameterized.expand([ - ("1234", login_link_text, "Login should have failed with non-existing user") - , (uid, logout_link_text, "Login should have been successful with existing user") - ]) - def testLogin(self, test_uid, expected, message): - url = self.gn2_url+"/n/login?type=github&uid="+test_uid - result = requests.get(url) - index = result.content.find(expected) - self.assertTrue(index >= 0, message) diff --git a/test/requests/test_login_local.py b/test/requests/test_login_local.py deleted file mode 100644 index 6691d135..00000000 --- a/test/requests/test_login_local.py +++ /dev/null @@ -1,57 +0,0 @@ -import requests -from parameterized import parameterized -from parametrized_test import ParametrizedTest - -login_link_text = 'Sign in' -logout_link_text = 'Sign out' - -class TestLoginLocal(ParametrizedTest): - - def setUp(self): - super(TestLoginLocal, self).setUp() - self.login_url = self.gn2_url +"/n/login" - data = { - "es_connection": self.es, - "email_address": "test@user.com", - "full_name": "Test User", - "organization": "Test Organisation", - "password": "test_password", - "password_confirm": "test_password" - } - - - @parameterized.expand([ - ( - { - "email_address": "non@existent.email", - "password": "doesitmatter?" - }, login_link_text, "Login should have failed with the wrong user details."), - ( - { - "email_address": "test@user.com", - "password": "test_password" - }, logout_link_text, "Login should have been successful with correct user details and neither import_collections nor remember_me set"), - ( - { - "email_address": "test@user.com", - "password": "test_password", - "import_collections": "y" - }, logout_link_text, "Login should have been successful with correct user details and only import_collections set"), - ( - { - "email_address": "test@user.com", - "password": "test_password", - "remember_me": "y" - }, logout_link_text, "Login should have been successful with correct user details and only remember_me set"), - ( - { - "email_address": "test@user.com", - "password": "test_password", - "remember_me": "y", - "import_collections": "y" - }, logout_link_text, "Login should have been successful with correct user details, and both remember_me, and import_collections set") - ]) - def testLogin(self, data, expected, message): - result = requests.post(self.login_url, data=data) - index = result.content.find(expected) - self.assertTrue(index >= 0, message) diff --git a/test/requests/test_login_orcid.py b/test/requests/test_login_orcid.py deleted file mode 100644 index ea15642e..00000000 --- a/test/requests/test_login_orcid.py +++ /dev/null @@ -1,47 +0,0 @@ -import uuid -import requests -from time import sleep -from wqflask import app -from parameterized import parameterized -from parametrized_test import ParametrizedTest - -login_link_text = 'Sign in' -logout_link_text = 'Sign out' -uid = str(uuid.uuid4()) - -class TestLoginOrcid(ParametrizedTest): - - def setUp(self): - super(TestLoginOrcid, self).setUp() - data = { - "user_id": uid - , "name": "A. T. Est User" - , "orcid": 345872 - , "user_url": "https://fake-orcid.org/atestuser" - , "login_type": "orcid" - , "organization": "" - , "active": 1 - , "confirmed": 1 - } - self.es.create(index="users", doc_type="local", body=data, id=uid) - sleep(1) - - def tearDown(self): - super(TestLoginOrcid, self).tearDown() - self.es.delete(index="users", doc_type="local", id=uid) - - def testLoginUrl(self): - login_button_text = 'a href="https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=' + app.config.get("ORCID_CLIENT_ID") + '&client_secret=' + app.config.get("ORCID_CLIENT_SECRET") + '" title="Login with ORCID" class="btn btn-info btn-group">Login with ORCID' - result = requests.get(self.gn2_url+"/n/login") - index = result.content.find(login_button_text) - self.assertTrue(index >= 0, "Should have found `Login with ORCID` button") - - @parameterized.expand([ - ("1234", login_link_text, "Login should have failed with non-existing user") - , (uid, logout_link_text, "Login should have been successful with existing user") - ]) - def testLogin(self, test_uid, expected, message): - url = self.gn2_url+"/n/login?type=orcid&uid="+test_uid - result = requests.get(url) - index = result.content.find(expected) - self.assertTrue(index >= 0, message) -- cgit v1.2.3 From 24480fad2d67a6e35b03997e23c6d849a60623a0 Mon Sep 17 00:00:00 2001 From: Arthur Centeno Date: Fri, 29 Oct 2021 17:31:17 +0000 Subject: updating tutorials and header on base 10-29-21a --- wqflask/wqflask/templates/base.html | 2 +- wqflask/wqflask/templates/tutorials.html | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index e62f571d..4a6674c6 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -68,7 +68,7 @@
    • References
      • -
    • Tutorials/Primers
      • +
    • Webinars, Tutorials/Primers
    • Blogs
    • Glossary of Term
    • FAQ
      • diff --git a/wqflask/wqflask/templates/tutorials.html b/wqflask/wqflask/templates/tutorials.html index 81fb5f5a..8f3447e3 100644 --- a/wqflask/wqflask/templates/tutorials.html +++ b/wqflask/wqflask/templates/tutorials.html @@ -17,25 +17,25 @@ - + - + - -
      - +
      - +

      -- cgit v1.2.3 From a59dee1304936407c3561c626d26992c3bfa5d1a Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 29 Oct 2021 20:17:15 +0000 Subject: Limit PCA trait table to something reasonable and set checkbox/index column widths --- wqflask/wqflask/templates/correlation_matrix.html | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/correlation_matrix.html b/wqflask/wqflask/templates/correlation_matrix.html index 8275f1dd..3da6981c 100644 --- a/wqflask/wqflask/templates/correlation_matrix.html +++ b/wqflask/wqflask/templates/correlation_matrix.html @@ -73,8 +73,13 @@
      {% if pca_works == "True" %}

      PCA Traits

      -
      +
      + + + + + -- cgit v1.2.3 From 09b3299bb00303f66ada401673f6684288182b6f Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 2 Nov 2021 17:52:46 +0000 Subject: Removed deferRender since it causes problems with Scroller (functions like scrolling don't work properly) --- wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js index 96ef7a04..6c72449f 100644 --- a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js +++ b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js @@ -167,9 +167,6 @@ if (js_data.sample_lists.length > 1){ } function loadDataTable(first_run=false, table_id, table_data){ - - console.log("COL DEFS:", columnDefs) - if (!first_run){ setUserColumnsDefWidths(table_id); } @@ -222,7 +219,6 @@ function loadDataTable(first_run=false, table_id, table_data){ "sDom": "iti", "destroy": true, "autoWidth": false, - "deferRender": true, "bSortClasses": false, "scrollY": "100vh", "scrollCollapse": true, -- cgit v1.2.3 From e98289020b0cf2aea19d55a15fd3ae61b8eb441e Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 2 Nov 2021 17:59:23 +0000 Subject: Removed the event for changing a sample value from initComplete and just put it in show_trait.js, since apparently the code in initComplete doesn't run with current DataTables settings --- .../wqflask/static/new/javascript/initialize_show_trait_tables.js | 5 ----- wqflask/wqflask/static/new/javascript/show_trait.js | 4 ++++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js index 6c72449f..e1026f8c 100644 --- a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js +++ b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js @@ -178,11 +178,6 @@ function loadDataTable(first_run=false, table_id, table_data){ } table_settings = { - 'initComplete': function(settings, json) { - $('.edit_sample_value').change(function() { - edit_data_change(); - }); - }, 'createdRow': function ( row, data, index ) { $(row).attr('id', table_type + "_" + data.this_id) $(row).addClass("value_se"); diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index f050d4ae..6b81b47e 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -1770,6 +1770,10 @@ $('#filter_by_value').click(function(){ edit_data_change(); }) +$('.edit_sample_value').change(function() { + edit_data_change(); +}); + $('#exclude_group').click(edit_data_change); $('#block_outliers').click(edit_data_change); $('#reset').click(edit_data_change); -- cgit v1.2.3 From 678127848128b5ded02df501b60c8edb4d036d76 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Fri, 22 Oct 2021 15:23:32 +0300 Subject: function to cache sample results --- wqflask/base/data_set.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 8906ab69..8188d90c 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -20,7 +20,7 @@ from dataclasses import dataclass from dataclasses import field from dataclasses import InitVar -from typing import Optional, Dict +from typing import Optional, Dict, List from db.call import fetchall, fetchone, fetch1 from utility.logger import getLogger from utility.tools import USE_GN_SERVER, USE_REDIS, flat_files, flat_file_exists, GN2_BASE_URL @@ -397,7 +397,8 @@ class DatasetGroup: self.parlist = [maternal, paternal] def get_study_samplelists(self): - study_sample_file = locate_ignore_error(self.name + ".json", 'study_sample_lists') + study_sample_file = locate_ignore_error( + self.name + ".json", 'study_sample_lists') try: f = open(study_sample_file) except: @@ -447,7 +448,6 @@ class DatasetGroup: # genotype_1 is Dataset Object without parents and f1 # genotype_2 is Dataset Object with parents and f1 (not for intercross) - # reaper barfs on unicode filenames, so here we ensure it's a string if self.genofile: if "RData" in self.genofile: # ZS: This is a temporary fix; I need to change the way the JSON files that point to multiple genotype files are structured to point to other file types like RData @@ -726,7 +726,6 @@ class DataSet: data_results = self.chunk_dataset(query_results, len(sample_ids)) self.samplelist = sorted_samplelist self.trait_data = data_results - def get_trait_data(self, sample_list=None): if sample_list: @@ -1242,3 +1241,27 @@ def geno_mrna_confidentiality(ob): if confidential: return True + + +def check_if_dataset_modified(dataset_name, cached_timestamp): + """function to check if the dataset has been modified""" + last_modified = "query results" + return (cached_timestamp == last_modified_timestamp) + + +def cache_dataset_results(dataset_name: str, query_results: List): + """function to cache dataset query results to file""" + # check if file exists clear if it does + # aslo check for the timestamp + # hash for unique name ??? are dataset name unique + # data computations actions + # store the file path on redis + + # hash functiob + + file_path = os.path.join(TMPDIR, f"{dataset_name}.json") + + query_results = [list(results) for result in query_results] + + with open(file_path, "w") as file_handler: + json.dump(query_results, file_handler) \ No newline at end of file -- cgit v1.2.3 From c83dc9f45d6df6faa7d95916a998ded9be3167fa Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Fri, 22 Oct 2021 15:26:20 +0300 Subject: fetch cached results --- wqflask/base/data_set.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 8188d90c..51244303 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -1243,7 +1243,7 @@ def geno_mrna_confidentiality(ob): return True -def check_if_dataset_modified(dataset_name, cached_timestamp): +def check_if_dataset_modified(dataset_name: str, cached_timestamp): """function to check if the dataset has been modified""" last_modified = "query results" return (cached_timestamp == last_modified_timestamp) @@ -1264,4 +1264,18 @@ def cache_dataset_results(dataset_name: str, query_results: List): query_results = [list(results) for result in query_results] with open(file_path, "w") as file_handler: - json.dump(query_results, file_handler) \ No newline at end of file + json.dump(query_results, file_handler) + +def fetch_cached_results(dataset_name: str): + """function to fetch the cached results""" + file_path = os.path.join(TMPDIR, f"{dataset_name}.json") + + try: + with open(file_path) as file_handler: + data = json.load(file_handler) + # check if table has been modified + if check_if_dataset_modified(dataset_name["timestamp"]): + return data[dataset_name] + except FileNotFoundError: + # take actions continue to fetch dataset results and fetch results + pass \ No newline at end of file -- cgit v1.2.3 From d64ebf750a50c7bc16e86dc5b16ca5ca45434b34 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Fri, 22 Oct 2021 17:30:39 +0300 Subject: generate filename hash --- wqflask/base/data_set.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 51244303..11ed2495 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -1243,10 +1243,12 @@ def geno_mrna_confidentiality(ob): return True -def check_if_dataset_modified(dataset_name: str, cached_timestamp): - """function to check if the dataset has been modified""" - last_modified = "query results" - return (cached_timestamp == last_modified_timestamp) +def generate_hash_file(dataset_name: str, dataset_timestamp: str): + """given the trait_name generate a unique name for this""" + + string_unicode = f"{dataset_name}{dataset_timestamp}".encode() + md5hash = hashlib.md5(str2hash.encode(string_unicode)) + return md5hash.hexdigest() def cache_dataset_results(dataset_name: str, query_results: List): @@ -1259,23 +1261,27 @@ def cache_dataset_results(dataset_name: str, query_results: List): # hash functiob - file_path = os.path.join(TMPDIR, f"{dataset_name}.json") + file_name = generate_hash_file(dataset_name, "dataset_timestamp") + + file_path = os.path.join(TMPDIR, f"{file_name}.json") query_results = [list(results) for result in query_results] with open(file_path, "w") as file_handler: json.dump(query_results, file_handler) + def fetch_cached_results(dataset_name: str): """function to fetch the cached results""" - file_path = os.path.join(TMPDIR, f"{dataset_name}.json") + + file_name = generate_hash_file(dataset_name,) + file_path = os.path.join(TMPDIR, f"{file_path}.json") try: with open(file_path) as file_handler: data = json.load(file_handler) # check if table has been modified - if check_if_dataset_modified(dataset_name["timestamp"]): - return data[dataset_name] + return data except FileNotFoundError: # take actions continue to fetch dataset results and fetch results - pass \ No newline at end of file + pass -- cgit v1.2.3 From 61d94ec99d0cfd1cb3a77d64629479d978e833e1 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sat, 23 Oct 2021 04:20:52 +0300 Subject: pep8 formatting ,minor fixes --- wqflask/base/data_set.py | 115 +++++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 54 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 11ed2495..1a208050 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -39,6 +39,7 @@ from db import webqtlDatabaseFunction from base import species from base import webqtlConfig from flask import Flask, g +from base.webqtlConfig import TMPDIR import os import math import string @@ -50,6 +51,7 @@ import requests import gzip import pickle as pickle import itertools +import hashlib from redis import Redis @@ -751,54 +753,64 @@ class DataSet: # Postgres doesn't have that limit, so we can get rid of this after we transition chunk_size = 50 number_chunks = int(math.ceil(len(sample_ids) / chunk_size)) - trait_sample_data = [] - for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): - if self.type == "Publish": - dataset_type = "Phenotype" - else: - dataset_type = self.type - temp = ['T%s.value' % item for item in sample_ids_step] - if self.type == "Publish": - query = "SELECT {}XRef.Id,".format(escape(self.type)) - else: - query = "SELECT {}.Name,".format(escape(dataset_type)) - data_start_pos = 1 - query += ', '.join(temp) - query += ' FROM ({}, {}XRef, {}Freeze) '.format(*mescape(dataset_type, - self.type, - self.type)) - - for item in sample_ids_step: - query += """ - left join {}Data as T{} on T{}.Id = {}XRef.DataId - and T{}.StrainId={}\n - """.format(*mescape(self.type, item, item, self.type, item, item)) - - if self.type == "Publish": - query += """ - WHERE {}XRef.InbredSetId = {}Freeze.InbredSetId - and {}Freeze.Name = '{}' - and {}.Id = {}XRef.{}Id - order by {}.Id - """.format(*mescape(self.type, self.type, self.type, self.name, - dataset_type, self.type, dataset_type, dataset_type)) - else: - query += """ - WHERE {}XRef.{}FreezeId = {}Freeze.Id - and {}Freeze.Name = '{}' - and {}.Id = {}XRef.{}Id - order by {}.Id - """.format(*mescape(self.type, self.type, self.type, self.type, - self.name, dataset_type, self.type, self.type, dataset_type)) + cached_results = fetch_cached_results(self.name) + # cached_results = None + if cached_results is None: + trait_sample_data = [] + for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): + if self.type == "Publish": + dataset_type = "Phenotype" + else: + dataset_type = self.type + temp = ['T%s.value' % item for item in sample_ids_step] + if self.type == "Publish": + query = "SELECT {}XRef.Id,".format(escape(self.type)) + else: + query = "SELECT {}.Name,".format(escape(dataset_type)) + data_start_pos = 1 + query += ', '.join(temp) + query += ' FROM ({}, {}XRef, {}Freeze) '.format(*mescape(dataset_type, + self.type, + self.type)) + + for item in sample_ids_step: + query += """ + left join {}Data as T{} on T{}.Id = {}XRef.DataId + and T{}.StrainId={}\n + """.format(*mescape(self.type, item, item, self.type, item, item)) + + if self.type == "Publish": + query += """ + WHERE {}XRef.InbredSetId = {}Freeze.InbredSetId + and {}Freeze.Name = '{}' + and {}.Id = {}XRef.{}Id + order by {}.Id + """.format(*mescape(self.type, self.type, self.type, self.name, + dataset_type, self.type, dataset_type, dataset_type)) + else: + query += """ + WHERE {}XRef.{}FreezeId = {}Freeze.Id + and {}Freeze.Name = '{}' + and {}.Id = {}XRef.{}Id + order by {}.Id + """.format(*mescape(self.type, self.type, self.type, self.type, + self.name, dataset_type, self.type, self.type, dataset_type)) - results = g.db.execute(query).fetchall() - trait_sample_data.append(results) + results = g.db.execute(query).fetchall() + trait_sample_data.append([list(result) for result in results]) + + cache_dataset_results( + self.name, "cached_time_stamp", trait_sample_data) + + else: + trait_sample_data = cached_results trait_count = len(trait_sample_data[0]) self.trait_data = collections.defaultdict(list) # put all of the separate data together into a dictionary where the keys are # trait names and values are lists of sample values + data_start_pos = 1 for trait_counter in range(trait_count): trait_name = trait_sample_data[0][trait_counter][0] for chunk_counter in range(int(number_chunks)): @@ -1247,39 +1259,34 @@ def generate_hash_file(dataset_name: str, dataset_timestamp: str): """given the trait_name generate a unique name for this""" string_unicode = f"{dataset_name}{dataset_timestamp}".encode() - md5hash = hashlib.md5(str2hash.encode(string_unicode)) + md5hash = hashlib.md5(string_unicode) return md5hash.hexdigest() -def cache_dataset_results(dataset_name: str, query_results: List): +def cache_dataset_results(dataset_name: str, dataset_timestamp: str, query_results: List): """function to cache dataset query results to file""" - # check if file exists clear if it does - # aslo check for the timestamp - # hash for unique name ??? are dataset name unique # data computations actions # store the file path on redis - # hash functiob - file_name = generate_hash_file(dataset_name, "dataset_timestamp") + file_name = generate_hash_file(dataset_name, dataset_timestamp) file_path = os.path.join(TMPDIR, f"{file_name}.json") - query_results = [list(results) for result in query_results] - with open(file_path, "w") as file_handler: json.dump(query_results, file_handler) def fetch_cached_results(dataset_name: str): """function to fetch the cached results""" + # store the timestamp in redis - file_name = generate_hash_file(dataset_name,) - file_path = os.path.join(TMPDIR, f"{file_path}.json") - + file_name = generate_hash_file(dataset_name, "cached_time_stamp") + file_path = os.path.join(TMPDIR, f"{file_name}.json") try: - with open(file_path) as file_handler: + with open(file_path, "r") as file_handler: data = json.load(file_handler) + # print(file_handler) # check if table has been modified return data except FileNotFoundError: -- cgit v1.2.3 From 09c1d0a93673965ffcf0fd93deff9b6ba5b84690 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 26 Oct 2021 12:54:15 +0300 Subject: init function to get the table timestamp --- wqflask/base/data_set.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 1a208050..131702b8 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -1255,6 +1255,21 @@ def geno_mrna_confidentiality(ob): return True +def query_table_timestamp(dataset_type: str): + """function to query the update timestamp of a given dataset_type""" + + # computation data and actions + + query_update_time = """ + SELECT UPDATE_TIME FROM information_schema.tables + WHERE TABLE_SCHEMA = 'db_webqtl' + AND TABLE_NAME = 'ProbeSetData' + """ + + # store the timestamp in redis + return g.db.execute(query_update_time).fetchone() + + def generate_hash_file(dataset_name: str, dataset_timestamp: str): """given the trait_name generate a unique name for this""" @@ -1268,7 +1283,6 @@ def cache_dataset_results(dataset_name: str, dataset_timestamp: str, query_resul # data computations actions # store the file path on redis - file_name = generate_hash_file(dataset_name, dataset_timestamp) file_path = os.path.join(TMPDIR, f"{file_name}.json") @@ -1285,10 +1299,8 @@ def fetch_cached_results(dataset_name: str): file_path = os.path.join(TMPDIR, f"{file_name}.json") try: with open(file_path, "r") as file_handler: - data = json.load(file_handler) - # print(file_handler) - # check if table has been modified - return data + + return json.load(file_handler) except FileNotFoundError: # take actions continue to fetch dataset results and fetch results pass -- cgit v1.2.3 From ab237042317e6c1708ef803679efdbd9c046867b Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 26 Oct 2021 13:57:35 +0300 Subject: store table timestamp on redis --- wqflask/base/data_set.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 131702b8..ef4b1e13 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -52,6 +52,7 @@ import gzip import pickle as pickle import itertools import hashlib +import datetime from redis import Redis @@ -753,7 +754,7 @@ class DataSet: # Postgres doesn't have that limit, so we can get rid of this after we transition chunk_size = 50 number_chunks = int(math.ceil(len(sample_ids) / chunk_size)) - cached_results = fetch_cached_results(self.name) + cached_results = fetch_cached_results(self.name, self.type) # cached_results = None if cached_results is None: trait_sample_data = [] @@ -800,7 +801,7 @@ class DataSet: trait_sample_data.append([list(result) for result in results]) cache_dataset_results( - self.name, "cached_time_stamp", trait_sample_data) + self.name, self.type, trait_sample_data) else: trait_sample_data = cached_results @@ -1260,42 +1261,48 @@ def query_table_timestamp(dataset_type: str): # computation data and actions - query_update_time = """ + query_update_time = f""" SELECT UPDATE_TIME FROM information_schema.tables WHERE TABLE_SCHEMA = 'db_webqtl' - AND TABLE_NAME = 'ProbeSetData' + AND TABLE_NAME = '{dataset_type}Data' """ - # store the timestamp in redis - return g.db.execute(query_update_time).fetchone() + # store the timestamp in redis= + date_time_obj = g.db.execute(query_update_time).fetchone()[0] + + f = "%Y-%m-%d %H:%M:%S" + return date_time_obj.strftime(f) def generate_hash_file(dataset_name: str, dataset_timestamp: str): """given the trait_name generate a unique name for this""" - string_unicode = f"{dataset_name}{dataset_timestamp}".encode() md5hash = hashlib.md5(string_unicode) return md5hash.hexdigest() -def cache_dataset_results(dataset_name: str, dataset_timestamp: str, query_results: List): +def cache_dataset_results(dataset_name: str, dataset_type: str, query_results: List): """function to cache dataset query results to file""" # data computations actions # store the file path on redis - file_name = generate_hash_file(dataset_name, dataset_timestamp) + table_timestamp = query_table_timestamp(dataset_type) + results = r.set(f"{dataset_type}timestamp", table_timestamp) + + file_name = generate_hash_file(dataset_name, table_timestamp) file_path = os.path.join(TMPDIR, f"{file_name}.json") with open(file_path, "w") as file_handler: json.dump(query_results, file_handler) -def fetch_cached_results(dataset_name: str): +def fetch_cached_results(dataset_name: str, dataset_type: str): """function to fetch the cached results""" - # store the timestamp in redis - file_name = generate_hash_file(dataset_name, "cached_time_stamp") + table_timestamp = r.get(f"{dataset_type}timestamp").decode("utf-8") + + file_name = generate_hash_file(dataset_name, table_timestamp) file_path = os.path.join(TMPDIR, f"{file_name}.json") try: with open(file_path, "r") as file_handler: -- cgit v1.2.3 From 3c773fc2b1123c116f35ab80e91dee531b6d3d2d Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 26 Oct 2021 16:30:59 +0300 Subject: cache traits metadata --- wqflask/wqflask/correlation/show_corr_results.py | 35 +++++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index d73965da..42010a1e 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -19,9 +19,12 @@ # This module is used by GeneNetwork project (www.genenetwork.org) import json +import os +from pathlib import Path from base.trait import create_trait, jsonable from base.data_set import create_dataset +from base.webqtlConfig import TMPDIR from utility import hmac @@ -82,13 +85,31 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe corr_results = correlation_data['correlation_results'] results_list = [] + + file_name = f"{target_dataset['name']}_metadata.json" + + file_path = os.path.join(TMPDIR, file_name) + new_traits_metadata = {} + + try: + with open(file_path,"r+") as file_handler: + dataset_metadata = json.load(file_handler) + + except FileNotFoundError: + Path(file_path).touch(exist_ok=True) + dataset_metadata = {} + for i, trait_dict in enumerate(corr_results): trait_name = list(trait_dict.keys())[0] trait = trait_dict[trait_name] - target_trait_ob = create_trait(dataset=target_dataset_ob, - name=trait_name, - get_qtl_info=True) - target_trait = jsonable(target_trait_ob, target_dataset_ob) + + target_trait = dataset_metadata.get(trait_name) + if target_trait is None: + target_trait_ob = create_trait(dataset=target_dataset_ob, + name=trait_name, + get_qtl_info=True) + target_trait = jsonable(target_trait_ob, target_dataset_ob) + new_traits_metadata[trait_name] = target_trait if target_trait['view'] == False: continue results_dict = {} @@ -163,6 +184,12 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe results_list.append(results_dict) + + if bool(new_traits_metadata): + # that means new traits exists + dataset_metadata.update(new_traits_metadata) + with open(file_path,"w+") as file_handler: + json.dump(dataset_metadata, file_handler) return json.dumps(results_list) -- cgit v1.2.3 From 7bb86ff7e0b89fd714077a6d6b01f9b17f2ce953 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 27 Oct 2021 15:59:36 +0300 Subject: check for null timestamp --- wqflask/base/data_set.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index ef4b1e13..70c58136 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -1263,7 +1263,7 @@ def query_table_timestamp(dataset_type: str): query_update_time = f""" SELECT UPDATE_TIME FROM information_schema.tables - WHERE TABLE_SCHEMA = 'db_webqtl' + WHERE TABLE_SCHEMA = 'db_webqtl_s' AND TABLE_NAME = '{dataset_type}Data' """ @@ -1300,7 +1300,12 @@ def cache_dataset_results(dataset_name: str, dataset_type: str, query_results: L def fetch_cached_results(dataset_name: str, dataset_type: str): """function to fetch the cached results""" - table_timestamp = r.get(f"{dataset_type}timestamp").decode("utf-8") + table_timestamp = r.get(f"{dataset_type}timestamp") + + if table_timestamp is not None: + table_timestamp = table_timestamp.decode("utf-8") + else: + table_timestamp = "" file_name = generate_hash_file(dataset_name, table_timestamp) file_path = os.path.join(TMPDIR, f"{file_name}.json") -- cgit v1.2.3 From fbbdca01308610813aa7f77deb120e95aed8df6e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 28 Oct 2021 12:49:20 +0300 Subject: Redirect to "no_access_page" for pages that need edit and edit-admin --- wqflask/wqflask/decorators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index 1ef8c188..319d9bd4 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -1,7 +1,7 @@ """This module contains gn2 decorators""" import redis -from flask import current_app, g, request +from flask import current_app, g, redirect, request, url_for from typing import Dict from urllib.parse import urljoin from functools import wraps @@ -51,7 +51,7 @@ are required""" response = {} if max([DataRole(role) for role in response.get( "data", ["no-access"])]) < DataRole.EDIT: - return "You need to have edit access", 401 + return redirect(url_for("no_access_page")) return f(*args, **kwargs) return wrap @@ -75,6 +75,6 @@ def edit_admins_access_required(f): response = {} if max([AdminRole(role) for role in response.get( "admin", ["not-admin"])]) < AdminRole.EDIT_ADMINS: - return "You need to have edit-admins access", 401 + return redirect(url_for("no_access_page")) return f(*args, **kwargs) return wrap -- cgit v1.2.3 From 77845700752ff3283f40f0293e3b6d90c0b86d0c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 28 Oct 2021 12:51:27 +0300 Subject: Delete noisy debugging Logging happens before every request, twice! Remove these. --- wqflask/wqflask/user_session.py | 1 - wqflask/wqflask/views.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py index d3c4a62f..cef50cd9 100644 --- a/wqflask/wqflask/user_session.py +++ b/wqflask/wqflask/user_session.py @@ -22,7 +22,6 @@ THIRTY_DAYS = 60 * 60 * 24 * 30 @app.before_request def get_user_session(): - logger.info("@app.before_request get_session") g.user_session = UserSession() # ZS: I think this should solve the issue of deleting the cookie and redirecting to the home page when a user's session has expired if not g.user_session: diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 220d9b87..f65dd143 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -109,17 +109,14 @@ logger = utility.logger.getLogger(__name__) @app.before_request def connect_db(): - logger.info("@app.before_request connect_db") db = getattr(g, '_database', None) if db is None: g.db = g._database = sqlalchemy.create_engine( SQL_URI, encoding="latin1") - logger.debug(g.db) @app.before_request def check_access_permissions(): - logger.debug("@app.before_request check_access_permissions") if 'dataset' in request.args: permissions = DEFAULT_PRIVILEGES if request.args['dataset'] != "Temp": @@ -145,7 +142,6 @@ def check_access_permissions(): def shutdown_session(exception=None): db = getattr(g, '_database', None) if db is not None: - logger.debug("remove db_session") db_session.remove() g.db = None -- cgit v1.2.3 From 482c19955b5b757ad6b79102c832678ae7339d20 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 28 Oct 2021 12:52:22 +0300 Subject: Delete "check_access_permissions" This check happens before every single request! Remove this. --- wqflask/wqflask/views.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index f65dd143..3a84141d 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -115,29 +115,6 @@ def connect_db(): SQL_URI, encoding="latin1") -@app.before_request -def check_access_permissions(): - if 'dataset' in request.args: - permissions = DEFAULT_PRIVILEGES - if request.args['dataset'] != "Temp": - dataset = create_dataset(request.args['dataset']) - - if dataset.type == "Temp": - permissions = DEFAULT_PRIVILEGES - elif 'trait_id' in request.args: - permissions = check_resource_availability( - dataset, request.args['trait_id']) - elif dataset.type != "Publish": - permissions = check_resource_availability(dataset) - - if type(permissions['data']) is list: - if 'view' not in permissions['data']: - return redirect(url_for("no_access_page")) - else: - if permissions['data'] == 'no-access': - return redirect(url_for("no_access_page")) - - @app.teardown_appcontext def shutdown_session(exception=None): db = getattr(g, '_database', None) -- cgit v1.2.3 From 28f67d85416c48c2196dff889bfab8e9659aa5b6 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 28 Oct 2021 12:53:26 +0300 Subject: Add "login_required" decorator to metadata edit endpoints --- wqflask/wqflask/metadata_edits.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index d232b32b..fad9c315 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -11,6 +11,7 @@ from flask import (Blueprint, current_app, redirect, from itertools import groupby from wqflask.decorators import edit_access_required +from wqflask.decorators import login_required from gn3.db import diff_from_dict from gn3.db import fetchall @@ -118,6 +119,7 @@ def edit_probeset(conn, name): @metadata_edit.route("//traits/") @edit_access_required +@login_required def display_phenotype_metadata(dataset_id: str, name: str): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), @@ -138,6 +140,7 @@ def display_phenotype_metadata(dataset_id: str, name: str): @metadata_edit.route("/traits/") @edit_access_required +@login_required def display_probeset_metadata(name: str): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), @@ -156,6 +159,7 @@ def display_probeset_metadata(name: str): @metadata_edit.route("//traits/", methods=("POST",)) @edit_access_required +@login_required def update_phenotype(dataset_id: str, name: str): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), @@ -283,6 +287,7 @@ def update_phenotype(dataset_id: str, name: str): @metadata_edit.route("/traits/", methods=("POST",)) @edit_access_required +@login_required def update_probeset(name: str): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), -- cgit v1.2.3 From c9e27583997dfb9ad99914ca7609b44e513c14b7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 28 Oct 2021 13:28:51 +0300 Subject: Move "get_sample_data_as_csv" to metadata_edits.py --- wqflask/wqflask/metadata_edits.py | 17 ++++++++++++++++- wqflask/wqflask/views.py | 16 ---------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index fad9c315..85f45592 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -7,7 +7,7 @@ import difflib from collections import namedtuple from flask import (Blueprint, current_app, redirect, - flash, g, render_template, request) + flash, g, render_template, request, Response) from itertools import groupby from wqflask.decorators import edit_access_required @@ -343,3 +343,18 @@ def update_probeset(name: str): return redirect(f"/datasets/traits/{name}" f"?resource-id={request.args.get('resource-id')}") + +@metadata_edit.route("//traits//csv") +def get_sample_data_as_csv(dataset_id: str, phenotype_id: int): + return Response( + get_trait_csv_sample_data( + conn=MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")), + trait_name=str(dataset_id), + phenotype_id=str(phenotype_id)), + mimetype="text/csv", + headers={"Content-disposition": + "attachment; filename=myplot.csv"} + ) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 3a84141d..3e53e4c5 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1055,22 +1055,6 @@ def json_default_handler(obj): type(obj), repr(obj))) -@app.route("/trait//sampledata/") -def get_sample_data_as_csv(trait_name: int, phenotype_id: int): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - csv_ = get_trait_csv_sample_data(conn, str(trait_name), - str(phenotype_id)) - return Response( - csv_, - mimetype="text/csv", - headers={"Content-disposition": - "attachment; filename=myplot.csv"} - ) - - @app.route("/admin/data-sample/diffs/") @edit_access_required def display_diffs_admin(): -- cgit v1.2.3 From 8ec8a9c4a62dfaf9c674926349f2e09b083711a4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 28 Oct 2021 13:29:25 +0300 Subject: edit_phenotype.html: Update endpoint for downloading csv --- wqflask/wqflask/templates/edit_phenotype.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index c3cde391..0daea51d 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -217,7 +217,7 @@ -- cgit v1.2.3 From 6d496d5734f48ed8477ad0666af77c377a9185df Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 28 Oct 2021 15:40:01 +0300 Subject: metadata_edits.py: Embed resource-id in a given file-name csv --- wqflask/wqflask/metadata_edits.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 85f45592..c28f5681 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -170,12 +170,10 @@ def update_phenotype(dataset_id: str, name: str): author = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") or g.user_session.record.get("user_id") or "") phenotype_id = str(data_.get('phenotype-id')) - if 'file' not in request.files: + if not (file_ := request.files.get("file")): flash("No sample-data has been uploaded", "warning") else: - file_ = request.files['file'] - SAMPLE_DATADIR = os.path.join(TMPDIR, "sample-data") - if not os.path.exists(SAMPLE_DATADIR): + if not os.path.exists(SAMPLE_DATADIR := os.path.join(TMPDIR, "sample-data")): os.makedirs(SAMPLE_DATADIR) if not os.path.exists(os.path.join(SAMPLE_DATADIR, "diffs")): @@ -189,7 +187,7 @@ def update_phenotype(dataset_id: str, name: str): new_file_name = (os.path.join(TMPDIR, "sample-data/updated/", (f"{author}." - f"{name}.{phenotype_id}." + f"{request.args.get('resource-id')}." f"{current_time}.csv"))) uploaded_file_name = (os.path.join( TMPDIR, -- cgit v1.2.3 From 1c456c5a394d4ba77ccc1858b11798ecac062e97 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 28 Oct 2021 17:03:21 +0300 Subject: Move authentication logic to gn3 --- wqflask/wqflask/__init__.py | 2 +- wqflask/wqflask/access_roles.py | 30 -------------- wqflask/wqflask/decorators.py | 4 +- wqflask/wqflask/resource_manager.py | 68 ++++---------------------------- wqflask/wqflask/show_trait/show_trait.py | 8 ++-- 5 files changed, 14 insertions(+), 98 deletions(-) delete mode 100644 wqflask/wqflask/access_roles.py diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index 169192c7..7d46fbad 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -9,7 +9,7 @@ from typing import Tuple from urllib.parse import urlparse from utility import formatting -from wqflask.access_roles import DataRole, AdminRole +from gn3.authentication import DataRole, AdminRole from wqflask.resource_manager import resource_management from wqflask.metadata_edits import metadata_edit diff --git a/wqflask/wqflask/access_roles.py b/wqflask/wqflask/access_roles.py deleted file mode 100644 index 6cffbc81..00000000 --- a/wqflask/wqflask/access_roles.py +++ /dev/null @@ -1,30 +0,0 @@ -import functools -from enum import Enum, unique - - -@functools.total_ordering -class OrderedEnum(Enum): - @classmethod - @functools.lru_cache(None) - def _member_list(cls): - return list(cls) - - def __lt__(self, other): - if self.__class__ is other.__class__: - member_list = self.__class__._member_list() - return member_list.index(self) < member_list.index(other) - return NotImplemented - - -@unique -class DataRole(OrderedEnum): - NO_ACCESS = "no-access" - VIEW = "view" - EDIT = "edit" - - -@unique -class AdminRole(OrderedEnum): - NOT_ADMIN = "not-admin" - EDIT_ACCESS = "edit-access" - EDIT_ADMINS = "edit-admins" diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index 319d9bd4..41d23084 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -5,8 +5,8 @@ from flask import current_app, g, redirect, request, url_for from typing import Dict from urllib.parse import urljoin from functools import wraps -from wqflask.access_roles import AdminRole -from wqflask.access_roles import DataRole +from gn3.authentication import AdminRole +from gn3.authentication import DataRole import json import requests diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index e338a22d..c0717314 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -11,11 +11,15 @@ from flask import render_template from flask import request from flask import url_for +from gn3.authentication import AdminRole +from gn3.authentication import DataRole +from gn3.authentication import get_user_membership +from gn3.authentication import get_highest_user_access_role + from typing import Dict, Tuple from urllib.parse import urljoin -from wqflask.access_roles import AdminRole -from wqflask.access_roles import DataRole + from wqflask.decorators import edit_access_required from wqflask.decorators import edit_admins_access_required from wqflask.decorators import login_required @@ -24,64 +28,6 @@ from wqflask.decorators import login_required resource_management = Blueprint('resource_management', __name__) -def get_user_membership(conn: redis.Redis, user_id: str, - group_id: str) -> Dict: - """Return a dictionary that indicates whether the `user_id` is a - member or admin of `group_id`. - - Args: - - conn: a Redis Connection with the responses decoded. - - user_id: a user's unique id - e.g. '8ad942fe-490d-453e-bd37-56f252e41603' - - group_id: a group's unique id - e.g. '7fa95d07-0e2d-4bc5-b47c-448fdc1260b2' - - Returns: - A dict indicating whether the user is an admin or a member of - the group: {"member": True, "admin": False} - - """ - results = {"member": False, "admin": False} - for key, value in conn.hgetall('groups').items(): - if key == group_id: - group_info = json.loads(value) - if user_id in group_info.get("admins"): - results["admin"] = True - if user_id in group_info.get("members"): - results["member"] = True - break - return results - - -def get_user_access_roles( - resource_id: str, - user_id: str, - gn_proxy_url: str = "http://localhost:8080") -> Dict: - """Get the highest access roles for a given user - - Args: - - resource_id: The unique id of a given resource. - - user_id: The unique id of a given user. - - gn_proxy_url: The URL where gn-proxy is running. - - Returns: - A dict indicating the highest access role the user has. - - """ - role_mapping = {} - for x, y in zip(DataRole, AdminRole): - role_mapping.update({x.value: x, }) - role_mapping.update({y.value: y, }) - access_role = {} - for key, value in json.loads( - requests.get(urljoin( - gn_proxy_url, - ("available?resource=" - f"{resource_id}&user={user_id}"))).content).items(): - access_role[key] = max(map(lambda x: role_mapping[x], value)) - return access_role - - def add_extra_resource_metadata(conn: redis.Redis, resource_id: str, resource: Dict) -> Dict: @@ -144,7 +90,7 @@ def view_resource(resource_id: str): conn=redis_conn, resource_id=resource_id, resource=json.loads(resource))), - access_role=get_user_access_roles( + access_role=get_highest_user_access_role( resource_id=resource_id, user_id=user_id, gn_proxy_url=current_app.config.get("GN2_PROXY"))) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index fa1206c9..6020bc16 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -23,9 +23,9 @@ from utility.tools import locate_ignore_error from utility.tools import GN_PROXY_URL from utility.redis_tools import get_redis_conn, get_resource_id -from wqflask.access_roles import AdminRole -from wqflask.access_roles import DataRole -from wqflask.resource_manager import get_user_access_roles +from gn3.authentication import AdminRole +from gn3.authentication import DataRole +from gn3.authentication import get_highest_user_access_role Redis = get_redis_conn() ONE_YEAR = 60 * 60 * 24 * 365 @@ -73,7 +73,7 @@ class ShowTrait: self.trait_vals = Redis.get(self.trait_id).split() self.resource_id = get_resource_id(self.dataset, self.trait_id) - self.admin_status = get_user_access_roles( + self.admin_status = get_highest_user_access_role( user_id=user_id, resource_id=(self.resource_id or ""), gn_proxy_url=GN_PROXY_URL) -- cgit v1.2.3 From 4d676728786c18ff10048f15613b8fe9b66a7edc Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 29 Oct 2021 08:30:08 +0300 Subject: Remove "author" from file name when storing csv --- wqflask/wqflask/metadata_edits.py | 48 +++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index c28f5681..31d056b4 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -3,7 +3,7 @@ import os import json import datetime import difflib - +import redis from collections import namedtuple from flask import (Blueprint, current_app, redirect, @@ -184,17 +184,13 @@ def update_phenotype(dataset_id: str, name: str): os.makedirs(os.path.join(SAMPLE_DATADIR, "updated")) current_time = str(datetime.datetime.now().isoformat()) + _file_name = (f"{request.args.get('resource-id')}." + f"{current_time}.csv") new_file_name = (os.path.join(TMPDIR, - "sample-data/updated/", - (f"{author}." - f"{request.args.get('resource-id')}." - f"{current_time}.csv"))) + f"sample-data/updated/{_file_name}.csv")) uploaded_file_name = (os.path.join( - TMPDIR, - "sample-data/updated/", - (f"updated.{author}." - f"{request.args.get('resource-id')}." - f"{current_time}.csv"))) + TMPDIR, "sample-data/updated/updated.", + f"{_file_name}.csv")) file_.save(new_file_name) publishdata_id = "" lines = [] @@ -213,8 +209,7 @@ def update_phenotype(dataset_id: str, name: str): f"'{uploaded_file_name}' '{new_file_name}' " "--format json")) diff_output = (f"{TMPDIR}/sample-data/diffs/" - f"{author}.{request.args.get('resource-id')}." - f"{current_time}.json") + f"{_file_name}.json") with open(diff_output, "w") as f: dict_ = json.loads(r.get("output")) dict_.update({ @@ -343,6 +338,7 @@ def update_probeset(name: str): @metadata_edit.route("//traits//csv") +@login_required def get_sample_data_as_csv(dataset_id: str, phenotype_id: int): return Response( get_trait_csv_sample_data( @@ -356,3 +352,31 @@ def get_sample_data_as_csv(dataset_id: str, phenotype_id: int): headers={"Content-disposition": "attachment; filename=myplot.csv"} ) + + +# @metadata_edit.route("/diffs") +# @login_required +# def display_diffs_admin(): +# TMPDIR = current_app.config.get("TMPDIR") +# DIFF_DIR = f"{TMPDIR}/sample-data/diffs" +# approved_files, rejected_files, files = {}, {}, {} +# user_id = ((g.user_session.record.get(b"user_id") or +# b"").decode("utf-8") +# or g.user_session.record.get("user_id") or "") +# redis_conn = redis.from_url(current_app.config["REDIS_URL"], +# decode_responses=True) +# if os.path.exists(DIFF_DIR): +# for name in os.listdir(DIFF_DIR): +# if name.endswith(".approved"): + +# approved_file.update{ + +# } +# approved_files.add(name) +# elif name.endswith(".rejected"): +# approved_files.add(name) +# else: +# files.append(name) +# return render_template("display_files_admin.html", +# files=files) +# http://localhost:5004/datasets/diffs/admin -- cgit v1.2.3 From b3a9b90343f70584b82df00a42cbcc5b97dbfc7f Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 1 Nov 2021 09:22:23 +0300 Subject: use gn3 imports --- wqflask/tests/unit/wqflask/test_resource_manager.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/wqflask/tests/unit/wqflask/test_resource_manager.py b/wqflask/tests/unit/wqflask/test_resource_manager.py index 01fb2021..f4335425 100644 --- a/wqflask/tests/unit/wqflask/test_resource_manager.py +++ b/wqflask/tests/unit/wqflask/test_resource_manager.py @@ -3,10 +3,10 @@ import json import unittest from unittest import mock -from wqflask.resource_manager import get_user_membership -from wqflask.resource_manager import get_user_access_roles -from wqflask.resource_manager import DataRole -from wqflask.resource_manager import AdminRole +from gn3.authentication import get_user_membership +from gn3.authentication import get_highest_user_access_role +from gn3.authentication import DataRole +from gn3.authentication import AdminRole class TestGetUserMembership(unittest.TestCase): @@ -56,7 +56,7 @@ class TestGetUserMembership(unittest.TestCase): class TestCheckUserAccessRole(unittest.TestCase): - """Test cases for `get_user_access_roles`""" + """Test cases for `get_highest_user_access_role`""" @mock.patch("wqflask.resource_manager.requests.get") def test_edit_access(self, requests_mock): @@ -69,7 +69,7 @@ class TestCheckUserAccessRole(unittest.TestCase): } )) type(requests_mock.return_value).content = response - self.assertEqual(get_user_access_roles( + self.assertEqual(get_highest_user_access_role( resource_id="0196d92e1665091f202f", user_id="8ad942fe-490d-453e-bd37"), {"data": DataRole.EDIT, @@ -86,7 +86,7 @@ class TestCheckUserAccessRole(unittest.TestCase): } )) type(requests_mock.return_value).content = response - self.assertEqual(get_user_access_roles( + self.assertEqual(get_highest_user_access_role( resource_id="0196d92e1665091f202f", user_id=""), {"data": DataRole.NO_ACCESS, -- cgit v1.2.3 From f6a2a07b726dbff09f73bd7e98b155fd90471862 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 1 Nov 2021 17:38:26 +0300 Subject: Sort import according to pep-8 spec --- wqflask/wqflask/metadata_edits.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 31d056b4..2be54620 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -1,18 +1,36 @@ -import MySQLdb -import os -import json import datetime -import difflib -import redis +import json +import os from collections import namedtuple -from flask import (Blueprint, current_app, redirect, - flash, g, render_template, request, Response) from itertools import groupby +from typing import Dict + +import MySQLdb +import difflib +import redis + +from flask import Blueprint +from flask import Response +from flask import current_app +from flask import flash +from flask import g +from flask import redirect +from flask import render_template +from flask import request +from flask import url_for from wqflask.decorators import edit_access_required +from wqflask.decorators import edit_access_required +from wqflask.decorators import edit_admins_access_required +from wqflask.decorators import login_required from wqflask.decorators import login_required +from gn3.authentication import AdminRole +from gn3.authentication import DataRole +from gn3.authentication import get_highest_user_access_role +from gn3.authentication import get_user_membership +from gn3.commands import run_cmd from gn3.db import diff_from_dict from gn3.db import fetchall from gn3.db import fetchone @@ -24,7 +42,6 @@ from gn3.db.phenotypes import Probeset from gn3.db.phenotypes import Publication from gn3.db.phenotypes import PublishXRef from gn3.db.phenotypes import probeset_mapping -from gn3.commands import run_cmd from gn3.db.traits import get_trait_csv_sample_data from gn3.db.traits import update_sample_data -- cgit v1.2.3 From 3b8f6eedaa9ca47cdbfdf2a7f87858e58949e8dd Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 1 Nov 2021 17:39:28 +0300 Subject: Move "*/diffs/*" to metadata_edits module --- wqflask/wqflask/metadata_edits.py | 162 ++++++++++++++++++++++++++++++-------- wqflask/wqflask/views.py | 62 --------------- 2 files changed, 130 insertions(+), 94 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 2be54620..85bd9b08 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -49,6 +49,45 @@ from gn3.db.traits import update_sample_data metadata_edit = Blueprint('metadata_edit', __name__) +def _get_diffs(diff_dir: str, + user_id: str, + redis_conn: redis.Redis, + gn_proxy_url: str): + def __get_file_metadata(file_name: str) -> Dict: + author, resource_id, time_stamp, *_ = file_name.split(".") + + return { + "resource_id": resource_id, + "file_name": file_name, + "author": json.loads(redis_conn.hget("users", + author)).get("full_name"), + "time_stamp": time_stamp, + "roles": get_highest_user_access_role( + resource_id=resource_id, + user_id=user_id, + gn_proxy_url=gn_proxy_url), + } + + approved, rejected, waiting = [], [], [] + if os.path.exists(diff_dir): + for name in os.listdir(diff_dir): + file_metadata = __get_file_metadata(file_name=name) + admin_status = file_metadata["roles"].get("admin") + append_p = (user_id in name or + admin_status > AdminRole.EDIT_ACCESS) + if name.endswith(".rejected") and append_p: + rejected.append(__get_file_metadata(file_name=name)) + elif name.endswith(".approved") and append_p: + approved.append(__get_file_metadata(file_name=name)) + elif append_p: # Normal file + waiting.append(__get_file_metadata(file_name=name)) + return { + "approved": approved, + "rejected": rejected, + "waiting": waiting, + } + + def edit_phenotype(conn, name, dataset_id): publish_xref = fetchone( conn=conn, @@ -185,7 +224,7 @@ def update_phenotype(dataset_id: str, name: str): data_ = request.form.to_dict() TMPDIR = current_app.config.get("TMPDIR") author = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") - or g.user_session.record.get("user_id") or "") + or g.user_session.record.get("user_id") or "") phenotype_id = str(data_.get('phenotype-id')) if not (file_ := request.files.get("file")): flash("No sample-data has been uploaded", "warning") @@ -201,13 +240,13 @@ def update_phenotype(dataset_id: str, name: str): os.makedirs(os.path.join(SAMPLE_DATADIR, "updated")) current_time = str(datetime.datetime.now().isoformat()) - _file_name = (f"{request.args.get('resource-id')}." - f"{current_time}.csv") + _file_name = (f"{author}.{request.args.get('resource-id')}." + f"{current_time}") new_file_name = (os.path.join(TMPDIR, f"sample-data/updated/{_file_name}.csv")) uploaded_file_name = (os.path.join( - TMPDIR, "sample-data/updated/updated.", - f"{_file_name}.csv")) + TMPDIR, "sample-data/updated/", + f"{_file_name}.csv.uploaded")) file_.save(new_file_name) publishdata_id = "" lines = [] @@ -331,7 +370,7 @@ def update_probeset(name: str): } diff_data = {} author = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") - or g.user_session.record.get("user_id") or "") + or g.user_session.record.get("user_id") or "") if (updated_probeset := update( conn, "ProbeSet", data=Probeset(**probeset_), @@ -371,29 +410,88 @@ def get_sample_data_as_csv(dataset_id: str, phenotype_id: int): ) -# @metadata_edit.route("/diffs") -# @login_required -# def display_diffs_admin(): -# TMPDIR = current_app.config.get("TMPDIR") -# DIFF_DIR = f"{TMPDIR}/sample-data/diffs" -# approved_files, rejected_files, files = {}, {}, {} -# user_id = ((g.user_session.record.get(b"user_id") or -# b"").decode("utf-8") -# or g.user_session.record.get("user_id") or "") -# redis_conn = redis.from_url(current_app.config["REDIS_URL"], -# decode_responses=True) -# if os.path.exists(DIFF_DIR): -# for name in os.listdir(DIFF_DIR): -# if name.endswith(".approved"): - -# approved_file.update{ - -# } -# approved_files.add(name) -# elif name.endswith(".rejected"): -# approved_files.add(name) -# else: -# files.append(name) -# return render_template("display_files_admin.html", -# files=files) -# http://localhost:5004/datasets/diffs/admin +@metadata_edit.route("/diffs") +@login_required +def list_diffs(): + files = _get_diffs( + diff_dir=f"{current_app.config.get('TMPDIR')}/sample-data/diffs", + user_id=((g.user_session.record.get(b"user_id") or + b"").decode("utf-8") + or g.user_session.record.get("user_id") or ""), + redis_conn=redis.from_url(current_app.config["REDIS_URL"], + decode_responses=True), + gn_proxy_url=current_app.config.get("GN2_PROXY")) + return render_template( + "display_files.html", + approved=files.get("approved"), + rejected=files.get("rejected"), + waiting=files.get("waiting")) + + +@metadata_edit.route("/diffs/") +def show_diff(name): + TMPDIR = current_app.config.get("TMPDIR") + with open(os.path.join(f"{TMPDIR}/sample-data/diffs", + name), 'r') as myfile: + content = myfile.read() + return Response(content, mimetype='text/json') + +# http://localhost:5004/datasets/diffs +@metadata_edit.route("/diffs//reject") +@edit_admins_access_required +@login_required +def reject_data(resource_id: str, file_name: str): + TMPDIR = current_app.config.get("TMPDIR") + os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), + os.path.join(f"{TMPDIR}/sample-data/diffs", + f"{file_name}.rejected")) + flash(f"{file_name} has been rejected!", "success") + return redirect(url_for('metadata_edit.list_diffs')) + + +@metadata_edit.route("/diffs//approve") +@edit_admins_access_required +@login_required +def approve_data(resource_id:str, file_name: str): + sample_data = {file_name: str} + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + TMPDIR = current_app.config.get("TMPDIR") + with open(os.path.join(f"{TMPDIR}/sample-data/diffs", + file_name), 'r') as myfile: + sample_data = json.load(myfile) + PUBLISH_ID = sample_data.get("publishdata_id") + modifications = [d for d in sample_data.get("Modifications")] + row_counts = len(modifications) + for modification in modifications: + if modification.get("Current"): + (strain_id, + strain_name, + value, se, count) = modification.get("Current").split(",") + update_sample_data( + conn=conn, + strain_name=strain_name, + strain_id=int(strain_id), + publish_data_id=int(PUBLISH_ID), + value=value, + error=se, + count=count + ) + insert(conn, + table="metadata_audit", + data=MetadataAudit( + dataset_id=name.split(".")[0], # use the dataset name + editor=sample_data.get("author"), + json_data=json.dumps(sample_data))) + if modifications: + # Once data is approved, rename it! + os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), + os.path.join(f"{TMPDIR}/sample-data/diffs", + f"{file_name}.approved")) + flash((f"Just updated data from: {file_name}; {row_counts} " + "row(s) modified!"), + "success") + return redirect(url_for('metadata_edit.list_diffs')) + diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 3e53e4c5..2cbaffd6 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1084,65 +1084,3 @@ def display_diffs_users(): files=files) -@app.route("/data-samples/approve/") -def approve_data(name): - sample_data = {} - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - TMPDIR = current_app.config.get("TMPDIR") - with open(os.path.join(f"{TMPDIR}/sample-data/diffs", - name), 'r') as myfile: - sample_data = json.load(myfile) - PUBLISH_ID = sample_data.get("publishdata_id") - modifications = [d for d in sample_data.get("Modifications")] - row_counts = len(modifications) - for modification in modifications: - if modification.get("Current"): - (strain_id, - strain_name, - value, se, count) = modification.get("Current").split(",") - update_sample_data( - conn=conn, - strain_name=strain_name, - strain_id=int(strain_id), - publish_data_id=int(PUBLISH_ID), - value=value, - error=se, - count=count - ) - insert(conn, - table="metadata_audit", - data=MetadataAudit( - dataset_id=name.split(".")[0], # use the dataset name - editor=sample_data.get("author"), - json_data=json.dumps(sample_data))) - if modifications: - # Once data is approved, rename it! - os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", name), - os.path.join(f"{TMPDIR}/sample-data/diffs", - f"{name}.approved")) - flash((f"Just updated data from: {name}; {row_counts} " - "row(s) modified!"), - "success") - return redirect("/admin/data-sample/diffs/") - - -@app.route("/data-samples/reject/") -def reject_data(name): - TMPDIR = current_app.config.get("TMPDIR") - os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", name), - os.path.join(f"{TMPDIR}/sample-data/diffs", - f"{name}.rejected")) - flash(f"{name} has been rejected!", "success") - return redirect("/admin/data-sample/diffs/") - - -@app.route("/display-file/") -def display_file(name): - TMPDIR = current_app.config.get("TMPDIR") - with open(os.path.join(f"{TMPDIR}/sample-data/diffs", - name), 'r') as myfile: - content = myfile.read() - return Response(content, mimetype='text/json') -- cgit v1.2.3 From 512200976ff5330c0716c4f2278745449ed52908 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 1 Nov 2021 17:39:54 +0300 Subject: Delete obsolete template files --- wqflask/wqflask/templates/display_files_admin.html | 32 ---------------------- wqflask/wqflask/templates/display_files_user.html | 31 --------------------- 2 files changed, 63 deletions(-) delete mode 100644 wqflask/wqflask/templates/display_files_admin.html delete mode 100644 wqflask/wqflask/templates/display_files_user.html diff --git a/wqflask/wqflask/templates/display_files_admin.html b/wqflask/wqflask/templates/display_files_admin.html deleted file mode 100644 index 4b4babc4..00000000 --- a/wqflask/wqflask/templates/display_files_admin.html +++ /dev/null @@ -1,32 +0,0 @@ -{% extends "base.html" %} -{% block title %}Trait Submission{% endblock %} -{% block content %} - -{% with messages = get_flashed_messages(with_categories=true) %} -{% if messages %} -{% for category, message in messages %} -
      -

      {{ message }}

      -
      -{% endfor %} -{% endif %} -{% endwith %} -Show files for approval - -
      - -
      -{%endblock%} - -{% block js %} - -{% endblock %} diff --git a/wqflask/wqflask/templates/display_files_user.html b/wqflask/wqflask/templates/display_files_user.html deleted file mode 100644 index b6bab709..00000000 --- a/wqflask/wqflask/templates/display_files_user.html +++ /dev/null @@ -1,31 +0,0 @@ -{% extends "base.html" %} -{% block title %}Trait Submission{% endblock %} -{% block content %} - -{% with messages = get_flashed_messages(with_categories=true) %} -{% if messages %} -{% for category, message in messages %} -
      -

      {{ message }}

      -
      -{% endfor %} -{% endif %} -{% endwith %} -Show files for approval - -
      - -
      -{%endblock%} - -{% block js %} - -{% endblock %} -- cgit v1.2.3 From 6633a39a54d7f1a66ee6671b37100f22388ccfab Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 4 Nov 2021 11:56:34 +0300 Subject: Remove variables that are only used once --- wqflask/wqflask/metadata_edits.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 85bd9b08..bc24e92f 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -462,9 +462,7 @@ def approve_data(resource_id:str, file_name: str): with open(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), 'r') as myfile: sample_data = json.load(myfile) - PUBLISH_ID = sample_data.get("publishdata_id") modifications = [d for d in sample_data.get("Modifications")] - row_counts = len(modifications) for modification in modifications: if modification.get("Current"): (strain_id, @@ -474,7 +472,7 @@ def approve_data(resource_id:str, file_name: str): conn=conn, strain_name=strain_name, strain_id=int(strain_id), - publish_data_id=int(PUBLISH_ID), + publish_data_id=int(sample_data.get("publishdata_id")), value=value, error=se, count=count @@ -490,7 +488,7 @@ def approve_data(resource_id:str, file_name: str): os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), os.path.join(f"{TMPDIR}/sample-data/diffs", f"{file_name}.approved")) - flash((f"Just updated data from: {file_name}; {row_counts} " + flash((f"Just updated data from: {file_name}; {len(modifications)} " "row(s) modified!"), "success") return redirect(url_for('metadata_edit.list_diffs')) -- cgit v1.2.3 From 4df8fc0be487005e471cb198f9168274463dac25 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 4 Nov 2021 12:41:39 +0300 Subject: Use the correct dataset_id in sql query --- wqflask/wqflask/metadata_edits.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index bc24e92f..572db080 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -271,7 +271,7 @@ def update_phenotype(dataset_id: str, name: str): dict_.update({ "author": author, "publishdata_id": publishdata_id, - "dataset_id": data_.get("dataset-name"), + "dataset_id": name, "timestamp": datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S") }) @@ -480,7 +480,7 @@ def approve_data(resource_id:str, file_name: str): insert(conn, table="metadata_audit", data=MetadataAudit( - dataset_id=name.split(".")[0], # use the dataset name + dataset_id=sample_data.get("dataset_id"), editor=sample_data.get("author"), json_data=json.dumps(sample_data))) if modifications: -- cgit v1.2.3 From 86a4fccfefb26738aeb054ffec4d9408aaa86ed5 Mon Sep 17 00:00:00 2001 From: jgart Date: Wed, 3 Nov 2021 20:48:57 -0400 Subject: Removes unused import --- wqflask/utility/chunks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wqflask/utility/chunks.py b/wqflask/utility/chunks.py index 484b5de6..f6e88cbe 100644 --- a/wqflask/utility/chunks.py +++ b/wqflask/utility/chunks.py @@ -1,5 +1,4 @@ import math -import time def divide_into_chunks(the_list, number_chunks): -- cgit v1.2.3 From 1e3092ced9489c50a18e403c56958694609130f7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 5 Nov 2021 13:05:58 +0300 Subject: Add template for displaying changed data --- wqflask/wqflask/templates/display_files.html | 119 +++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 wqflask/wqflask/templates/display_files.html diff --git a/wqflask/wqflask/templates/display_files.html b/wqflask/wqflask/templates/display_files.html new file mode 100644 index 00000000..d5295fb4 --- /dev/null +++ b/wqflask/wqflask/templates/display_files.html @@ -0,0 +1,119 @@ +{% extends "base.html" %} +{% block title %}Trait Submission{% endblock %} +{% block content %} + +{% with messages = get_flashed_messages(with_categories=true) %} +{% if messages %} +{% for category, message in messages %} +
      +

      {{ message }}

      +
      +{% endfor %} +{% endif %} +{% endwith %} + + +
      + + {% if waiting %} +
      + Files for approval: +
      +
      + + + + + + + + {% for data in waiting %} + + {% set file_url = url_for('metadata_edit.show_diff', name=data.get('file_name')) %} + + + + {% if data.get("roles").get("admin") > AdminRole.EDIT_ACCESS %} + {% set reject_url = url_for('metadata_edit.reject_data', resource_id=data.get('resource_id'), file_name=data.get('file_name')) %} + {% set approve_url = url_for('metadata_edit.approve_data', resource_id=data.get('resource_id'), file_name=data.get('file_name')) %} + + + {% endif %} + + {% endfor %} + +
      Resource Id + AuthorTimeStamp
      {{ data.get("resource_id") }}{{ data.get("author")}}{{ data.get("time_stamp")}} + + + +
      +
      +
      + {% endif %} + + {% if rejected %} +
      + Rejected Files: +
      + + + + + + + {% for data in rejected %} + + {% set file_url = url_for('metadata_edit.show_diff', name=data.get('file_name')) %} + + + + + {% endfor %} + +
      Resource Id + AuthorTimeStamp
      {{ data.get("resource_id") }}{{ data.get("author")}}{{ data.get("time_stamp")}}
      +
      +
      + {% endif %} + + {% if approved %} +
      + Approved Data: +
      + + + + + + + {% for data in approved %} + + {% set file_url = url_for('metadata_edit.show_diff', name=data.get('file_name')) %} + + + + + {% endfor %} + +
      Resource Id + AuthorTimeStamp
      {{ data.get("resource_id") }}{{ data.get("author")}}{{ data.get("time_stamp")}}
      +
      +
      + {% endif %} +
      + +{%endblock%} + +{% block js %} + +{% endblock %} -- cgit v1.2.3 From f75ffe73e592bc79f4decf268d50b225ade43488 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 5 Nov 2021 13:40:44 +0300 Subject: Add script for manually adding "editors" to each resource --- scripts/authentication/editors.py | 45 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 scripts/authentication/editors.py diff --git a/scripts/authentication/editors.py b/scripts/authentication/editors.py new file mode 100755 index 00000000..12f48872 --- /dev/null +++ b/scripts/authentication/editors.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Manually add editors users""" +import redis +import json +import uuid +import datetime + +if __name__ == "__main__": + conn = redis.Redis(decode_responses=True) + group_uid = "" + for guid in conn.hgetall("groups"): + group_details = json.loads(conn.hget("groups", guid)) + if group_details.get("name") == "editors": + group_uid = guid + break + + if not group_uid: + group_uid = str(uuid.uuid4()) + timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + conn.hset( + "groups", + group_uid, + json.dumps( + { + "name": "editors", + "admins": [], + "members": ["labwilliams@gmail.com"], + "changed_timestamp": timestamp, + })) + + for resource in conn.hgetall("resources"): + _resource = json.loads(conn.hget("resources", resource)) + _resource["default_mask"] = { + 'data': 'view', + 'metadata': 'view', + 'admin': 'not-admin', + } + _resource["group_masks"] = { + group_uid: { + 'metadata': 'edit', + 'data': 'edit', + 'admin': 'edit-admins', + }} + conn.hset("resources", resource, json.dumps(_resource)) + print("Done adding editor's group to resources!") -- cgit v1.2.3 From 5c8429d6365e0a03ea2686633df5fd555c9b8ae5 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 5 Nov 2021 13:44:01 +0300 Subject: Use members uuid --- scripts/authentication/editors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/authentication/editors.py b/scripts/authentication/editors.py index 12f48872..750dc309 100755 --- a/scripts/authentication/editors.py +++ b/scripts/authentication/editors.py @@ -24,7 +24,7 @@ if __name__ == "__main__": { "name": "editors", "admins": [], - "members": ["labwilliams@gmail.com"], + "members": ["8ad942fe-490d-453e-bd37-56f252e41603"], "changed_timestamp": timestamp, })) -- cgit v1.2.3 From 178fe53c907b8857865236d9066d7f6f3cf3d9a7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 5 Nov 2021 17:35:13 +0300 Subject: Add created_timestamp when creating a new group --- scripts/authentication/editors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/authentication/editors.py b/scripts/authentication/editors.py index 750dc309..dc3b1075 100755 --- a/scripts/authentication/editors.py +++ b/scripts/authentication/editors.py @@ -25,6 +25,7 @@ if __name__ == "__main__": "name": "editors", "admins": [], "members": ["8ad942fe-490d-453e-bd37-56f252e41603"], + "created_timestamp": timestamp, "changed_timestamp": timestamp, })) -- cgit v1.2.3 From 990d2e4305ed89c2bc78ff370b71a3a4f68e5f01 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 5 Nov 2021 17:40:52 +0300 Subject: Add created_timestamp when creating a new group --- scripts/authentication/group.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index 08652454..dd9ba808 100755 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -123,6 +123,7 @@ def create_group_data(users: Dict, target_group: str, "name": target_group, "admins": list(admin_ids), "members": list(member_ids), + "created_timestamp": timestamp, "changed_timestamp": timestamp, })} -- cgit v1.2.3 From d905866c9a025f6af4d08432d978349ab09348c0 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 8 Nov 2021 05:02:36 +0300 Subject: Build UI for Jupyter Notebooks Launcher * Build a HTML UI for the Jupyter Notebooks Launcher feature that fits in, aesthetically, with that of the whole GeneNetwork look and feel. --- wqflask/wqflask/jupyter_notebooks.py | 21 +++++++++++++++++++++ wqflask/wqflask/templates/jupyter_notebooks.html | 24 ++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 wqflask/wqflask/jupyter_notebooks.py create mode 100644 wqflask/wqflask/templates/jupyter_notebooks.html diff --git a/wqflask/wqflask/jupyter_notebooks.py b/wqflask/wqflask/jupyter_notebooks.py new file mode 100644 index 00000000..c7d5ffa3 --- /dev/null +++ b/wqflask/wqflask/jupyter_notebooks.py @@ -0,0 +1,21 @@ +from Flask import BluePrint, render_template + +jupyter_notebooks = Blueprint('jupyter_notebooks', __name__) + +@jupyter_notebooks.route("/jupyter-notebooks-launcher", methods=("GET",)) +def jupyter_notebooks_launcher(): + links = ( + { + "main_url": "http://notebook.genenetwork.org/51091/tree?", + "notebook_name": "COVID-19 in mybinder.org federation", + "src_link_url": "https://github.com/jgarte/covid19_in_binder"}, + { + "main_url": "http://notebook.genenetwork.org/35639/tree?", + "notebook_name": "Simple requirements.txt based example", + "src_link_url": "https://github.com/jgarte/requirements"}, + { + "main_url": "http://notebook.genenetwork.org/40733/tree?", + "notebook_name": "Guile Jupyter Notebook Querying GeneNetwork API", + "src_link_url": "https://github.com/jgarte/guile-notebook-genenetwork-api"}) + + return render_template("jupyter_notebooks.html", links=links) diff --git a/wqflask/wqflask/templates/jupyter_notebooks.html b/wqflask/wqflask/templates/jupyter_notebooks.html new file mode 100644 index 00000000..4dce0f27 --- /dev/null +++ b/wqflask/wqflask/templates/jupyter_notebooks.html @@ -0,0 +1,24 @@ +{%extends "base.html"%} + +{%block title%} +Jupyter Notebooks +{%endblock%} + +{%block content%} + +
      +

      Current Notebooks

      + + {%for item in links:%} +
      + {{item.get("notebook_name")}} + View Source +
      + {%endfor%} +
      + +{%endblock%} -- cgit v1.2.3 From 8176593c4e2ef93fb02dd7f30e24dba9f37bcf4b Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 8 Nov 2021 05:16:35 +0300 Subject: Hook up new UI page to the main menu * Hook up the new Jupyter Notebooks UI to the main menu --- wqflask/wqflask/__init__.py | 3 +++ wqflask/wqflask/jupyter_notebooks.py | 4 ++-- wqflask/wqflask/templates/base.html | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index 7d46fbad..9223ec5d 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -23,6 +23,8 @@ from wqflask.api.markdown import facilities_blueprint from wqflask.api.markdown import blogs_blueprint from wqflask.api.markdown import news_blueprint +from wqflask.jupyter_notebooks import jupyter_notebooks + app = Flask(__name__) @@ -60,6 +62,7 @@ app.register_blueprint(environments_blueprint, url_prefix="/environments") app.register_blueprint(facilities_blueprint, url_prefix="/facilities") app.register_blueprint(blogs_blueprint, url_prefix="/blogs") app.register_blueprint(news_blueprint, url_prefix="/news") +app.register_blueprint(jupyter_notebooks, url_prefix="jupyter_notebooks") app.register_blueprint(resource_management, url_prefix="/resource-management") diff --git a/wqflask/wqflask/jupyter_notebooks.py b/wqflask/wqflask/jupyter_notebooks.py index c7d5ffa3..348b252d 100644 --- a/wqflask/wqflask/jupyter_notebooks.py +++ b/wqflask/wqflask/jupyter_notebooks.py @@ -2,8 +2,8 @@ from Flask import BluePrint, render_template jupyter_notebooks = Blueprint('jupyter_notebooks', __name__) -@jupyter_notebooks.route("/jupyter-notebooks-launcher", methods=("GET",)) -def jupyter_notebooks_launcher(): +@jupyter_notebooks.route("/launcher", methods=("GET",)) +def launcher(): links = ( { "main_url": "http://notebook.genenetwork.org/51091/tree?", diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index 4a6674c6..ab8b644f 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -87,7 +87,7 @@
    • Systems Genetics PheWAS
    • Genome Browser
    • BXD Power Calculator
      • -
    • Jupyter Notebook Launcher
      • +
    • Jupyter Notebook Launcher
    • Interplanetary File System
    -- cgit v1.2.3 From 8cef34c83fd2c013e0671ece54bfe43a3c4bb510 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 8 Nov 2021 08:24:19 +0300 Subject: Fix runtime errors * wqflask/wqflask/__init__.py: when registering a URL, the url_prefix should begin with a trailing slash. * wqflask/wqflask/jupyter_notebooks.py: fix imports. --- wqflask/wqflask/__init__.py | 2 +- wqflask/wqflask/jupyter_notebooks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index 9223ec5d..fbf1c9f5 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -62,7 +62,7 @@ app.register_blueprint(environments_blueprint, url_prefix="/environments") app.register_blueprint(facilities_blueprint, url_prefix="/facilities") app.register_blueprint(blogs_blueprint, url_prefix="/blogs") app.register_blueprint(news_blueprint, url_prefix="/news") -app.register_blueprint(jupyter_notebooks, url_prefix="jupyter_notebooks") +app.register_blueprint(jupyter_notebooks, url_prefix="/jupyter_notebooks") app.register_blueprint(resource_management, url_prefix="/resource-management") diff --git a/wqflask/wqflask/jupyter_notebooks.py b/wqflask/wqflask/jupyter_notebooks.py index 348b252d..dbea04dd 100644 --- a/wqflask/wqflask/jupyter_notebooks.py +++ b/wqflask/wqflask/jupyter_notebooks.py @@ -1,4 +1,4 @@ -from Flask import BluePrint, render_template +from flask import Blueprint, render_template jupyter_notebooks = Blueprint('jupyter_notebooks', __name__) -- cgit v1.2.3 From 9f7e3035d13b286238d878a0baa45cb4c1708cee Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 8 Nov 2021 16:19:40 +0300 Subject: Fix URL --- wqflask/wqflask/templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index ab8b644f..6e922f24 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -87,7 +87,7 @@
  • Systems Genetics PheWAS
  • Genome Browser
  • BXD Power Calculator
    • -
  • Jupyter Notebook Launcher
    • +
  • Jupyter Notebook Launcher
  • Interplanetary File System
    • -- cgit v1.2.3 From 14f3d5da0de60dea8b0a24f68c2e658d92268e3a Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 8 Nov 2021 16:40:22 +0300 Subject: Add some styling for the Jupyter links --- wqflask/wqflask/static/new/css/jupyter_notebooks.css | 12 ++++++++++++ wqflask/wqflask/templates/jupyter_notebooks.html | 4 ++++ 2 files changed, 16 insertions(+) create mode 100644 wqflask/wqflask/static/new/css/jupyter_notebooks.css diff --git a/wqflask/wqflask/static/new/css/jupyter_notebooks.css b/wqflask/wqflask/static/new/css/jupyter_notebooks.css new file mode 100644 index 00000000..b916f03b --- /dev/null +++ b/wqflask/wqflask/static/new/css/jupyter_notebooks.css @@ -0,0 +1,12 @@ +.jupyter-links .main-link:nth-of-type(2n) { + background: #EEEEEE; +} + +.jupyter-links .main-link { + font-size: larger; + display: block; +} + +.jupyter-links .src-link { + font-size: smaller; +} diff --git a/wqflask/wqflask/templates/jupyter_notebooks.html b/wqflask/wqflask/templates/jupyter_notebooks.html index 4dce0f27..da250b2a 100644 --- a/wqflask/wqflask/templates/jupyter_notebooks.html +++ b/wqflask/wqflask/templates/jupyter_notebooks.html @@ -4,6 +4,10 @@ Jupyter Notebooks {%endblock%} +{%block css%} + +{%endblock} + {%block content%}
    -- cgit v1.2.3 From c63d10c565f82413ae8b5e553b1f00485439530a Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 8 Nov 2021 16:43:50 +0300 Subject: Fix syntax error --- wqflask/wqflask/templates/jupyter_notebooks.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/jupyter_notebooks.html b/wqflask/wqflask/templates/jupyter_notebooks.html index da250b2a..afc95a15 100644 --- a/wqflask/wqflask/templates/jupyter_notebooks.html +++ b/wqflask/wqflask/templates/jupyter_notebooks.html @@ -6,7 +6,7 @@ Jupyter Notebooks {%block css%} -{%endblock} +{%endblock%} {%block content%} -- cgit v1.2.3 From 5a14bbc53dafd7b85c3e903a3dcb2dab87a6fed2 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 8 Nov 2021 16:45:05 +0300 Subject: Fix styling of divs --- wqflask/wqflask/static/new/css/jupyter_notebooks.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/static/new/css/jupyter_notebooks.css b/wqflask/wqflask/static/new/css/jupyter_notebooks.css index b916f03b..663d1746 100644 --- a/wqflask/wqflask/static/new/css/jupyter_notebooks.css +++ b/wqflask/wqflask/static/new/css/jupyter_notebooks.css @@ -1,4 +1,4 @@ -.jupyter-links .main-link:nth-of-type(2n) { +.jupyter-links:nth-of-type(2n) { background: #EEEEEE; } -- cgit v1.2.3 From a2f2d89a5fbe720d3c1f1ac9655adca8b2ff503d Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 8 Nov 2021 16:48:32 +0300 Subject: Add some padding --- wqflask/wqflask/static/new/css/jupyter_notebooks.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/wqflask/static/new/css/jupyter_notebooks.css b/wqflask/wqflask/static/new/css/jupyter_notebooks.css index 663d1746..db972a17 100644 --- a/wqflask/wqflask/static/new/css/jupyter_notebooks.css +++ b/wqflask/wqflask/static/new/css/jupyter_notebooks.css @@ -1,3 +1,7 @@ +.jupyter-links { + padding: 1.5em; +} + .jupyter-links:nth-of-type(2n) { background: #EEEEEE; } -- cgit v1.2.3 From 2fcf4d0343244936d6b066e85a1a06ebb33adf26 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 9 Nov 2021 19:29:52 +0000 Subject: Temporary fix to get temp trait submission working again; maybe should be dealt with in GN3 in some way --- wqflask/wqflask/show_trait/show_trait.py | 10 ++++++---- wqflask/wqflask/templates/show_trait_details.html | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 6020bc16..678e3132 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -40,10 +40,15 @@ ONE_YEAR = 60 * 60 * 24 * 365 class ShowTrait: def __init__(self, user_id, kw): + self.admin_status = None if 'trait_id' in kw and kw['dataset'] != "Temp": self.temp_trait = False self.trait_id = kw['trait_id'] helper_functions.get_species_dataset_trait(self, kw) + self.admin_status = get_highest_user_access_role( + user_id=user_id, + resource_id=(self.resource_id or ""), + gn_proxy_url=GN_PROXY_URL) elif 'group' in kw: self.temp_trait = True self.trait_id = "Temp_" + kw['species'] + "_" + kw['group'] + \ @@ -73,10 +78,7 @@ class ShowTrait: self.trait_vals = Redis.get(self.trait_id).split() self.resource_id = get_resource_id(self.dataset, self.trait_id) - self.admin_status = get_highest_user_access_role( - user_id=user_id, - resource_id=(self.resource_id or ""), - gn_proxy_url=GN_PROXY_URL) + # ZS: Get verify/rna-seq link URLs try: blatsequence = self.this_trait.blatseq diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 3e59a3ee..6b125221 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -234,7 +234,7 @@ {% endif %} {% endif %} - {% if admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} + {% if admin_status != None and admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} {% if this_trait.dataset.type == 'Publish' %} {% endif %} -- cgit v1.2.3 From fe549317a5e0996cec246a011eb044b065f7ed8c Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 9 Nov 2021 22:04:24 +0000 Subject: Fixed error mistakenly introduced by the last commit --- wqflask/wqflask/show_trait/show_trait.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 678e3132..93f95852 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -45,6 +45,8 @@ class ShowTrait: self.temp_trait = False self.trait_id = kw['trait_id'] helper_functions.get_species_dataset_trait(self, kw) + self.resource_id = get_resource_id(self.dataset, + self.trait_id) self.admin_status = get_highest_user_access_role( user_id=user_id, resource_id=(self.resource_id or ""), @@ -76,8 +78,6 @@ class ShowTrait: name=self.trait_id, cellid=None) self.trait_vals = Redis.get(self.trait_id).split() - self.resource_id = get_resource_id(self.dataset, - self.trait_id) # ZS: Get verify/rna-seq link URLs try: -- cgit v1.2.3 From bf1620406c3700d7e211a02fe13c3d8df9a9532d Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 10 Nov 2021 10:39:53 +0300 Subject: rename:loading correlation results to computing correlations --- wqflask/wqflask/templates/loading.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index ccf810b0..b9e31ad0 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -66,11 +66,11 @@ {% endif %} {% endif %} {% else %} -

    Loading {{ start_vars.tool_used }} Results...

    +

     {{ start_vars.tool_used }} Computation in progress ...

    {% endif %}

    - +
    {% if start_vars.vals_diff|length != 0 and start_vars.transform == "" %}

    -- cgit v1.2.3 From b5b44f401e0d05089534d7f8e6631d9a092fd0d7 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 10 Nov 2021 10:40:11 +0300 Subject: add compute gif --- wqflask/wqflask/static/gif/waitAnima2.gif | Bin 0 -> 54013 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 wqflask/wqflask/static/gif/waitAnima2.gif diff --git a/wqflask/wqflask/static/gif/waitAnima2.gif b/wqflask/wqflask/static/gif/waitAnima2.gif new file mode 100644 index 00000000..50aff7f2 Binary files /dev/null and b/wqflask/wqflask/static/gif/waitAnima2.gif differ -- cgit v1.2.3 From f3ff381a90733d6c64349ed1dd116df83b5565d6 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 11 Nov 2021 02:51:08 +0300 Subject: add fast compute from gn3 --- wqflask/wqflask/correlation/correlation_gn3_api.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 1e3a40f2..7b828016 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -1,5 +1,7 @@ """module that calls the gn3 api's to do the correlation """ import json +import time +from functools import wraps from wqflask.correlation import correlation_functions @@ -9,6 +11,7 @@ from base.trait import create_trait from base.trait import retrieve_sample_data from gn3.computations.correlations import compute_all_sample_correlation +from gn3.computations.correlations import fast_compute_all_sample_correlation from gn3.computations.correlations import map_shared_keys_to_values from gn3.computations.correlations import compute_all_lit_correlation from gn3.computations.correlations import compute_tissue_correlation @@ -19,9 +22,11 @@ def create_target_this_trait(start_vars): """this function creates the required trait and target dataset for correlation""" if start_vars['dataset'] == "Temp": - this_dataset = data_set.create_dataset(dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) + this_dataset = data_set.create_dataset( + dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) else: - this_dataset = data_set.create_dataset(dataset_name=start_vars['dataset']) + this_dataset = data_set.create_dataset( + dataset_name=start_vars['dataset']) target_dataset = data_set.create_dataset( dataset_name=start_vars['corr_dataset']) this_trait = create_trait(dataset=this_dataset, @@ -187,10 +192,10 @@ def compute_correlation(start_vars, method="pearson", compute_all=False): if corr_type == "sample": (this_trait_data, target_dataset_data) = fetch_sample_data( start_vars, this_trait, this_dataset, target_dataset) - correlation_results = compute_all_sample_correlation(corr_method=method, - this_trait=this_trait_data, - target_dataset=target_dataset_data) + correlation_results = fast_compute_all_sample_correlation(corr_method=method, + this_trait=this_trait_data, + target_dataset=target_dataset_data) elif corr_type == "tissue": trait_symbol_dict = this_dataset.retrieve_genes("Symbol") tissue_input = get_tissue_correlation_input( -- cgit v1.2.3 From bbc75dcef80c3df600ab01c1804a27cdfdce1b80 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 11 Nov 2021 02:51:44 +0300 Subject: init test for precomputing sample correlation --- wqflask/wqflask/correlation/pre_computes.py | 72 +++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 wqflask/wqflask/correlation/pre_computes.py diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py new file mode 100644 index 00000000..1db9f61b --- /dev/null +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -0,0 +1,72 @@ +"""module contains the code to do the +precomputations of sample data between +two entire datasets""" + +import json +from typing import List +from base import data_set + +from gn3.computations.correlations import fast_compute_all_sample_correlation +from gn3.computations.correlations import map_shared_keys_to_values + +def get_dataset_dict_data(dataset_obj): + """function to get the dataset data mapped to key""" + dataset_obj.get_trait_data() + return map_shared_keys_to_values(dataset_obj.samplelist, + dataset_obj.trait_data) + + +def fetch_datasets(base_dataset_name: str, target_dataset_name: str) ->List: + """query to fetch create datasets and fetch traits + all traits of a dataset""" + + # doesnt work for temp + + base_dataset = data_set.create_dataset(dataset_name=base_dataset_name) + + target_dataset = data_set.create_dataset(dataset_name=target_dataset_name) + # replace with map + + return (map(get_dataset_dict_data, + [base_dataset, target_dataset])) + + +# in the base dataset we just need the traits +def pre_compute_sample_correlation(base_dataset: List, + target_dataset: List) -> List: + """function compute the correlation between the + a whole dataset against a target + input: target&base_dataset(contains traits and sample results) + output: list containing the computed results + + precaution:function is expensive;targets only Exon and + """ + + for trait_info in base_dataset: + + yield fast_compute_all_sample_correlation(corr_method="pearson", + this_trait=trait_info, + target_dataset=target_dataset) + + +def cache_to_file(base_dataset_name: str, target_dataset_name: str): + """function to cache the results to file""" + + # validate the datasets expiry first + + base_dataset_data, target_dataset_data = [list(dataset) for dataset in list( + fetch_datasets(base_dataset_name, target_dataset_name))] + + + try: + with open("unique_file_name.json", "w") as file_handler: + file_handler.write() + + dataset_correlation_results = list(pre_compute_sample_correlation( + base_dataset_data, target_dataset_data)) + + print(dataset_correlation_results) + + json.dump(dataset_correlation_results, file_handler) + except Exception as error: + raise error -- cgit v1.2.3 From 721a45006697830c5bf780133c03d909c35b63a3 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 8 Nov 2021 08:55:02 +0300 Subject: Apply pep-8 to file --- wqflask/wqflask/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index fbf1c9f5..98416b41 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -10,8 +10,8 @@ from urllib.parse import urlparse from utility import formatting from gn3.authentication import DataRole, AdminRole -from wqflask.resource_manager import resource_management +from wqflask.group_manager import group_management from wqflask.metadata_edits import metadata_edit from wqflask.api.markdown import glossary_blueprint @@ -65,7 +65,6 @@ app.register_blueprint(news_blueprint, url_prefix="/news") app.register_blueprint(jupyter_notebooks, url_prefix="/jupyter_notebooks") app.register_blueprint(resource_management, url_prefix="/resource-management") - app.register_blueprint(metadata_edit, url_prefix="/datasets/") @app.before_request -- cgit v1.2.3 From d330150c71deb27c29c29bfce8f14856e49ca157 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 8 Nov 2021 08:56:53 +0300 Subject: Delete all methods in group_manager.py --- wqflask/wqflask/group_manager.py | 175 --------------------------------------- 1 file changed, 175 deletions(-) diff --git a/wqflask/wqflask/group_manager.py b/wqflask/wqflask/group_manager.py index 04a100ba..e69de29b 100644 --- a/wqflask/wqflask/group_manager.py +++ b/wqflask/wqflask/group_manager.py @@ -1,175 +0,0 @@ -import random -import string - -from flask import (Flask, g, render_template, url_for, request, make_response, - redirect, flash) - -from wqflask import app -from wqflask.user_login import send_verification_email, send_invitation_email, basic_info, set_password - -from utility.redis_tools import get_user_groups, get_group_info, save_user, create_group, delete_group, add_users_to_group, remove_users_from_group, \ - change_group_name, save_verification_code, check_verification_code, get_user_by_unique_column, get_resources, get_resource_info - -from utility.logger import getLogger -logger = getLogger(__name__) - - -@app.route("/groups/manage", methods=('GET', 'POST')) -def manage_groups(): - params = request.form if request.form else request.args - if "add_new_group" in params: - return redirect(url_for('add_group')) - else: - admin_groups, member_groups = get_user_groups(g.user_session.user_id) - return render_template("admin/group_manager.html", admin_groups=admin_groups, member_groups=member_groups) - - -@app.route("/groups/view", methods=('GET', 'POST')) -def view_group(): - params = request.form if request.form else request.args - group_id = params['id'] - group_info = get_group_info(group_id) - admins_info = [] - user_is_admin = False - if g.user_session.user_id in group_info['admins']: - user_is_admin = True - for user_id in group_info['admins']: - if user_id: - user_info = get_user_by_unique_column("user_id", user_id) - admins_info.append(user_info) - members_info = [] - for user_id in group_info['members']: - if user_id: - user_info = get_user_by_unique_column("user_id", user_id) - members_info.append(user_info) - - # ZS: This whole part might not scale well with many resources - resources_info = [] - all_resources = get_resources() - for resource_id in all_resources: - resource_info = get_resource_info(resource_id) - group_masks = resource_info['group_masks'] - if group_id in group_masks: - this_resource = {} - privileges = group_masks[group_id] - this_resource['id'] = resource_id - this_resource['name'] = resource_info['name'] - this_resource['data'] = privileges['data'] - this_resource['metadata'] = privileges['metadata'] - this_resource['admin'] = privileges['admin'] - resources_info.append(this_resource) - - return render_template("admin/view_group.html", group_info=group_info, admins=admins_info, members=members_info, user_is_admin=user_is_admin, resources=resources_info) - - -@app.route("/groups/remove", methods=('POST',)) -def remove_groups(): - group_ids_to_remove = request.form['selected_group_ids'] - for group_id in group_ids_to_remove.split(":"): - delete_group(g.user_session.user_id, group_id) - - return redirect(url_for('manage_groups')) - - -@app.route("/groups/remove_users", methods=('POST',)) -def remove_users(): - group_id = request.form['group_id'] - admin_ids_to_remove = request.form['selected_admin_ids'] - member_ids_to_remove = request.form['selected_member_ids'] - - remove_users_from_group(g.user_session.user_id, admin_ids_to_remove.split( - ":"), group_id, user_type="admins") - remove_users_from_group(g.user_session.user_id, member_ids_to_remove.split( - ":"), group_id, user_type="members") - - return redirect(url_for('view_group', id=group_id)) - - -@app.route("/groups/add_", methods=('POST',)) -def add_users(user_type='members'): - group_id = request.form['group_id'] - if user_type == "admins": - user_emails = request.form['admin_emails_to_add'].split(",") - add_users_to_group(g.user_session.user_id, group_id, - user_emails, admins=True) - elif user_type == "members": - user_emails = request.form['member_emails_to_add'].split(",") - add_users_to_group(g.user_session.user_id, group_id, - user_emails, admins=False) - - return redirect(url_for('view_group', id=group_id)) - - -@app.route("/groups/change_name", methods=('POST',)) -def change_name(): - group_id = request.form['group_id'] - new_name = request.form['new_name'] - group_info = change_group_name(g.user_session.user_id, group_id, new_name) - - return new_name - - -@app.route("/groups/create", methods=('GET', 'POST')) -def add_or_edit_group(): - params = request.form if request.form else request.args - if "group_name" in params: - member_user_ids = set() - admin_user_ids = set() - # ZS: Always add the user creating the group as an admin - admin_user_ids.add(g.user_session.user_id) - if "admin_emails_to_add" in params: - admin_emails = params['admin_emails_to_add'].split(",") - for email in admin_emails: - user_details = get_user_by_unique_column( - "email_address", email) - if user_details: - admin_user_ids.add(user_details['user_id']) - #send_group_invites(params['group_id'], user_email_list = admin_emails, user_type="admins") - if "member_emails_to_add" in params: - member_emails = params['member_emails_to_add'].split(",") - for email in member_emails: - user_details = get_user_by_unique_column( - "email_address", email) - if user_details: - member_user_ids.add(user_details['user_id']) - #send_group_invites(params['group_id'], user_email_list = user_emails, user_type="members") - - create_group(list(admin_user_ids), list( - member_user_ids), params['group_name']) - return redirect(url_for('manage_groups')) - else: - return render_template("admin/create_group.html") - -# ZS: Will integrate this later, for now just letting users be added directly - - -def send_group_invites(group_id, user_email_list=[], user_type="members"): - for user_email in user_email_list: - user_details = get_user_by_unique_column("email_address", user_email) - if user_details: - group_info = get_group_info(group_id) - # ZS: Probably not necessary since the group should normally always exist if group_id is being passed here, - # but it's technically possible to hit it if Redis is cleared out before submitting the new users or something - if group_info: - # ZS: Don't add user if they're already an admin or if they're being added a regular user and are already a regular user, - # but do add them if they're a regular user and are added as an admin - if (user_details['user_id'] in group_info['admins']) or \ - ((user_type == "members") and (user_details['user_id'] in group_info['members'])): - continue - else: - send_verification_email(user_details, template_name="email/group_verification.txt", - key_prefix="verification_code", subject="You've been invited to join a GeneNetwork user group") - else: - temp_password = ''.join(random.choice( - string.ascii_uppercase + string.digits) for _ in range(6)) - user_details = { - 'user_id': str(uuid.uuid4()), - 'email_address': user_email, - 'registration_info': basic_info(), - 'password': set_password(temp_password), - 'confirmed': 0 - } - save_user(user_details, user_details['user_id']) - send_invitation_email(user_email, temp_password) - -# @app.route() -- cgit v1.2.3 From 1184fa8b540b1fef205fd87e3a8f0a8a2c702d76 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 8 Nov 2021 08:57:19 +0300 Subject: Register "group_management" blueprint --- wqflask/wqflask/__init__.py | 2 ++ wqflask/wqflask/group_manager.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index 98416b41..05e040ed 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -12,6 +12,7 @@ from utility import formatting from gn3.authentication import DataRole, AdminRole from wqflask.group_manager import group_management +from wqflask.resource_manager import resource_management from wqflask.metadata_edits import metadata_edit from wqflask.api.markdown import glossary_blueprint @@ -66,6 +67,7 @@ app.register_blueprint(jupyter_notebooks, url_prefix="/jupyter_notebooks") app.register_blueprint(resource_management, url_prefix="/resource-management") app.register_blueprint(metadata_edit, url_prefix="/datasets/") +app.register_blueprint(group_management, url_prefix="/group-management") @app.before_request def before_request(): diff --git a/wqflask/wqflask/group_manager.py b/wqflask/wqflask/group_manager.py index e69de29b..4edafc66 100644 --- a/wqflask/wqflask/group_manager.py +++ b/wqflask/wqflask/group_manager.py @@ -0,0 +1,31 @@ +import redis + +from flask import current_app +from flask import Blueprint +from flask import g +from flask import render_template +from gn3.authentication import get_groups_by_user_uid +from wqflask.decorators import login_required + +group_management = Blueprint("group_management", __name__) + + +@group_management.route("/groups") +@login_required +def view_groups(): + groups = get_groups_by_user_uid( + user_uid=(g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")), + conn=redis.from_url( + current_app.config["REDIS_URL"], + decode_responses=True)) + return render_template("admin/group_manager.html", + admin_groups=groups.get("admin"), + member_groups=groups.get("member")) + + +@group_management.route("/groups/create", methods=("GET",)) +@login_required +def view_create_group_page(): + return render_template("admin/create_group.html") -- cgit v1.2.3 From 8766b3104be9cdbe00f594c936442b76f3b9a83e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 8 Nov 2021 08:57:57 +0300 Subject: Update "create_group" endpoint in template --- wqflask/wqflask/templates/admin/group_manager.html | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/wqflask/wqflask/templates/admin/group_manager.html b/wqflask/wqflask/templates/admin/group_manager.html index 692a7abc..c5000869 100644 --- a/wqflask/wqflask/templates/admin/group_manager.html +++ b/wqflask/wqflask/templates/admin/group_manager.html @@ -12,7 +12,11 @@

    Manage Groups

    {% if admin_groups|length != 0 or member_groups|length != 0 %}
    - + + +
    {% endif %} @@ -23,7 +27,11 @@ {% if admin_groups|length == 0 and member_groups|length == 0 %}

    You currently aren't a member or admin of any groups.


    - + + + {% else %}

    Admin Groups

    @@ -119,11 +127,6 @@ return $("#groups_form").submit(); }; - $("#create_group").on("click", function() { - url = $(this).data("url") - return submit_special(url) - }); - $("#remove_groups").on("click", function() { url = $(this).data("url") groups = [] -- cgit v1.2.3 From a70b6421221db391a1774ae1133a0a244f00f248 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 8 Nov 2021 09:12:31 +0300 Subject: Replace "var" with "let" --- wqflask/wqflask/static/new/javascript/group_manager.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/wqflask/wqflask/static/new/javascript/group_manager.js b/wqflask/wqflask/static/new/javascript/group_manager.js index 4c172cbf..cd56133a 100644 --- a/wqflask/wqflask/static/new/javascript/group_manager.js +++ b/wqflask/wqflask/static/new/javascript/group_manager.js @@ -16,23 +16,22 @@ $('#clear_members').click(function(){ function add_emails(user_type){ - var email_address = $('input[name=user_email]').val(); - var email_list_string = $('input[name=' + user_type + '_emails_to_add]').val().trim() - console.log(email_list_string) + let email_address = $('input[name=user_email]').val(); + let email_list_string = $('input[name=' + user_type + '_emails_to_add]').val().trim() if (email_list_string == ""){ - var email_set = new Set(); + let email_set = new Set(); } else { - var email_set = new Set(email_list_string.split(",")) + let email_set = new Set(email_list_string.split(",")) } email_set.add(email_address) $('input[name=' + user_type + '_emails_to_add]').val(Array.from(email_set).join(',')) - var emails_display_string = Array.from(email_set).join('\n') + let emails_display_string = Array.from(email_set).join('\n') $('.added_' + user_type + 's').val(emails_display_string) } function clear_emails(user_type){ $('input[name=' + user_type + '_emails_to_add]').val("") $('.added_' + user_type + 's').val("") -} \ No newline at end of file +} -- cgit v1.2.3 From f8bdd6cabd97a66bd3e3f8eeff910a0a9a79cf55 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 8 Nov 2021 09:12:51 +0300 Subject: Remove empty JavaScript script-block --- wqflask/wqflask/templates/admin/create_group.html | 6 ------ 1 file changed, 6 deletions(-) diff --git a/wqflask/wqflask/templates/admin/create_group.html b/wqflask/wqflask/templates/admin/create_group.html index 21ef5653..7332ca77 100644 --- a/wqflask/wqflask/templates/admin/create_group.html +++ b/wqflask/wqflask/templates/admin/create_group.html @@ -73,17 +73,11 @@
    - - - {% endblock %} {% block js %} - - {% endblock %} -- cgit v1.2.3 From 42a0bd960095e96b536a51249f2355e82b59fd30 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 8 Nov 2021 15:41:00 +0300 Subject: group_manager.py: Add methods for creating and deleting groups --- wqflask/wqflask/group_manager.py | 59 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/wqflask/wqflask/group_manager.py b/wqflask/wqflask/group_manager.py index 4edafc66..3434eab9 100644 --- a/wqflask/wqflask/group_manager.py +++ b/wqflask/wqflask/group_manager.py @@ -1,10 +1,16 @@ +import json import redis from flask import current_app from flask import Blueprint from flask import g from flask import render_template +from flask import request +from flask import redirect +from flask import url_for from gn3.authentication import get_groups_by_user_uid +from gn3.authentication import get_user_info_by_key +from gn3.authentication import create_group from wqflask.decorators import login_required group_management = Blueprint("group_management", __name__) @@ -29,3 +35,56 @@ def view_groups(): @login_required def view_create_group_page(): return render_template("admin/create_group.html") + + +@group_management.route("/groups/create", methods=("POST",)) +@login_required +def create_new_group(): + conn = redis.from_url(current_app.config["REDIS_URL"], + decode_responses=True) + if group_name := request.form.get("group_name"): + members_uid, admins_uid = set(), set() + admins_uid.add(user_uid := ( + g.user_session.record.get( + b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", ""))) + if admin_string := request.form.get("admin_emails_to_add"): + for email in admin_string.split(","): + user_info = get_user_info_by_key(key="email_address", + value=email, + conn=conn) + if user_uid := user_info.get("user_id"): + admins_uid.add(user_uid) + if member_string := request.form.get("member_emails_to_add"): + for email in member_string.split(","): + user_info = get_user_info_by_key(key="email_address", + value=email, + conn=conn) + if user_uid := user_info.get("user_id"): + members_uid.add(user_uid) + + # Create the new group: + create_group(conn=conn, + group_name=group_name, + member_user_uids=list(members_uid), + admin_user_uids=list(admins_uid)) + return redirect(url_for('group_management.view_groups')) + return redirect(url_for('group_management.create_groups')) + + +@group_management.route("/groups/delete", methods=("POST",)) +@login_required +def delete_groups(): + conn = redis.from_url(current_app.config["REDIS_URL"], + decode_responses=True) + user_uid = (g.user_session.record.get(b"user_id", b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) + current_app.logger.info(request.form.get("selected_group_ids")) + for group_uid in request.form.get("selected_group_ids", "").split(":"): + if group_info := conn.hget("groups", group_uid): + group_info = json.loads(group_info) + # A user who is an admin can delete things + if user_uid in group_info.get("admins"): + conn.hdel("groups", group_uid) + return redirect(url_for('group_management.view_groups')) -- cgit v1.2.3 From 5763744f5044710066b3c94efed0413061412689 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 8 Nov 2021 15:41:25 +0300 Subject: Update URL for removing groups in template --- wqflask/wqflask/templates/admin/group_manager.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/admin/group_manager.html b/wqflask/wqflask/templates/admin/group_manager.html index c5000869..c62cac20 100644 --- a/wqflask/wqflask/templates/admin/group_manager.html +++ b/wqflask/wqflask/templates/admin/group_manager.html @@ -17,7 +17,10 @@ Create Group - +
    {% endif %}
    -- cgit v1.2.3 From 965c25016c25256a1074131ac4c2c79b9ee97161 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 8 Nov 2021 15:41:45 +0300 Subject: Update URL for creating a new group --- wqflask/wqflask/templates/admin/create_group.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/admin/create_group.html b/wqflask/wqflask/templates/admin/create_group.html index 7332ca77..b1d214ea 100644 --- a/wqflask/wqflask/templates/admin/create_group.html +++ b/wqflask/wqflask/templates/admin/create_group.html @@ -6,7 +6,8 @@

    Create Group

    -
    +
    -- cgit v1.2.3 From c11ac641592da44a823fdf41c0b81cd7bc7971f1 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 9 Nov 2021 12:05:48 +0300 Subject: Update URL for viewing a particular group --- wqflask/wqflask/templates/admin/group_manager.html | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/admin/group_manager.html b/wqflask/wqflask/templates/admin/group_manager.html index c62cac20..eedfe138 100644 --- a/wqflask/wqflask/templates/admin/group_manager.html +++ b/wqflask/wqflask/templates/admin/group_manager.html @@ -58,11 +58,12 @@ {{ loop.index }} - {{ group.name }} + {% set group_url = url_for('group_management.view_group', group_id=group.uuid) %} + {{ group.name }} {{ group.admins|length + group.members|length }} {{ group.created_timestamp }} {{ group.changed_timestamp }} - {{ group.id }} + {{ group.uuid }} {% endfor %} @@ -92,7 +93,8 @@ {{ loop.index }} - {{ group.name }} + {% set group_url = url_for('group_management.view_group', group_id=group.uuid) %} + {{ group.name }} {{ group.admins|length + group.members|length }} {{ group.created_timestamp }} {{ group.changed_timestamp }} -- cgit v1.2.3 From e0be9d3d6685b76a01c376655556295606ffea4e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 9 Nov 2021 12:06:50 +0300 Subject: Rename "view_groups" to "display_groups" --- wqflask/wqflask/group_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/group_manager.py b/wqflask/wqflask/group_manager.py index 3434eab9..0e1fa56e 100644 --- a/wqflask/wqflask/group_manager.py +++ b/wqflask/wqflask/group_manager.py @@ -18,7 +18,7 @@ group_management = Blueprint("group_management", __name__) @group_management.route("/groups") @login_required -def view_groups(): +def display_groups(): groups = get_groups_by_user_uid( user_uid=(g.user_session.record.get(b"user_id", b"").decode("utf-8") or @@ -69,7 +69,7 @@ def create_new_group(): group_name=group_name, member_user_uids=list(members_uid), admin_user_uids=list(admins_uid)) - return redirect(url_for('group_management.view_groups')) + return redirect(url_for('group_management.display_groups')) return redirect(url_for('group_management.create_groups')) -- cgit v1.2.3 From e7fb408c3ab09b3aed5c862848f02d28d1a25f35 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 9 Nov 2021 12:52:55 +0300 Subject: view_group.html: Remove redundant bool check --- wqflask/wqflask/templates/admin/view_group.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/templates/admin/view_group.html b/wqflask/wqflask/templates/admin/view_group.html index 26692fe8..b14fd14e 100644 --- a/wqflask/wqflask/templates/admin/view_group.html +++ b/wqflask/wqflask/templates/admin/view_group.html @@ -14,7 +14,7 @@ - {% if user_is_admin == true %} + {% if is_admin %}
    @@ -51,7 +51,7 @@ {% endfor %} - {% if user_is_admin == true %} + {% if is_admin %}
    E-mail of user to add to admins (multiple e-mails can be added separated by commas): @@ -88,7 +88,7 @@ {% endfor %} - {% if user_is_admin == true %} + {% if is_admin %}
    E-mail of user to add to members (multiple e-mails can be added separated by commas): @@ -99,7 +99,7 @@ {% endif %} {% else %} There are currently no members in this group. - {% if user_is_admin == true %} + {% if is_admin %}
    E-mail of user to add to members (multiple e-mails can be added separated by commas): -- cgit v1.2.3 From 61270336f52d93eb1312fe27760962f927fc97c8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 9 Nov 2021 12:53:10 +0300 Subject: view_group.html: Add html indentation --- wqflask/wqflask/templates/admin/view_group.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/admin/view_group.html b/wqflask/wqflask/templates/admin/view_group.html index b14fd14e..abbdcf0e 100644 --- a/wqflask/wqflask/templates/admin/view_group.html +++ b/wqflask/wqflask/templates/admin/view_group.html @@ -1,9 +1,9 @@ {% extends "base.html" %} {% block title %}View and Edit Group{% endblock %} {% block css %} - - - + + + {% endblock %} {% block content %} -- cgit v1.2.3 From 1fe23d19c099305a7f3fe8afa1477964038e57a7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 9 Nov 2021 12:54:05 +0300 Subject: view_group.html: Use GROUP_URL for adding admins and users --- wqflask/wqflask/templates/admin/view_group.html | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/templates/admin/view_group.html b/wqflask/wqflask/templates/admin/view_group.html index abbdcf0e..a89ec54d 100644 --- a/wqflask/wqflask/templates/admin/view_group.html +++ b/wqflask/wqflask/templates/admin/view_group.html @@ -7,6 +7,7 @@ {% endblock %} {% block content %} +{% set GROUP_URL = url_for('group_management.view_group', group_id=guid) %}

    @@ -57,7 +58,7 @@

    - +
    {% endif %}
    @@ -94,7 +95,7 @@
    - +
    {% endif %} {% else %} @@ -230,7 +231,7 @@ new_name = $('input[name=new_group_name]').val() $.ajax({ type: "POST", - url: "/groups/change_name", + url: "{{ GROUP_URL }} ", data: { group_id: $('input[name=group_id]').val(), new_name: new_name -- cgit v1.2.3 From c2ca2dc66ec1b6be726eafde187eee561f074ad4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 9 Nov 2021 12:54:53 +0300 Subject: view_group.html: Display the group's UID if the user is an admin --- wqflask/wqflask/templates/admin/view_group.html | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/wqflask/wqflask/templates/admin/view_group.html b/wqflask/wqflask/templates/admin/view_group.html index a89ec54d..a7d80388 100644 --- a/wqflask/wqflask/templates/admin/view_group.html +++ b/wqflask/wqflask/templates/admin/view_group.html @@ -38,6 +38,9 @@ Name Email Address Organization + {% if is_admin %} + UID + {% endif %} @@ -48,6 +51,9 @@ {% if 'full_name' in admin %}{{ admin.full_name }}{% elif 'name' in admin %}{{ admin.name }}{% else %}N/A{% endif %} {% if 'email_address' in admin %}{{ admin.email_address }}{% else %}N/A{% endif %} {% if 'organization' in admin %}{{ admin.organization }}{% else %}N/A{% endif %} + {% if is_admin %} + {{admin.user_id}} + {% endif %} {% endfor %} @@ -75,6 +81,9 @@ Name Email Address Organization + {% if is_admin %} + UID + {% endif %} @@ -85,6 +94,10 @@ {% if 'full_name' in member %}{{ member.full_name }}{% elif 'name' in admin %}{{ admin.name }}{% else %}N/A{% endif %} {% if 'email_address' in member %}{{ member.email_address }}{% else %}N/A{% endif %} {% if 'organization' in member %}{{ member.organization }}{% else %}N/A{% endif %} + {% if is_admin %} + {{ member }} + {% endif %} + {% endfor %} -- cgit v1.2.3 From b4ed3cb7beca79b89ee05117daae264487d442cb Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 9 Nov 2021 12:55:41 +0300 Subject: view_group.html: Prepend "Name" before the group name --- wqflask/wqflask/templates/admin/view_group.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/admin/view_group.html b/wqflask/wqflask/templates/admin/view_group.html index a7d80388..b4bac7d2 100644 --- a/wqflask/wqflask/templates/admin/view_group.html +++ b/wqflask/wqflask/templates/admin/view_group.html @@ -11,8 +11,8 @@

    - {{ group_info.name }} + Name: {{ group_info.name }}

    {% if is_admin %} -- cgit v1.2.3 From 511d3450e4301173a1c0c8af371b6c9354b5bff1 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 9 Nov 2021 12:56:36 +0300 Subject: view_group.html: Hide the "Change Group Name" button for non-admins --- wqflask/wqflask/templates/admin/view_group.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/admin/view_group.html b/wqflask/wqflask/templates/admin/view_group.html index b4bac7d2..4a3ecede 100644 --- a/wqflask/wqflask/templates/admin/view_group.html +++ b/wqflask/wqflask/templates/admin/view_group.html @@ -11,9 +11,11 @@

    - Name: {{ group_info.name }} + + {% if is_admin %} + {% endif %}

    {% if is_admin %}
    -- cgit v1.2.3 From d545936f3eb8dd491c2f691da0fcdc89722e2c64 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 9 Nov 2021 12:57:14 +0300 Subject: view_group.html: Hide checkbox for non-users --- wqflask/wqflask/templates/admin/view_group.html | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/admin/view_group.html b/wqflask/wqflask/templates/admin/view_group.html index 4a3ecede..4a20c0dd 100644 --- a/wqflask/wqflask/templates/admin/view_group.html +++ b/wqflask/wqflask/templates/admin/view_group.html @@ -91,7 +91,12 @@ {% for member in members %} - + + + {% if is_admin %} + + {% endif %} + {{ loop.index }} {% if 'full_name' in member %}{{ member.full_name }}{% elif 'name' in admin %}{{ admin.name }}{% else %}N/A{% endif %} {% if 'email_address' in member %}{{ member.email_address }}{% else %}N/A{% endif %} -- cgit v1.2.3 From 17b91bc55a502fc90bd0e6b4275253880e99f7c1 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 11 Nov 2021 09:28:26 +0300 Subject: Add new group endpoints --- wqflask/wqflask/group_manager.py | 69 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/group_manager.py b/wqflask/wqflask/group_manager.py index 0e1fa56e..3936e36e 100644 --- a/wqflask/wqflask/group_manager.py +++ b/wqflask/wqflask/group_manager.py @@ -1,5 +1,6 @@ import json import redis +import datetime from flask import current_app from flask import Blueprint @@ -87,4 +88,70 @@ def delete_groups(): # A user who is an admin can delete things if user_uid in group_info.get("admins"): conn.hdel("groups", group_uid) - return redirect(url_for('group_management.view_groups')) + return redirect(url_for('group_management.display_groups')) + + +@group_management.route("/groups/") +@login_required +def view_group(group_id: str): + conn = redis.from_url(current_app.config["REDIS_URL"], + decode_responses=True) + user_uid = (g.user_session.record.get(b"user_id", b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) + + resource_info = [] + for resource_uid, resource in conn.hgetall("resources").items(): + resource = json.loads(resource) + if group_id in (group_mask := resource.get("group_masks")): + __dict = {} + for val in group_mask.values(): + __dict.update(val) + __dict.update({ + "id": resource_uid, + "name": resource.get("name"), + }) + resource_info.append(__dict) + group_info = json.loads(conn.hget("groups", + group_id)) + group_info["guid"] = group_id + + return render_template( + "admin/view_group.html", + group_info=group_info, + admins=[get_user_info_by_key(key="user_id", + value=user_id, + conn=conn) + for user_id in group_info.get("admins")], + members=[get_user_info_by_key(key="user_id", + value=user_id, + conn=conn) + for user_id in group_info.get("members")], + is_admin = (True if user_uid in group_info.get("admins") else False), + resources=resource_info) + + +@group_management.route("/groups/", methods=("POST",)) +def update_group(group_id: str): + conn = redis.from_url(current_app.config["REDIS_URL"], + decode_responses=True) + user_uid = (g.user_session.record.get(b"user_id", b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) + group = json.loads(conn.hget("groups", group_id)) + timestamp = group["changed_timestamp"] + timestamp_ = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + if user_uid in group.get("admins"): + if name := request.form.get("new_name"): + group["name"] = name + group["changed_timestamp"] = timestamp_ + if admins := request.form.get("admin_emails_to_add"): + group["admins"] = list(set(admins.split(":") + + group.get("admins"))) + group["changed_timestamp"] = timestamp_ + if members := request.form.get("member_emails_to_add"): + print(f"\n+++++\n{members}\n+++++\n") + group["members"] = list(set(members.split(":") + + group.get("members"))) + group["changed_timestamp"] = timestamp_ + conn.hset("groups", group_id, json.dumps(group)) + return redirect(url_for('group_management.view_group', + group_id=group_id)) -- cgit v1.2.3 From 166bca11dee1a09d346c4bd72edbf51832f13820 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 11 Nov 2021 09:28:45 +0300 Subject: Update links for updating groups --- wqflask/wqflask/templates/admin/view_group.html | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/templates/admin/view_group.html b/wqflask/wqflask/templates/admin/view_group.html index 4a20c0dd..c88ce0e7 100644 --- a/wqflask/wqflask/templates/admin/view_group.html +++ b/wqflask/wqflask/templates/admin/view_group.html @@ -7,8 +7,9 @@ {% endblock %} {% block content %} -{% set GROUP_URL = url_for('group_management.view_group', group_id=guid) %} -
    +{% set GROUP_URL = url_for('group_management.view_group', group_id=group_info.guid) %} +{% set UPDATE_GROUP_URL = url_for('group_management.update_group', group_id=group_info.guid) %} +

    Name: {{ group_info.name }} @@ -23,7 +24,7 @@

    {% endif %}
    - + @@ -66,7 +67,7 @@
    - +
    {% endif %}
    @@ -126,7 +127,7 @@
    - +
    {% endif %} {% endif %} @@ -240,6 +241,7 @@ $("#add_admins, #add_members").on("click", function() { url = $(this).data("url"); + console.log(url) return submit_special(url) }); -- cgit v1.2.3 From 6ced33f201e8a4e389a077a91ba9ed8bf5c19fa0 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 11 Nov 2021 15:56:31 +0300 Subject: fix issue with number for samples --- wqflask/wqflask/correlation/pre_computes.py | 37 +++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index 1db9f61b..f1c9e1bd 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -6,12 +6,14 @@ import json from typing import List from base import data_set +from gn3.computations.correlations import compute_all_sample_correlation from gn3.computations.correlations import fast_compute_all_sample_correlation from gn3.computations.correlations import map_shared_keys_to_values + def get_dataset_dict_data(dataset_obj): """function to get the dataset data mapped to key""" - dataset_obj.get_trait_data() + dataset_obj.get_trait_data(dataset_obj.group.all_samples_ordered()) return map_shared_keys_to_values(dataset_obj.samplelist, dataset_obj.trait_data) @@ -42,11 +44,21 @@ def pre_compute_sample_correlation(base_dataset: List, precaution:function is expensive;targets only Exon and """ + results = [] + for trait_info in base_dataset: - yield fast_compute_all_sample_correlation(corr_method="pearson", - this_trait=trait_info, - target_dataset=target_dataset) + result = fast_compute_all_sample_correlation(corr_method="pearson", + this_trait=trait_info, + target_dataset=target_dataset) + + # results.append(fast_compute_all_sample_correlation(corr_method="pearson", + # this_trait=trait_info, + # target_dataset=target_dataset)) + print("finished") + print(result) + + return results def cache_to_file(base_dataset_name: str, target_dataset_name: str): @@ -57,16 +69,21 @@ def cache_to_file(base_dataset_name: str, target_dataset_name: str): base_dataset_data, target_dataset_data = [list(dataset) for dataset in list( fetch_datasets(base_dataset_name, target_dataset_name))] + # print(target_dataset_data) try: - with open("unique_file_name.json", "w") as file_handler: - file_handler.write() - - dataset_correlation_results = list(pre_compute_sample_correlation( - base_dataset_data, target_dataset_data)) + # with open("unique_file_name.json", "w") as file_handler: + # file_handler.write() + dataset_correlation_results = pre_compute_sample_correlation( + base_dataset_data, target_dataset_data) print(dataset_correlation_results) - json.dump(dataset_correlation_results, file_handler) + # json.dump(dataset_correlation_results, file_handler) except Exception as error: raise error + + +def check_cached_files_validity(): + """function to check the validity of cached files""" + pass -- cgit v1.2.3 From 5935e2cc3e0ac3a8004ccd5224557d34b62359d8 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 11 Nov 2021 20:28:06 +0300 Subject: code to cache frequently run probeset correlation --- wqflask/wqflask/correlation/pre_computes.py | 124 ++++++++++++++-------------- 1 file changed, 63 insertions(+), 61 deletions(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index f1c9e1bd..d0caca60 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -1,89 +1,91 @@ -"""module contains the code to do the -precomputations of sample data between -two entire datasets""" -import json -from typing import List -from base import data_set +import os +import hashlib -from gn3.computations.correlations import compute_all_sample_correlation -from gn3.computations.correlations import fast_compute_all_sample_correlation -from gn3.computations.correlations import map_shared_keys_to_values +from base.data_set import query_table_timestamp +from base.webqtlConfig import TMPDIR -def get_dataset_dict_data(dataset_obj): - """function to get the dataset data mapped to key""" - dataset_obj.get_trait_data(dataset_obj.group.all_samples_ordered()) - return map_shared_keys_to_values(dataset_obj.samplelist, - dataset_obj.trait_data) +def generate_filename(**kwargs): + """generate unique filename""" + base_dataset_name = kwargs["base_dataset"] + target_dataset_name = kwargs["target_dataset"] + base_timestamp = kwargs["base_timestamp"] + target_dataset_timestamp = kwargs["target_timestamp"] -def fetch_datasets(base_dataset_name: str, target_dataset_name: str) ->List: - """query to fetch create datasets and fetch traits - all traits of a dataset""" + string_unicode = f"{base_dataset_name}{target_dataset_name}{base_timestamp}{target_dataset_timestamp}sample_corr_compute".encode() + hashlib.md5(string_unicode).hexdigest() - # doesnt work for temp - base_dataset = data_set.create_dataset(dataset_name=base_dataset_name) +def cache_compute_results(start_vars, + base_dataset_type, + correlation_results, + trait_name): + # pass - target_dataset = data_set.create_dataset(dataset_name=target_dataset_name) - # replace with map + # init assumption only caching probeset type + # fix redis;issue potential redis_cache!=current_timestamp + base_timestamp = r.get(f"{base_dataset_type}timestamp") - return (map(get_dataset_dict_data, - [base_dataset, target_dataset])) + if base_timestamp is None: + # fetch the timestamp + base_timestamp = target_dataset_timestamp = query_table_timestamp( + dataset_type) + r.set(f"{dataset_type}timestamp", target_dataset_timestamp) -# in the base dataset we just need the traits -def pre_compute_sample_correlation(base_dataset: List, - target_dataset: List) -> List: - """function compute the correlation between the - a whole dataset against a target - input: target&base_dataset(contains traits and sample results) - output: list containing the computed results + file_name = generate_filename( + base_dataset_name, target_dataset_name, + base_timestamp, target_dataset_timestamp) - precaution:function is expensive;targets only Exon and - """ + file_path = os.path.join(TMPDIR, f"{file_name}.json") - results = [] + try: - for trait_info in base_dataset: + with open(file_path, "r+") as json_handler: - result = fast_compute_all_sample_correlation(corr_method="pearson", - this_trait=trait_info, - target_dataset=target_dataset) + results = json.load(json_handler) - # results.append(fast_compute_all_sample_correlation(corr_method="pearson", - # this_trait=trait_info, - # target_dataset=target_dataset)) - print("finished") - print(result) + if results.get(trait_name) is not None: + results.update({trait_name: correlation_results}) - return results + json.dump(results, json_handler) + except FileNotFoundError: + with open(file_path, "w") as json_handler: + json.dump({trait_name: correlation_results}, json_handler) -def cache_to_file(base_dataset_name: str, target_dataset_name: str): - """function to cache the results to file""" +def fetch_precompute_results(base_dataset_name,target_dataset_name,trait_name): + """function to check for precomputed results""" - # validate the datasets expiry first + # check for redis timestamp + + # fix rely on the fact correlation run oftenly probeset is set + + base_timestamp = target_dataset_timestamp = r.get(dataset_type) + + + if base_timestamp is None: + return + + else: + file_name = generate_filename( + base_dataset_name, target_dataset_name, + base_timestamp, target_dataset_timestamp) + + try: + with open(file_path,"r") as json_handler: + correlation_results = json.load(json_handler) + + return correlation_results.get(trait_name) + + except FileNotFoundError: + pass - base_dataset_data, target_dataset_data = [list(dataset) for dataset in list( - fetch_datasets(base_dataset_name, target_dataset_name))] - # print(target_dataset_data) - try: - # with open("unique_file_name.json", "w") as file_handler: - # file_handler.write() - dataset_correlation_results = pre_compute_sample_correlation( - base_dataset_data, target_dataset_data) - print(dataset_correlation_results) - # json.dump(dataset_correlation_results, file_handler) - except Exception as error: - raise error -def check_cached_files_validity(): - """function to check the validity of cached files""" - pass -- cgit v1.2.3 From ca4a5fdda8a7225dc5bebc17c61837ba5373ec68 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 11 Nov 2021 20:29:04 +0300 Subject: minor fix for generating file_name --- wqflask/wqflask/correlation/pre_computes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index d0caca60..55f25f0b 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -15,7 +15,7 @@ def generate_filename(**kwargs): target_dataset_timestamp = kwargs["target_timestamp"] string_unicode = f"{base_dataset_name}{target_dataset_name}{base_timestamp}{target_dataset_timestamp}sample_corr_compute".encode() - hashlib.md5(string_unicode).hexdigest() + return hashlib.md5(string_unicode).hexdigest() def cache_compute_results(start_vars, -- cgit v1.2.3 From b4594a6f2dc5c0c0a8e62a327674126668391d6b Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 11 Nov 2021 20:32:13 +0300 Subject: minor fix for updating dict --- wqflask/wqflask/correlation/pre_computes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index 55f25f0b..4244fcfb 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -47,7 +47,7 @@ def cache_compute_results(start_vars, results = json.load(json_handler) - if results.get(trait_name) is not None: + if results.get(trait_name) is None: results.update({trait_name: correlation_results}) json.dump(results, json_handler) -- cgit v1.2.3 From a20e20c79b054350b84e70af6e7d5ef2a0407786 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 11 Nov 2021 22:18:21 +0300 Subject: pep8 formatting + minor fixing for writing to files --- wqflask/wqflask/correlation/pre_computes.py | 75 +++++++++++++---------------- 1 file changed, 34 insertions(+), 41 deletions(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index 4244fcfb..1d832fde 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -1,28 +1,28 @@ - +import json import os import hashlib from base.data_set import query_table_timestamp from base.webqtlConfig import TMPDIR +from redis import Redis +r = Redis() -def generate_filename(**kwargs): - """generate unique filename""" - base_dataset_name = kwargs["base_dataset"] - target_dataset_name = kwargs["target_dataset"] - base_timestamp = kwargs["base_timestamp"] - target_dataset_timestamp = kwargs["target_timestamp"] +def generate_filename(base_dataset_name, target_dataset_name, base_timestamp, target_dataset_timestamp): + """generate unique filename""" string_unicode = f"{base_dataset_name}{target_dataset_name}{base_timestamp}{target_dataset_timestamp}sample_corr_compute".encode() return hashlib.md5(string_unicode).hexdigest() -def cache_compute_results(start_vars, - base_dataset_type, - correlation_results, - trait_name): +def cache_compute_results(base_dataset_type, + base_dataset_name, + target_dataset_name, + correlation_results, + trait_name): # pass + """function to cache correlation results for heavy computations""" # init assumption only caching probeset type # fix redis;issue potential redis_cache!=current_timestamp @@ -30,10 +30,11 @@ def cache_compute_results(start_vars, if base_timestamp is None: # fetch the timestamp - base_timestamp = target_dataset_timestamp = query_table_timestamp( - dataset_type) + base_timestamp = query_table_timestamp( + base_dataset_type) + r.set(f"{base_dataset_type}timestamp", base_timestamp) - r.set(f"{dataset_type}timestamp", target_dataset_timestamp) + target_dataset_timestamp = base_timestamp file_name = generate_filename( base_dataset_name, target_dataset_name, @@ -41,51 +42,43 @@ def cache_compute_results(start_vars, file_path = os.path.join(TMPDIR, f"{file_name}.json") - try: + try: + + with open(file_path, "r+") as json_handler: - with open(file_path, "r+") as json_handler: + results = json.load(json_handler) + results[trait_name] = correlation_results - results = json.load(json_handler) + json.dump(results, json_handler) - if results.get(trait_name) is None: - results.update({trait_name: correlation_results}) + except FileNotFoundError: - json.dump(results, json_handler) + with open(file_path, "w+") as write_json_handler: + json.dump({trait_name: correlation_results}, write_json_handler) - except FileNotFoundError: - with open(file_path, "w") as json_handler: - json.dump({trait_name: correlation_results}, json_handler) -def fetch_precompute_results(base_dataset_name,target_dataset_name,trait_name): +def fetch_precompute_results(base_dataset_name, target_dataset_name, dataset_type, trait_name): """function to check for precomputed results""" # check for redis timestamp # fix rely on the fact correlation run oftenly probeset is set - base_timestamp = target_dataset_timestamp = r.get(dataset_type) - + base_timestamp = target_dataset_timestamp = r.get(f"{dataset_type}timestamp") if base_timestamp is None: return - else: - file_name = generate_filename( + file_name = generate_filename( base_dataset_name, target_dataset_name, base_timestamp, target_dataset_timestamp) - try: - with open(file_path,"r") as json_handler: - correlation_results = json.load(json_handler) - - return correlation_results.get(trait_name) - - except FileNotFoundError: - pass - - - - - + file_path = os.path.join(TMPDIR, f"{file_name}.json") + try: + with open(file_path, "r") as json_handler: + correlation_results = json.load(json_handler) + return correlation_results.get(trait_name) + except FileNotFoundError: + pass -- cgit v1.2.3 From 50580bf1d8114c8cc0fafb429232fa29fab10415 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 12 Nov 2021 06:59:28 +0000 Subject: Added check for too_many_results before writing the table JS --- wqflask/wqflask/templates/search_result_page.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 95842316..dade6ba5 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -176,7 +176,7 @@ return params; }; - {% if results|count > 0 %} + {% if results|count > 0 and not too_many_results %} var tableId = "trait_table"; var width_change = 0; //ZS: For storing the change in width so overall table width can be adjusted by that amount -- cgit v1.2.3 From d6398ea7db0072694b0d8f79d909c276fae75bdd Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 12 Nov 2021 07:08:12 +0000 Subject: Change the max results to display depending upon data type, since ProbeSet results can be displayed in larger numbers more easily --- wqflask/wqflask/search_results.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 5ca1f9ca..33f9319c 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -62,8 +62,10 @@ class SearchResultPage: self.search_term_exists = True self.results = [] + max_result_count = 100000 # max number of results to display type = kw.get('type') if type == "Phenotypes": # split datatype on type field + max_result_count = 50000 dataset_type = "Publish" elif type == "Genotypes": dataset_type = "Geno" @@ -81,7 +83,7 @@ class SearchResultPage: self.too_many_results = False if self.search_term_exists: - if len(self.results) > 50000: + if len(self.results) > max_result_count: self.trait_list = [] self.too_many_results = True else: -- cgit v1.2.3 From b983afd6be0b8fc5ae8065a1b0745296645da593 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Fri, 12 Nov 2021 12:47:37 +0300 Subject: bugfix:cases where trait symbol is none --- wqflask/wqflask/correlation/correlation_gn3_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 1e3a40f2..20c0d99a 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -290,7 +290,7 @@ def get_tissue_correlation_input(this_trait, trait_symbol_dict): """Gets tissue expression values for the primary trait and target tissues values""" primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( symbol_list=[this_trait.symbol]) - if this_trait.symbol.lower() in primary_trait_tissue_vals_dict: + if this_trait.symbol and this_trait.symbol.lower() in primary_trait_tissue_vals_dict: primary_trait_tissue_values = primary_trait_tissue_vals_dict[this_trait.symbol.lower( )] corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( -- cgit v1.2.3 From e2647916b051f5d46ccb24e08a6c97cbb8f668f2 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 12 Nov 2021 16:27:03 +0000 Subject: This change prevents an error caused by trying to export mapping results for a ProbeSet trait when there is no gene symbol --- wqflask/wqflask/marker_regression/run_mapping.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index 769b9240..9d70bb15 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -556,7 +556,8 @@ def export_mapping_results(dataset, trait, markers, results_path, mapping_method transform_text = "" output_file.write(transform_text + "\n") if dataset.type == "ProbeSet": - output_file.write("Gene Symbol: " + trait.symbol + "\n") + if trait.symbol: + output_file.write("Gene Symbol: " + trait.symbol + "\n") output_file.write("Location: " + str(trait.chr) + \ " @ " + str(trait.mb) + " Mb\n") if len(covariates) > 0: -- cgit v1.2.3 From 01d42255f52a61c6d3d007ffd1e5e02765a76730 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 15 Nov 2021 17:38:09 +0300 Subject: fix for truncating files --- wqflask/wqflask/correlation/pre_computes.py | 53 ++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index 1d832fde..01fa1a3d 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -4,6 +4,7 @@ import hashlib from base.data_set import query_table_timestamp from base.webqtlConfig import TMPDIR +from json.decoder import JSONDecodeError from redis import Redis r = Redis() @@ -26,35 +27,49 @@ def cache_compute_results(base_dataset_type, # init assumption only caching probeset type # fix redis;issue potential redis_cache!=current_timestamp - base_timestamp = r.get(f"{base_dataset_type}timestamp") - if base_timestamp is None: - # fetch the timestamp - base_timestamp = query_table_timestamp( - base_dataset_type) - r.set(f"{base_dataset_type}timestamp", base_timestamp) + base_timestamp = query_table_timestamp(base_dataset_type) + + r.set(f"{base_dataset_type}timestamp", base_timestamp) target_dataset_timestamp = base_timestamp + + file_name = generate_filename( base_dataset_name, target_dataset_name, base_timestamp, target_dataset_timestamp) - file_path = os.path.join(TMPDIR, f"{file_name}.json") + + file_path = os.path.join(TMPDIR,f"{file_name}.json") + try: + with open(file_path,"r+") as json_file_handler: + data = json.load(json_file_handler) - with open(file_path, "r+") as json_handler: + data[trait_name] = correlation_results - results = json.load(json_handler) - results[trait_name] = correlation_results + json_file_handler.seek(0) - json.dump(results, json_handler) + json.dump(data,json_file_handler) + json_file_handler.truncate() + except FileNotFoundError: + with open(file_path,"w+") as file_handler: + data = {} + data[trait_name] =correlation_results + + json.dump(data,file_handler) - with open(file_path, "w+") as write_json_handler: - json.dump({trait_name: correlation_results}, write_json_handler) + + + # create the file only if it does not exists + + # else open the file to cache the results + + def fetch_precompute_results(base_dataset_name, target_dataset_name, dataset_type, trait_name): @@ -65,20 +80,26 @@ def fetch_precompute_results(base_dataset_name, target_dataset_name, dataset_typ # fix rely on the fact correlation run oftenly probeset is set base_timestamp = target_dataset_timestamp = r.get(f"{dataset_type}timestamp") - if base_timestamp is None: return + else: + base_timestamp = target_dataset_timestamp = base_timestamp.decode("utf-8") + file_name = generate_filename( base_dataset_name, target_dataset_name, base_timestamp, target_dataset_timestamp) file_path = os.path.join(TMPDIR, f"{file_name}.json") + results = None + try: - with open(file_path, "r") as json_handler: + with open(file_path, "r+") as json_handler: correlation_results = json.load(json_handler) - return correlation_results.get(trait_name) + # print(correlation_results) + + return correlation_results.get(trait_name) except FileNotFoundError: pass -- cgit v1.2.3 From 18b53441a0136071db94c72b112a746e056ef971 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 15 Nov 2021 18:03:55 +0300 Subject: refactor function to fetch datasets data for precomputes --- wqflask/wqflask/correlation/pre_computes.py | 72 ++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 16 deletions(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index 01fa1a3d..e7147ddf 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -20,6 +20,7 @@ def generate_filename(base_dataset_name, target_dataset_name, base_timestamp, ta def cache_compute_results(base_dataset_type, base_dataset_name, target_dataset_name, + corr_method, correlation_results, trait_name): # pass @@ -34,43 +35,35 @@ def cache_compute_results(base_dataset_type, target_dataset_timestamp = base_timestamp - - file_name = generate_filename( base_dataset_name, target_dataset_name, base_timestamp, target_dataset_timestamp) - - file_path = os.path.join(TMPDIR,f"{file_name}.json") - + file_path = os.path.join(TMPDIR, f"{file_name}.json") try: - with open(file_path,"r+") as json_file_handler: + with open(file_path, "r+") as json_file_handler: data = json.load(json_file_handler) data[trait_name] = correlation_results json_file_handler.seek(0) - json.dump(data,json_file_handler) + json.dump(data, json_file_handler) json_file_handler.truncate() - + except FileNotFoundError: - with open(file_path,"w+") as file_handler: + with open(file_path, "w+") as file_handler: data = {} - data[trait_name] =correlation_results - - json.dump(data,file_handler) - + data[trait_name] = correlation_results + json.dump(data, file_handler) # create the file only if it does not exists # else open the file to cache the results - - def fetch_precompute_results(base_dataset_name, target_dataset_name, dataset_type, trait_name): """function to check for precomputed results""" @@ -84,7 +77,8 @@ def fetch_precompute_results(base_dataset_name, target_dataset_name, dataset_typ return else: - base_timestamp = target_dataset_timestamp = base_timestamp.decode("utf-8") + base_timestamp = target_dataset_timestamp = base_timestamp.decode( + "utf-8") file_name = generate_filename( base_dataset_name, target_dataset_name, @@ -103,3 +97,49 @@ def fetch_precompute_results(base_dataset_name, target_dataset_name, dataset_typ except FileNotFoundError: pass + + +def pre_compute_dataset_vs_dataset(base_dataset, target_dataset, corr_method): + """compute sample correlation between dataset vs dataset + wn:heavy function should be invoked less frequently + input:datasets_data(two dicts),corr_method + + output:correlation results for entire dataset against entire dataset + """ + dataset_correlation_results = {} + + for (trait_name, strain_values) in target_dataset.trait_data: + + this_trait_data = { + "trait_sample_data": strain_values, + "trait_id": trait_name + } + + trait_correlation_result = fast_compute_all_sample_correlation( + corr_method=corr_method, this_trait=this_trait_data, target_dataset=target_dataset_data) + + dataset_correlation_results[trait_name] = trait_correlation_result + + return dataset_correlation_results + + +def get_datasets_data(base_dataset, target_dataset_data): + """required to pass data in a given format to the pre compute + function + + output:two dicts for datasets with key==trait and value==strains + """ + target_traits_data = target_dataset.get_trait_data( + base_dataset.group.all_samples_ordered()) + + base_traits_data = base_dataset.get_trait_data( + base_dataset.group.all_samples_ordered()) + + samples_fetched = base_dataset.group.all_samples_ordered() + + target_results = map_shared_keys_to_values( + samples_fetched, target_traits_data) + base_results = map_shared_keys_to_values( + samples_fetched, base_traits_data) + + return (target_results, base_results) -- cgit v1.2.3 From 6f6e28d216e0a2adbf939b9b29f8794ae45d9aa8 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 15 Nov 2021 18:08:10 +0300 Subject: pep8 formatting & fix variable names --- wqflask/wqflask/correlation/pre_computes.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index e7147ddf..d8629706 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -108,17 +108,20 @@ def pre_compute_dataset_vs_dataset(base_dataset, target_dataset, corr_method): """ dataset_correlation_results = {} - for (trait_name, strain_values) in target_dataset.trait_data: + target_traits_data, base_traits_data = get_datasets_data( + base_dataset, target_dataset_data) + + for (primary_trait_name, strain_values) in base_traits_data: this_trait_data = { "trait_sample_data": strain_values, - "trait_id": trait_name + "trait_id": primary_trait_name } trait_correlation_result = fast_compute_all_sample_correlation( - corr_method=corr_method, this_trait=this_trait_data, target_dataset=target_dataset_data) + corr_method=corr_method, this_trait=this_trait_data, target_dataset=target_traits_data) - dataset_correlation_results[trait_name] = trait_correlation_result + dataset_correlation_results[primary_trait_name] = trait_correlation_result return dataset_correlation_results -- cgit v1.2.3 From 2982ba491e89acee5ead69206691c786be1cf728 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 15 Nov 2021 18:10:56 +0300 Subject: test precomppute caching integration --- wqflask/wqflask/correlation/correlation_gn3_api.py | 23 +++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 7b828016..191a748a 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -4,7 +4,8 @@ import time from functools import wraps from wqflask.correlation import correlation_functions - +from wqflask.correlation.pre_computes import fetch_precompute_results +from wqflask.correlation.pre_computes import cache_compute_results from base import data_set from base.trait import create_trait @@ -193,9 +194,21 @@ def compute_correlation(start_vars, method="pearson", compute_all=False): (this_trait_data, target_dataset_data) = fetch_sample_data( start_vars, this_trait, this_dataset, target_dataset) - correlation_results = fast_compute_all_sample_correlation(corr_method=method, - this_trait=this_trait_data, - target_dataset=target_dataset_data) + correlation_results = fetch_precompute_results( + this_dataset.name, target_dataset.name, this_dataset.type, this_trait.name) + + if correlation_results is None: + correlation_results = fast_compute_all_sample_correlation(corr_method=method, + this_trait=this_trait_data, + target_dataset=target_dataset_data) + + cache_compute_results(this_dataset.type, + this_dataset.name, + target_dataset.name, + corr_method, + correlation_results, + this_trait.name) + elif corr_type == "tissue": trait_symbol_dict = this_dataset.retrieve_genes("Symbol") tissue_input = get_tissue_correlation_input( @@ -295,7 +308,7 @@ def get_tissue_correlation_input(this_trait, trait_symbol_dict): """Gets tissue expression values for the primary trait and target tissues values""" primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( symbol_list=[this_trait.symbol]) - if this_trait.symbol.lower() in primary_trait_tissue_vals_dict: + if this_trait.symbol and this_trait.symbol.lower() in primary_trait_tissue_vals_dict: primary_trait_tissue_values = primary_trait_tissue_vals_dict[this_trait.symbol.lower( )] corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( -- cgit v1.2.3 From aab6393dd60872a6a3b6e7db2a7c087c4ec41295 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 15 Nov 2021 18:18:03 +0300 Subject: fetch only strains from the primary datasets --- wqflask/wqflask/correlation/pre_computes.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index d8629706..355701f2 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -86,7 +86,6 @@ def fetch_precompute_results(base_dataset_name, target_dataset_name, dataset_typ file_path = os.path.join(TMPDIR, f"{file_name}.json") - results = None try: with open(file_path, "r+") as json_handler: @@ -130,15 +129,19 @@ def get_datasets_data(base_dataset, target_dataset_data): """required to pass data in a given format to the pre compute function + (works for bxd only probeset datasets) + + # fix issue with fetching of the datasets + output:two dicts for datasets with key==trait and value==strains """ + samples_fetched = base_dataset.group.all_samples_ordered() target_traits_data = target_dataset.get_trait_data( - base_dataset.group.all_samples_ordered()) + samples_fetched) base_traits_data = base_dataset.get_trait_data( - base_dataset.group.all_samples_ordered()) + samples_fetched) - samples_fetched = base_dataset.group.all_samples_ordered() target_results = map_shared_keys_to_values( samples_fetched, target_traits_data) -- cgit v1.2.3 From 04452c274d51621a0cab1b8dce5b8101c69496b6 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 16 Nov 2021 14:41:41 +0300 Subject: refactor:fix on the query :modify cache point --- wqflask/base/data_set.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 70c58136..a3a720ad 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -747,7 +747,9 @@ class DataSet: and Species.name = '{}' """.format(create_in_clause(self.samplelist), *mescape(self.group.species)) results = dict(g.db.execute(query).fetchall()) - sample_ids = [results[item] for item in self.samplelist] + sample_ids = [results.get(item) for item in self.samplelist] + + sample_ids = [ids for ids in sample_ids if ids is not None] # MySQL limits the number of tables that can be used in a join to 61, # so we break the sample ids into smaller chunks @@ -800,25 +802,22 @@ class DataSet: results = g.db.execute(query).fetchall() trait_sample_data.append([list(result) for result in results]) - cache_dataset_results( - self.name, self.type, trait_sample_data) + trait_count = len(trait_sample_data[0]) + self.trait_data = collections.defaultdict(list) - else: - trait_sample_data = cached_results - - trait_count = len(trait_sample_data[0]) - self.trait_data = collections.defaultdict(list) - - # put all of the separate data together into a dictionary where the keys are - # trait names and values are lists of sample values - data_start_pos = 1 - for trait_counter in range(trait_count): - trait_name = trait_sample_data[0][trait_counter][0] - for chunk_counter in range(int(number_chunks)): - self.trait_data[trait_name] += ( + data_start_pos = 1 + for trait_counter in range(trait_count): + trait_name = trait_sample_data[0][trait_counter][0] + for chunk_counter in range(int(number_chunks)): + self.trait_data[trait_name] += ( trait_sample_data[chunk_counter][trait_counter][data_start_pos:]) + cache_dataset_results( + self.name, self.type, self.trait_data) + + else: + self.trait_data = cached_results class PhenotypeDataSet(DataSet): DS_NAME_MAP['Publish'] = 'PhenotypeDataSet' @@ -1282,7 +1281,9 @@ def generate_hash_file(dataset_name: str, dataset_timestamp: str): def cache_dataset_results(dataset_name: str, dataset_type: str, query_results: List): - """function to cache dataset query results to file""" + """function to cache dataset query results to file + input dataset_name and type query_results(already processed in default dict format) + """ # data computations actions # store the file path on redis -- cgit v1.2.3 From a8ccaf03ba151f9ceca2f0224af33db230a8c8b3 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 16 Nov 2021 15:53:50 +0300 Subject: test generate new files --- wqflask/base/data_set.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index a3a720ad..cae1a2a7 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -810,7 +810,7 @@ class DataSet: trait_name = trait_sample_data[0][trait_counter][0] for chunk_counter in range(int(number_chunks)): self.trait_data[trait_name] += ( - trait_sample_data[chunk_counter][trait_counter][data_start_pos:]) + trait_sample_data[chunk_counter][trait_counter][data_start_pos:]) cache_dataset_results( self.name, self.type, self.trait_data) @@ -818,6 +818,8 @@ class DataSet: else: self.trait_data = cached_results + + class PhenotypeDataSet(DataSet): DS_NAME_MAP['Publish'] = 'PhenotypeDataSet' @@ -1291,7 +1293,7 @@ def cache_dataset_results(dataset_name: str, dataset_type: str, query_results: L results = r.set(f"{dataset_type}timestamp", table_timestamp) - file_name = generate_hash_file(dataset_name, table_timestamp) + file_name = generate_hash_file(dataset_name, dataset_type, table_timestamp) file_path = os.path.join(TMPDIR, f"{file_name}.json") with open(file_path, "w") as file_handler: @@ -1308,7 +1310,7 @@ def fetch_cached_results(dataset_name: str, dataset_type: str): else: table_timestamp = "" - file_name = generate_hash_file(dataset_name, table_timestamp) + file_name = generate_hash_file(dataset_name, dataset_type, table_timestamp) file_path = os.path.join(TMPDIR, f"{file_name}.json") try: with open(file_path, "r") as file_handler: -- cgit v1.2.3 From 4725a8c20d1d4209d59b3b113f00bbc467c5bd31 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 16 Nov 2021 19:43:15 +0300 Subject: init disable fast compute:memory fork issues --- wqflask/wqflask/correlation/correlation_gn3_api.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 191a748a..635ef5ed 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -194,20 +194,8 @@ def compute_correlation(start_vars, method="pearson", compute_all=False): (this_trait_data, target_dataset_data) = fetch_sample_data( start_vars, this_trait, this_dataset, target_dataset) - correlation_results = fetch_precompute_results( - this_dataset.name, target_dataset.name, this_dataset.type, this_trait.name) - - if correlation_results is None: - correlation_results = fast_compute_all_sample_correlation(corr_method=method, - this_trait=this_trait_data, - target_dataset=target_dataset_data) - - cache_compute_results(this_dataset.type, - this_dataset.name, - target_dataset.name, - corr_method, - correlation_results, - this_trait.name) + correlation_results = compute_all_sample_correlation( + corr_method=method, this_trait=this_trait_data, target_dataset=target_dataset_data) elif corr_type == "tissue": trait_symbol_dict = this_dataset.retrieve_genes("Symbol") -- cgit v1.2.3 From 06fbab6427cadf7706da4e954874a7e5da1bd32d Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 16 Nov 2021 19:48:11 +0300 Subject: pep8 formatting remove debug statements --- wqflask/base/data_set.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index cae1a2a7..37f35121 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -1263,7 +1263,7 @@ def query_table_timestamp(dataset_type: str): # computation data and actions query_update_time = f""" - SELECT UPDATE_TIME FROM information_schema.tables + SELECT UPDATE_TIME FROM information_schfema.tables WHERE TABLE_SCHEMA = 'db_webqtl_s' AND TABLE_NAME = '{dataset_type}Data' """ @@ -1275,7 +1275,7 @@ def query_table_timestamp(dataset_type: str): return date_time_obj.strftime(f) -def generate_hash_file(dataset_name: str, dataset_timestamp: str): +def generate_hash_file(dataset_name: str, dataset_type: str, dataset_timestamp: str): """given the trait_name generate a unique name for this""" string_unicode = f"{dataset_name}{dataset_timestamp}".encode() md5hash = hashlib.md5(string_unicode) @@ -1317,5 +1317,4 @@ def fetch_cached_results(dataset_name: str, dataset_type: str): return json.load(file_handler) except FileNotFoundError: - # take actions continue to fetch dataset results and fetch results - pass + pass \ No newline at end of file -- cgit v1.2.3 From 679051788a475dfcefd4cb93dc82ec3a4b86edc3 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 16 Nov 2021 19:54:55 +0300 Subject: use comprehension list;fix typo --- wqflask/base/data_set.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 37f35121..553530d4 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -747,9 +747,7 @@ class DataSet: and Species.name = '{}' """.format(create_in_clause(self.samplelist), *mescape(self.group.species)) results = dict(g.db.execute(query).fetchall()) - sample_ids = [results.get(item) for item in self.samplelist] - - sample_ids = [ids for ids in sample_ids if ids is not None] + sample_ids = [results.get(item) for item in self.samplelist if item is not None] # MySQL limits the number of tables that can be used in a join to 61, # so we break the sample ids into smaller chunks @@ -1263,7 +1261,7 @@ def query_table_timestamp(dataset_type: str): # computation data and actions query_update_time = f""" - SELECT UPDATE_TIME FROM information_schfema.tables + SELECT UPDATE_TIME FROM information_schema.tables WHERE TABLE_SCHEMA = 'db_webqtl_s' AND TABLE_NAME = '{dataset_type}Data' """ @@ -1317,4 +1315,4 @@ def fetch_cached_results(dataset_name: str, dataset_type: str): return json.load(file_handler) except FileNotFoundError: - pass \ No newline at end of file + pass -- cgit v1.2.3 From cddae6a6593a3f65cdb9565207a9a58c338e9024 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 17 Nov 2021 01:19:00 +0300 Subject: temp disable query caching --- wqflask/base/data_set.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 70c58136..768ad49b 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -754,8 +754,8 @@ class DataSet: # Postgres doesn't have that limit, so we can get rid of this after we transition chunk_size = 50 number_chunks = int(math.ceil(len(sample_ids) / chunk_size)) - cached_results = fetch_cached_results(self.name, self.type) - # cached_results = None + # cached_results = fetch_cached_results(self.name, self.type) + cached_results = None if cached_results is None: trait_sample_data = [] for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): @@ -800,8 +800,6 @@ class DataSet: results = g.db.execute(query).fetchall() trait_sample_data.append([list(result) for result in results]) - cache_dataset_results( - self.name, self.type, trait_sample_data) else: trait_sample_data = cached_results -- cgit v1.2.3 From f5dd4b9f6b02ebe20733df0e8bd5e1b5e89f9e93 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 17 Nov 2021 01:31:38 +0300 Subject: temp disable traits metadata cache --- wqflask/wqflask/correlation/show_corr_results.py | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 42010a1e..55915a74 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -86,30 +86,17 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe corr_results = correlation_data['correlation_results'] results_list = [] - file_name = f"{target_dataset['name']}_metadata.json" - - file_path = os.path.join(TMPDIR, file_name) - new_traits_metadata = {} - - try: - with open(file_path,"r+") as file_handler: - dataset_metadata = json.load(file_handler) - - except FileNotFoundError: - Path(file_path).touch(exist_ok=True) - dataset_metadata = {} for i, trait_dict in enumerate(corr_results): trait_name = list(trait_dict.keys())[0] trait = trait_dict[trait_name] - target_trait = dataset_metadata.get(trait_name) + target_trait = None if target_trait is None: target_trait_ob = create_trait(dataset=target_dataset_ob, name=trait_name, get_qtl_info=True) target_trait = jsonable(target_trait_ob, target_dataset_ob) - new_traits_metadata[trait_name] = target_trait if target_trait['view'] == False: continue results_dict = {} @@ -184,12 +171,6 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe results_list.append(results_dict) - - if bool(new_traits_metadata): - # that means new traits exists - dataset_metadata.update(new_traits_metadata) - with open(file_path,"w+") as file_handler: - json.dump(dataset_metadata, file_handler) return json.dumps(results_list) -- cgit v1.2.3 From 60fe836dc6c2f00cb99844572eb3fd29aee0163e Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 17 Nov 2021 08:07:02 +0300 Subject: use a dynamic value for the db_name --- wqflask/base/data_set.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 553530d4..2e401c8e 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -40,6 +40,7 @@ from base import species from base import webqtlConfig from flask import Flask, g from base.webqtlConfig import TMPDIR +from gn3.db_utils import parse_db_url import os import math import string @@ -747,7 +748,8 @@ class DataSet: and Species.name = '{}' """.format(create_in_clause(self.samplelist), *mescape(self.group.species)) results = dict(g.db.execute(query).fetchall()) - sample_ids = [results.get(item) for item in self.samplelist if item is not None] + sample_ids = [results.get(item) + for item in self.samplelist if item is not None] # MySQL limits the number of tables that can be used in a join to 61, # so we break the sample ids into smaller chunks @@ -1260,9 +1262,11 @@ def query_table_timestamp(dataset_type: str): # computation data and actions + fetch_db_name = parse_db_url() + query_update_time = f""" SELECT UPDATE_TIME FROM information_schema.tables - WHERE TABLE_SCHEMA = 'db_webqtl_s' + WHERE TABLE_SCHEMA = {fetch_db_name[-1]} AND TABLE_NAME = '{dataset_type}Data' """ -- cgit v1.2.3 From 71a859c9facc7ae49d43e3e995166ad8dcb586cb Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 17 Nov 2021 08:11:12 +0300 Subject: isolate SQL_URI parse to a function --- wqflask/base/data_set.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 2e401c8e..f0a930a5 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -40,7 +40,8 @@ from base import species from base import webqtlConfig from flask import Flask, g from base.webqtlConfig import TMPDIR -from gn3.db_utils import parse_db_url +from urllib.parse import urlparse +from utility.tools import SQL_URI import os import math import string @@ -1257,6 +1258,13 @@ def geno_mrna_confidentiality(ob): return True + +def parse_db_url(): + parsed_db = urlparse(SQL_URI) + + return (parsed_db.hostname, parsed_db.username, + parsed_db.password, parsed_db.path[1:]) + def query_table_timestamp(dataset_type: str): """function to query the update timestamp of a given dataset_type""" -- cgit v1.2.3 From 5a407a34442860ebaea2886f2278be9e1eb33a8d Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 17 Nov 2021 08:13:41 +0300 Subject: replace redis fetch for cached timestamp with a query --- wqflask/base/data_set.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index f0a930a5..52d1d254 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -1301,8 +1301,6 @@ def cache_dataset_results(dataset_name: str, dataset_type: str, query_results: L table_timestamp = query_table_timestamp(dataset_type) - results = r.set(f"{dataset_type}timestamp", table_timestamp) - file_name = generate_hash_file(dataset_name, dataset_type, table_timestamp) file_path = os.path.join(TMPDIR, f"{file_name}.json") @@ -1313,12 +1311,7 @@ def cache_dataset_results(dataset_name: str, dataset_type: str, query_results: L def fetch_cached_results(dataset_name: str, dataset_type: str): """function to fetch the cached results""" - table_timestamp = r.get(f"{dataset_type}timestamp") - - if table_timestamp is not None: - table_timestamp = table_timestamp.decode("utf-8") - else: - table_timestamp = "" + table_timestamp = query_table_timestamp(dataset_type) file_name = generate_hash_file(dataset_name, dataset_type, table_timestamp) file_path = os.path.join(TMPDIR, f"{file_name}.json") -- cgit v1.2.3 From 1090674ac9497dad22803e7bf8e51d77245f8a0c Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 17 Nov 2021 08:27:05 +0300 Subject: isolate function to fetch the traits metadata --- wqflask/wqflask/correlation/pre_computes.py | 32 +++++++++++++++++------------ 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index 355701f2..638ae860 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -9,6 +9,25 @@ from redis import Redis r = Redis() +# code to isolate metadata caching + + +def fetch_all_cached_metadata(dataset_name): + """in a gvein dataset fetch all the traits metadata""" + file_name = f"{dataset_name}_metadata.json" + + file_path = os.path.join(TMPDIR, file_name) + + with open(file_path, "r+") as file_handler: + dataset_metadata = json.load(file_handler) + + except FileNotFoundError: + Path(file_path).touch(exist_ok=True) + return {} + + return dataset_metadata + + def generate_filename(base_dataset_name, target_dataset_name, base_timestamp, target_dataset_timestamp): """generate unique filename""" @@ -60,18 +79,10 @@ def cache_compute_results(base_dataset_type, json.dump(data, file_handler) - # create the file only if it does not exists - - # else open the file to cache the results - def fetch_precompute_results(base_dataset_name, target_dataset_name, dataset_type, trait_name): """function to check for precomputed results""" - # check for redis timestamp - - # fix rely on the fact correlation run oftenly probeset is set - base_timestamp = target_dataset_timestamp = r.get(f"{dataset_type}timestamp") if base_timestamp is None: return @@ -86,11 +97,9 @@ def fetch_precompute_results(base_dataset_name, target_dataset_name, dataset_typ file_path = os.path.join(TMPDIR, f"{file_name}.json") - try: with open(file_path, "r+") as json_handler: correlation_results = json.load(json_handler) - # print(correlation_results) return correlation_results.get(trait_name) @@ -131,8 +140,6 @@ def get_datasets_data(base_dataset, target_dataset_data): (works for bxd only probeset datasets) - # fix issue with fetching of the datasets - output:two dicts for datasets with key==trait and value==strains """ samples_fetched = base_dataset.group.all_samples_ordered() @@ -142,7 +149,6 @@ def get_datasets_data(base_dataset, target_dataset_data): base_traits_data = base_dataset.get_trait_data( samples_fetched) - target_results = map_shared_keys_to_values( samples_fetched, target_traits_data) base_results = map_shared_keys_to_values( -- cgit v1.2.3 From c872594d21ab743ae55ae4f1d037d13394ef8c67 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 17 Nov 2021 08:34:24 +0300 Subject: isolate function to cache new traits metadata --- wqflask/wqflask/correlation/pre_computes.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index 638ae860..9270bdd4 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -1,6 +1,7 @@ import json import os import hashlib +from pathlib import Path from base.data_set import query_table_timestamp from base.webqtlConfig import TMPDIR @@ -25,8 +26,22 @@ def fetch_all_cached_metadata(dataset_name): Path(file_path).touch(exist_ok=True) return {} - return dataset_metadata + return (file_path, dataset_metadata) + if bool(new_traits_metadata): + # that means new traits exists + dataset_metadata.update(new_traits_metadata) + with open(file_path, "w+") as file_handler: + json.dump(dataset_metadata, file_handler) + + +def cache_new_traits_metadata(dataset_metadata: dict, new_traits_metadata, file_path: str): + """function to cache the new traits metadata""" + + if bool(new_traits_metadata): + dataset_metadata.update(new_traits_metadata) + with open(file_path,"w+") as file_handler: + json.dump(dataset_metadata,file_handler) def generate_filename(base_dataset_name, target_dataset_name, base_timestamp, target_dataset_timestamp): -- cgit v1.2.3 From a35ae60965d7cada41acad661afd88a8fc58e78e Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 17 Nov 2021 08:42:47 +0300 Subject: pep8 formatting;delete remove redis dependency --- wqflask/wqflask/correlation/pre_computes.py | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index 9270bdd4..403d60c9 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -5,12 +5,6 @@ from pathlib import Path from base.data_set import query_table_timestamp from base.webqtlConfig import TMPDIR -from json.decoder import JSONDecodeError -from redis import Redis - -r = Redis() - -# code to isolate metadata caching def fetch_all_cached_metadata(dataset_name): @@ -28,20 +22,14 @@ def fetch_all_cached_metadata(dataset_name): return (file_path, dataset_metadata) - if bool(new_traits_metadata): - # that means new traits exists - dataset_metadata.update(new_traits_metadata) - with open(file_path, "w+") as file_handler: - json.dump(dataset_metadata, file_handler) - def cache_new_traits_metadata(dataset_metadata: dict, new_traits_metadata, file_path: str): """function to cache the new traits metadata""" if bool(new_traits_metadata): dataset_metadata.update(new_traits_metadata) - with open(file_path,"w+") as file_handler: - json.dump(dataset_metadata,file_handler) + with open(file_path, "w+") as file_handler: + json.dump(dataset_metadata, file_handler) def generate_filename(base_dataset_name, target_dataset_name, base_timestamp, target_dataset_timestamp): @@ -98,14 +86,8 @@ def cache_compute_results(base_dataset_type, def fetch_precompute_results(base_dataset_name, target_dataset_name, dataset_type, trait_name): """function to check for precomputed results""" - base_timestamp = target_dataset_timestamp = r.get(f"{dataset_type}timestamp") - if base_timestamp is None: - return - - else: - base_timestamp = target_dataset_timestamp = base_timestamp.decode( - "utf-8") - + base_timestamp = target_dataset_timestamp = query_table_timestamp( + dataset_type) file_name = generate_filename( base_dataset_name, target_dataset_name, base_timestamp, target_dataset_timestamp) -- cgit v1.2.3 From 9577152dfed7ae3fab2fb677a89b78ed88b9e127 Mon Sep 17 00:00:00 2001 From: jgart Date: Wed, 17 Nov 2021 02:37:17 -0500 Subject: update jupyter notebook link name This does not point to the launcher app but to the curated notebooks. --- wqflask/wqflask/templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index 6e922f24..a3bd1b63 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -87,7 +87,7 @@
  • Systems Genetics PheWAS
  • Genome Browser
  • BXD Power Calculator
    • -
  • Jupyter Notebook Launcher
    • +
  • Jupyter Notebooks
  • Interplanetary File System
    • -- cgit v1.2.3 From ea2ce4c05f9b367510ba60a9adc5da9e8337f032 Mon Sep 17 00:00:00 2001 From: jgart Date: Wed, 17 Nov 2021 02:47:52 -0500 Subject: update live jupyter notebooks --- wqflask/wqflask/jupyter_notebooks.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/wqflask/wqflask/jupyter_notebooks.py b/wqflask/wqflask/jupyter_notebooks.py index dbea04dd..7d76828e 100644 --- a/wqflask/wqflask/jupyter_notebooks.py +++ b/wqflask/wqflask/jupyter_notebooks.py @@ -6,16 +6,12 @@ jupyter_notebooks = Blueprint('jupyter_notebooks', __name__) def launcher(): links = ( { - "main_url": "http://notebook.genenetwork.org/51091/tree?", - "notebook_name": "COVID-19 in mybinder.org federation", - "src_link_url": "https://github.com/jgarte/covid19_in_binder"}, + "main_url": "http://notebook.genenetwork.org/34301/notebooks/genenetwork-api-using-r.ipynb", + "notebook_name": "R notebook showing how to query the GeneNetwork API.", + "src_link_url": "https://github.com/jgarte/genenetwork-api-r-jupyter-notebook"}, { - "main_url": "http://notebook.genenetwork.org/35639/tree?", - "notebook_name": "Simple requirements.txt based example", - "src_link_url": "https://github.com/jgarte/requirements"}, - { - "main_url": "http://notebook.genenetwork.org/40733/tree?", - "notebook_name": "Guile Jupyter Notebook Querying GeneNetwork API", - "src_link_url": "https://github.com/jgarte/guile-notebook-genenetwork-api"}) + "main_url": "http://notebook.genenetwork.org/57675/notebooks/genenetwork.ipynb", + "notebook_name": "Querying the GeneNetwork API declaratively with python.", + "src_link_url": "https://github.com/jgarte/genenetwork-jupyter-notebook-example"}) return render_template("jupyter_notebooks.html", links=links) -- cgit v1.2.3 From 9c17856e0aa047408d91b0fe0bd6591d305cca42 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 17 Nov 2021 04:07:08 -0600 Subject: Moven GN API doc to gn-docs repo --- doc/API_readme.md | 168 +----------------------------------------------------- 1 file changed, 1 insertion(+), 167 deletions(-) diff --git a/doc/API_readme.md b/doc/API_readme.md index be6668dc..17d10e44 100644 --- a/doc/API_readme.md +++ b/doc/API_readme.md @@ -1,169 +1,3 @@ # API Query Documentation # ---- -# Fetching Dataset/Trait info/data # ---- -## Fetch Species List ## -To get a list of species with data available in GN (and their associated names and ids): -``` -curl http://genenetwork.org/api/v_pre1/species -[ { "FullName": "Mus musculus", "Id": 1, "Name": "mouse", "TaxonomyId": 10090 }, ... { "FullName": "Populus trichocarpa", "Id": 10, "Name": "poplar", "TaxonomyId": 3689 } ] -``` - -Or to get a single species info: -``` -curl http://genenetwork.org/api/v_pre1/species/mouse -``` -OR -``` -curl http://genenetwork.org/api/v_pre1/species/mouse.json -``` - -*For all queries where the last field is a user-specified name/ID, there will be the option to append a file format type. Currently there is only JSON (and it will default to JSON if none is provided), but other formats will be added later* - -## Fetch Groups/RISets ## - -This query can optionally filter by species: - -``` -curl http://genenetwork.org/api/v_pre1/groups (for all species) -``` -OR -``` -curl http://genenetwork.org/api/v_pre1/groups/mouse (for just mouse groups/RISets) -[ { "DisplayName": "BXD", "FullName": "BXD RI Family", "GeneticType": "riset", "Id": 1, "MappingMethodId": "1", "Name": "BXD", "SpeciesId": 1, "public": 2 }, ... { "DisplayName": "AIL LGSM F34 and F39-43 (GBS)", "FullName": "AIL LGSM F34 and F39-43 (GBS)", "GeneticType": "intercross", "Id": 72, "MappingMethodId": "2", "Name": "AIL-LGSM-F34-F39-43-GBS", "SpeciesId": 1, "public": 2 } ] -``` - -## Fetch Genotypes for Group/RISet ## -``` -curl http://genenetwork.org/api/v_pre1/genotypes/bimbam/BXD -curl http://genenetwork.org/api/v_pre1/genotypes/BXD.bimbam -``` -Returns a group's genotypes in one of several formats - bimbam, rqtl2, or geno (a format used by qtlreaper which is just a CSV file consisting of marker positions and genotypes) - -Rqtl2 genotype queries can also include the dataset name and will return a zip of the genotypes, phenotypes, and gene map (marker names/positions). For example: -``` -curl http://genenetwork.org/api/v_pre1/genotypes/rqtl2/BXD/HC_M2_0606_P.zip -``` - -## Fetch Datasets ## -``` -curl http://genenetwork.org/api/v_pre1/datasets/bxd -``` -OR -``` -curl http://genenetwork.org/api/v_pre1/datasets/mouse/bxd -[ { "AvgID": 1, "CreateTime": "Fri, 01 Aug 2003 00:00:00 GMT", "DataScale": "log2", "FullName": "UTHSC/ETHZ/EPFL BXD Liver Polar Metabolites Extraction A, CD Cohorts (Mar 2017) log2", "Id": 1, "Long_Abbreviation": "BXDMicroArray_ProbeSet_August03", "ProbeFreezeId": 3, "ShortName": "Brain U74Av2 08/03 MAS5", "Short_Abbreviation": "Br_U_0803_M", "confidentiality": 0, "public": 0 }, ... { "AvgID": 3, "CreateTime": "Tue, 14 Aug 2018 00:00:00 GMT", "DataScale": "log2", "FullName": "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Aug18) RMA", "Id": 859, "Long_Abbreviation": "EPFLMouseLiverCDRMAApr18", "ProbeFreezeId": 181, "ShortName": "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Aug18) RMA", "Short_Abbreviation": "EPFLMouseLiverCDRMA0818", "confidentiality": 0, "public": 1 } ] -``` -(I added the option to specify species just in case we end up with the same group name across multiple species at some point, though it's currently unnecessary) - -## Fetch Individual Dataset Info ## -### For mRNA Assay/"ProbeSet" ### - -``` -curl http://genenetwork.org/api/v_pre1/dataset/HC_M2_0606_P -``` -OR -``` -curl http://genenetwork.org/api/v_pre1/dataset/bxd/HC_M2_0606_P``` -{ "confidential": 0, "data_scale": "log2", "dataset_type": "mRNA expression", "full_name": "Hippocampus Consortium M430v2 (Jun06) PDNN", "id": 112, "name": "HC_M2_0606_P", "public": 2, "short_name": "Hippocampus M430v2 BXD 06/06 PDNN", "tissue": "Hippocampus mRNA", "tissue_id": 9 } -``` -(This also has the option to specify group/riset) - -### For "Phenotypes" (basically non-mRNA Expression; stuff like weight, sex, etc) ### -For these traits, the query fetches publication info and takes the group and phenotype 'ID' as input. For example: -``` -curl http://genenetwork.org/api/v_pre1/dataset/bxd/10001 -{ "dataset_type": "phenotype", "description": "Central nervous system, morphology: Cerebellum weight, whole, bilateral in adults of both sexes [mg]", "id": 10001, "name": "CBLWT2", "pubmed_id": 11438585, "title": "Genetic control of the mouse cerebellum: identification of quantitative trait loci modulating size and architecture", "year": "2001" } -``` - -## Fetch Sample Data for Dataset ## -``` -curl http://genenetwork.org/api/v_pre1/sample_data/HSNIH-PalmerPublish.csv -``` - -Returns a CSV file with sample/strain names as the columns and trait IDs as rows - -## Fetch Sample Data for Single Trait ## -``` -curl http://genenetwork.org/api/v_pre1/sample_data/HC_M2_0606_P/1436869_at -[ { "data_id": 23415463, "sample_name": "129S1/SvImJ", "sample_name_2": "129S1/SvImJ", "se": 0.123, "value": 8.201 }, { "data_id": 23415463, "sample_name": "A/J", "sample_name_2": "A/J", "se": 0.046, "value": 8.413 }, { "data_id": 23415463, "sample_name": "AKR/J", "sample_name_2": "AKR/J", "se": 0.134, "value": 8.856 }, ... ] -``` - -## Fetch Trait List for Dataset ## -``` -curl http://genenetwork.org/api/v_pre1/traits/HXBBXHPublish.json -[ { "Additive": 0.0499967532467532, "Id": 10001, "LRS": 16.2831307029479, "Locus": "rs106114574", "PhenotypeId": 1449, "PublicationId": 319, "Sequence": 1 }, ... ] -``` - -Both JSON and CSV formats can be specified, with JSON as default. There is also an optional "ids_only" and "names_only" parameter that will only return a list of trait IDs or names, respectively. - -## Fetch Trait Info (Name, Description, Location, etc) ## -### For mRNA Expression/"ProbeSet" ### -``` -curl http://genenetwork.org/api/v_pre1/trait/HC_M2_0606_P/1436869_at -{ "additive": -0.214087568058076, "alias": "HHG1; HLP3; HPE3; SMMCI; Dsh; Hhg1", "chr": "5", "description": "sonic hedgehog (hedgehog)", "id": 99602, "locus": "rs8253327", "lrs": 12.7711275309832, "mb": 28.457155, "mean": 9.27909090909091, "name": "1436869_at", "p_value": 0.306, "se": null, "symbol": "Shh" } -``` - -### For "Phenotypes" ### -For phenotypes this just gets the max LRS, its location, and additive effect (as calculated by qtlreaper) - -Since each group/riset only has one phenotype "dataset", this query takes either the group/riset name or the group/riset name + "Publish" (for example "BXDPublish", which is the dataset name in the DB) as input -``` -curl http://genenetwork.org/api/v_pre1/trait/BXD/10001 -{ "additive": 2.39444435069444, "id": 4, "locus": "rs48756159", "lrs": 13.4974911471087 } -``` - ---- - -# Analyses # ---- -## Mapping ## -Currently two mapping tools can be used - GEMMA and R/qtl. qtlreaper will be added later with Christian Fischer's RUST implementation - https://github.com/chfi/rust-qtlreaper - -Each method's query takes the following parameters respectively (more will be added): -### GEMMA ### -* trait_id (*required*) - ID for trait being mapped -* db (*required*) - DB name for trait above (Short_Abbreviation listed when you query for datasets) -* use_loco - Whether to use LOCO (leave one chromosome out) method (default = false) -* maf - minor allele frequency (default = 0.01) - -Example query: -``` -curl http://genenetwork.org/api/v_pre1/mapping?trait_id=10015&db=BXDPublish&method=gemma&use_loco=true -``` - -### R/qtl ### -(See the R/qtl guide for information on some of these options - http://www.rqtl.org/manual/qtl-manual.pdf) -* trait_id (*required*) - ID for trait being mapped -* db (*required*) - DB name for trait above (Short_Abbreviation listed when you query for datasets) -* rqtl_method - hk (default) | ehk | em | imp | mr | mr-imp | mr-argmax ; Corresponds to the "method" option for the R/qtl scanone function. -* rqtl_model - normal (default) | binary | 2-part | np ; corresponds to the "model" option for the R/qtl scanone function -* num_perm - number of permutations; 0 by default -* control_marker - Name of marker to use as control; this relies on the user knowing the name of the marker they want to use as a covariate -* interval_mapping - Whether to use interval mapping; "false" by default -* pair_scan - *NYI* - -Example query: -``` -curl http://genenetwork.org/api/v_pre1/mapping?trait_id=1418701_at&db=HC_M2_0606_P&method=rqtl&num_perm=100 -``` - -Some combinations of methods/models may not make sense. The R/qtl manual should be referred to for any questions on its use (specifically the scanone function in this case) - -## Calculate Correlation ## -Currently only Sample and Tissue correlations are implemented - -This query currently takes the following parameters (though more will be added): -* trait_id (*required*) - ID for trait used for correlation -* db (*required*) - DB name for the trait above (this is the Short_Abbreviation listed when you query for datasets) -* target_db (*required*) - Target DB name to be correlated against -* type - sample (default) | tissue -* method - pearson (default) | spearman -* return - Number of results to return (default = 500) - -Example query: -``` -curl http://genenetwork.org/api/v_pre1/correlation?trait_id=1427571_at&db=HC_M2_0606_P&target_db=BXDPublish&type=sample&return_count=100 -[ { "#_strains": 6, "p_value": 0.004804664723032055, "sample_r": -0.942857142857143, "trait": 20511 }, { "#_strains": 6, "p_value": 0.004804664723032055, "sample_r": -0.942857142857143, "trait": 20724 }, { "#_strains": 12, "p_value": 1.8288943424888848e-05, "sample_r": -0.9233615170820528, "trait": 13536 }, { "#_strains": 7, "p_value": 0.006807187408935392, "sample_r": 0.8928571428571429, "trait": 10157 }, { "#_strains": 7, "p_value": 0.006807187408935392, "sample_r": -0.8928571428571429, "trait": 20392 }, ... ] -``` +This document has moved to [gn-docs](https://github.com/genenetwork/gn-docs/blob/master/api/GN2-REST-API.md)! -- cgit v1.2.3 From 83719648e505747f16281d7e8e14f1003297be5c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 17 Nov 2021 13:42:26 +0300 Subject: README.org: Replace broken link with environments page --- doc/README.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/README.org b/doc/README.org index 8839aefc..e1c6b614 100644 --- a/doc/README.org +++ b/doc/README.org @@ -26,7 +26,7 @@ * Introduction -Large system deployments can get very [[http://biogems.info/contrib/genenetwork/gn2.svg ][complex]]. In this document we +Large system deployments can get very [[http://genenetwork.org/environments/][complex]]. In this document we explain the GeneNetwork version 2 (GN2) reproducible deployment system which is based on GNU Guix (see also [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix system can be used to install GN with all its files and dependencies. -- cgit v1.2.3 From 6786712e95cbb885b6b19b3ecd34e6c8ee893172 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 17 Nov 2021 20:20:07 +0300 Subject: refactor sql query & date formatting --- wqflask/base/data_set.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 52d1d254..2687738d 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -1258,31 +1258,27 @@ def geno_mrna_confidentiality(ob): return True - def parse_db_url(): parsed_db = urlparse(SQL_URI) return (parsed_db.hostname, parsed_db.username, parsed_db.password, parsed_db.path[1:]) + def query_table_timestamp(dataset_type: str): """function to query the update timestamp of a given dataset_type""" # computation data and actions fetch_db_name = parse_db_url() - query_update_time = f""" SELECT UPDATE_TIME FROM information_schema.tables - WHERE TABLE_SCHEMA = {fetch_db_name[-1]} + WHERE TABLE_SCHEMA = '{fetch_db_name[-1]}' AND TABLE_NAME = '{dataset_type}Data' """ - # store the timestamp in redis= date_time_obj = g.db.execute(query_update_time).fetchone()[0] - - f = "%Y-%m-%d %H:%M:%S" - return date_time_obj.strftime(f) + return date_time_obj.strftime("%Y-%m-%d %H:%M:%S") def generate_hash_file(dataset_name: str, dataset_type: str, dataset_timestamp: str): @@ -1301,6 +1297,7 @@ def cache_dataset_results(dataset_name: str, dataset_type: str, query_results: L table_timestamp = query_table_timestamp(dataset_type) + file_name = generate_hash_file(dataset_name, dataset_type, table_timestamp) file_path = os.path.join(TMPDIR, f"{file_name}.json") -- cgit v1.2.3 From 1a3b85c4ebc66d54e3bda06c3742e8046e4c8159 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 17 Nov 2021 20:38:20 +0300 Subject: add generic functio for generating filename --- wqflask/wqflask/correlation/pre_computes.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index 403d60c9..241b0730 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -13,8 +13,9 @@ def fetch_all_cached_metadata(dataset_name): file_path = os.path.join(TMPDIR, file_name) - with open(file_path, "r+") as file_handler: - dataset_metadata = json.load(file_handler) + try: + with open(file_path, "r+") as file_handler: + dataset_metadata = json.load(file_handler) except FileNotFoundError: Path(file_path).touch(exist_ok=True) @@ -32,6 +33,13 @@ def cache_new_traits_metadata(dataset_metadata: dict, new_traits_metadata, file_ json.dump(dataset_metadata, file_handler) +def generate_file_name(*args, prefix=""): + """given a list of args generate a unique filename""" + + string_unicode = f"{*args,}{prefix}".encode() + return hashlib.md5(string_unicode).hexdigest() + + def generate_filename(base_dataset_name, target_dataset_name, base_timestamp, target_dataset_timestamp): """generate unique filename""" -- cgit v1.2.3 From 747544906c4d331a8d46ed692070e0f9d335191b Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 17 Nov 2021 18:31:59 +0000 Subject: Fixes gene global search error caused by some genes not having a symbol (not sure why this is in the DB; seems to have only happened recently) --- wqflask/wqflask/gsearch.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index 2516e4fb..bf44408e 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -82,12 +82,13 @@ class GSearch: this_trait['species'] = line[0] this_trait['group'] = line[1] this_trait['tissue'] = line[2] - this_trait['symbol'] = line[6] + this_trait['symbol'] = "N/A" + if line[6]: + this_trait['symbol'] = line[6] + this_trait['description'] = "N/A" if line[7]: this_trait['description'] = line[7].decode( 'utf-8', 'replace') - else: - this_trait['description'] = "N/A" this_trait['location_repr'] = 'N/A' if (line[8] != "NULL" and line[8] != "") and (line[9] != 0): this_trait['location_repr'] = 'Chr%s: %.6f' % ( -- cgit v1.2.3 From 3325440faef1161da90d51444543d8d847fb9cdd Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 17 Nov 2021 18:27:56 +0000 Subject: Fix issue where gene global search throws an error due to some traits not having a gene symbol (not sure why they don't in the DB, but the code defaults it to N/A now) --- wqflask/wqflask/gsearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index bf44408e..31f3305c 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -89,7 +89,7 @@ class GSearch: if line[7]: this_trait['description'] = line[7].decode( 'utf-8', 'replace') - this_trait['location_repr'] = 'N/A' + this_trait['location_repr'] = "N/A" if (line[8] != "NULL" and line[8] != "") and (line[9] != 0): this_trait['location_repr'] = 'Chr%s: %.6f' % ( line[8], float(line[9])) -- cgit v1.2.3 From dea1afa01025026a0754de60615b974a5cb3fd24 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 17 Nov 2021 21:53:20 +0000 Subject: Changed the URL for editing trait metadata to use the dataset group ID instead of the PublishFreezeId --- wqflask/wqflask/templates/show_trait_details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 6b125221..4e9ea0fb 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -236,7 +236,7 @@ {% if admin_status != None and admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} {% if this_trait.dataset.type == 'Publish' %} - + {% endif %} {% if this_trait.dataset.type == 'ProbeSet' %} -- cgit v1.2.3 From 56b574b903244a64aecaa54e5305b25bb642b254 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 18 Nov 2021 12:02:10 +0300 Subject: pep8 formatting;minor fixes --- wqflask/wqflask/correlation/pre_computes.py | 37 ++++++++++++++++++----------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index 241b0730..77592a3a 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -9,7 +9,7 @@ from base.webqtlConfig import TMPDIR def fetch_all_cached_metadata(dataset_name): """in a gvein dataset fetch all the traits metadata""" - file_name = f"{dataset_name}_metadata.json" + file_name = generate_file_name(dataset_name, suffix="metadata") file_path = os.path.join(TMPDIR, file_name) @@ -33,11 +33,11 @@ def cache_new_traits_metadata(dataset_metadata: dict, new_traits_metadata, file_ json.dump(dataset_metadata, file_handler) -def generate_file_name(*args, prefix=""): +def generate_file_name(*args, suffix="", file_ext="json"): """given a list of args generate a unique filename""" - string_unicode = f"{*args,}{prefix}".encode() - return hashlib.md5(string_unicode).hexdigest() + string_unicode = f"{*args,}".encode() + return f"{hashlib.md5(string_unicode).hexdigest()}_{suffix}.{file_ext}" def generate_filename(base_dataset_name, target_dataset_name, base_timestamp, target_dataset_timestamp): @@ -65,11 +65,12 @@ def cache_compute_results(base_dataset_type, target_dataset_timestamp = base_timestamp - file_name = generate_filename( + file_name = generate_file_name( base_dataset_name, target_dataset_name, - base_timestamp, target_dataset_timestamp) + base_timestamp, target_dataset_timestamp, + suffix="corr_precomputes") - file_path = os.path.join(TMPDIR, f"{file_name}.json") + file_path = os.path.join(TMPDIR, file_name) try: with open(file_path, "r+") as json_file_handler: @@ -91,16 +92,20 @@ def cache_compute_results(base_dataset_type, json.dump(data, file_handler) -def fetch_precompute_results(base_dataset_name, target_dataset_name, dataset_type, trait_name): +def fetch_precompute_results(base_dataset_name, + target_dataset_name, + dataset_type, + trait_name): """function to check for precomputed results""" base_timestamp = target_dataset_timestamp = query_table_timestamp( dataset_type) - file_name = generate_filename( + file_name = generate_file_name( base_dataset_name, target_dataset_name, - base_timestamp, target_dataset_timestamp) + base_timestamp, target_dataset_timestamp, + suffix="corr_precomputes") - file_path = os.path.join(TMPDIR, f"{file_name}.json") + file_path = os.path.join(TMPDIR, file_name) try: with open(file_path, "r+") as json_handler: @@ -112,7 +117,9 @@ def fetch_precompute_results(base_dataset_name, target_dataset_name, dataset_typ pass -def pre_compute_dataset_vs_dataset(base_dataset, target_dataset, corr_method): +def pre_compute_dataset_vs_dataset(base_dataset, + target_dataset, + corr_method): """compute sample correlation between dataset vs dataset wn:heavy function should be invoked less frequently input:datasets_data(two dicts),corr_method @@ -131,8 +138,10 @@ def pre_compute_dataset_vs_dataset(base_dataset, target_dataset, corr_method): "trait_id": primary_trait_name } - trait_correlation_result = fast_compute_all_sample_correlation( - corr_method=corr_method, this_trait=this_trait_data, target_dataset=target_traits_data) + trait_correlation_result = compute_all_sample_correlation( + corr_method=corr_method, + this_trait=this_trait_data, + target_dataset=target_traits_data) dataset_correlation_results[primary_trait_name] = trait_correlation_result -- cgit v1.2.3 From 4de623130dca019d15f956e91ec999fddc2e2a0f Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 18 Nov 2021 12:03:48 +0300 Subject: remove unused functions rename function names --- wqflask/wqflask/correlation/pre_computes.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index 77592a3a..b95ceba5 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -9,7 +9,7 @@ from base.webqtlConfig import TMPDIR def fetch_all_cached_metadata(dataset_name): """in a gvein dataset fetch all the traits metadata""" - file_name = generate_file_name(dataset_name, suffix="metadata") + file_name = generate_filename(dataset_name, suffix="metadata") file_path = os.path.join(TMPDIR, file_name) @@ -33,20 +33,13 @@ def cache_new_traits_metadata(dataset_metadata: dict, new_traits_metadata, file_ json.dump(dataset_metadata, file_handler) -def generate_file_name(*args, suffix="", file_ext="json"): +def generate_filename(*args, suffix="", file_ext="json"): """given a list of args generate a unique filename""" string_unicode = f"{*args,}".encode() return f"{hashlib.md5(string_unicode).hexdigest()}_{suffix}.{file_ext}" -def generate_filename(base_dataset_name, target_dataset_name, base_timestamp, target_dataset_timestamp): - """generate unique filename""" - - string_unicode = f"{base_dataset_name}{target_dataset_name}{base_timestamp}{target_dataset_timestamp}sample_corr_compute".encode() - return hashlib.md5(string_unicode).hexdigest() - - def cache_compute_results(base_dataset_type, base_dataset_name, target_dataset_name, @@ -65,7 +58,7 @@ def cache_compute_results(base_dataset_type, target_dataset_timestamp = base_timestamp - file_name = generate_file_name( + file_name = generate_filename( base_dataset_name, target_dataset_name, base_timestamp, target_dataset_timestamp, suffix="corr_precomputes") @@ -100,7 +93,7 @@ def fetch_precompute_results(base_dataset_name, base_timestamp = target_dataset_timestamp = query_table_timestamp( dataset_type) - file_name = generate_file_name( + file_name = generate_filename( base_dataset_name, target_dataset_name, base_timestamp, target_dataset_timestamp, suffix="corr_precomputes") -- cgit v1.2.3 From 24d87cb4e75136f822b316a3c9f936b8e5efb5e9 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 18 Nov 2021 13:40:54 +0300 Subject: refactor code for metadata --- wqflask/wqflask/correlation/pre_computes.py | 11 ++------- wqflask/wqflask/correlation/show_corr_results.py | 29 +++++++++--------------- 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index b95ceba5..ad0bc6ef 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -16,12 +16,11 @@ def fetch_all_cached_metadata(dataset_name): try: with open(file_path, "r+") as file_handler: dataset_metadata = json.load(file_handler) + return (file_path, dataset_metadata) except FileNotFoundError: Path(file_path).touch(exist_ok=True) - return {} - - return (file_path, dataset_metadata) + return (file_path, {}) def cache_new_traits_metadata(dataset_metadata: dict, new_traits_metadata, file_path: str): @@ -46,16 +45,10 @@ def cache_compute_results(base_dataset_type, corr_method, correlation_results, trait_name): - # pass """function to cache correlation results for heavy computations""" - # init assumption only caching probeset type - # fix redis;issue potential redis_cache!=current_timestamp - base_timestamp = query_table_timestamp(base_dataset_type) - r.set(f"{base_dataset_type}timestamp", base_timestamp) - target_dataset_timestamp = base_timestamp file_name = generate_filename( diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 42010a1e..f5600f13 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -26,6 +26,9 @@ from base.trait import create_trait, jsonable from base.data_set import create_dataset from base.webqtlConfig import TMPDIR +from wqflask.correlation.pre_computes import fetch_all_cached_metadata +from wqflask.correlation.pre_computes import cache_new_traits_metadata + from utility import hmac @@ -34,7 +37,8 @@ def set_template_vars(start_vars, correlation_data): corr_method = start_vars['corr_sample_method'] if start_vars['dataset'] == "Temp": - this_dataset_ob = create_dataset(dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) + this_dataset_ob = create_dataset( + dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) else: this_dataset_ob = create_dataset(dataset_name=start_vars['dataset']) this_trait = create_trait(dataset=this_dataset_ob, @@ -86,25 +90,17 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe corr_results = correlation_data['correlation_results'] results_list = [] - file_name = f"{target_dataset['name']}_metadata.json" - - file_path = os.path.join(TMPDIR, file_name) new_traits_metadata = {} - try: - with open(file_path,"r+") as file_handler: - dataset_metadata = json.load(file_handler) - - except FileNotFoundError: - Path(file_path).touch(exist_ok=True) - dataset_metadata = {} + (file_path, dataset_metadata) = fetch_all_cached_metadata( + target_dataset['name']) for i, trait_dict in enumerate(corr_results): trait_name = list(trait_dict.keys())[0] trait = trait_dict[trait_name] target_trait = dataset_metadata.get(trait_name) - if target_trait is None: + if target_trait is None: target_trait_ob = create_trait(dataset=target_dataset_ob, name=trait_name, get_qtl_info=True) @@ -184,12 +180,9 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe results_list.append(results_dict) - - if bool(new_traits_metadata): - # that means new traits exists - dataset_metadata.update(new_traits_metadata) - with open(file_path,"w+") as file_handler: - json.dump(dataset_metadata, file_handler) + cache_new_traits_metadata(dataset_metadata, + new_traits_metadata, + file_path) return json.dumps(results_list) -- cgit v1.2.3 From 42eee3a5f5d4055210e971db02e94e33b66d0b71 Mon Sep 17 00:00:00 2001 From: Arthur Centeno Date: Thu, 18 Nov 2021 17:53:11 +0000 Subject: AC Updating base, index_page and tutorials on 11-18-21 --- wqflask/wqflask/templates/base.html | 9 +++++- wqflask/wqflask/templates/index_page.html | 52 ++++++++++++++++++++++++++++--- wqflask/wqflask/templates/tutorials.html | 27 +++++++--------- 3 files changed, 67 insertions(+), 21 deletions(-) diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index 6e922f24..a231c48c 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -22,6 +22,12 @@ + + + + + + {% block css %} @@ -87,7 +93,7 @@
  • Systems Genetics PheWAS
  • Genome Browser
  • BXD Power Calculator
    • -
  • Jupyter Notebook Launcher
    • +
  • Jupyter Notebook Launcher
  • Interplanetary File System
    • @@ -227,6 +233,7 @@ Registered with Nif +
    diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html index 7b103305..cfb1bed4 100755 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -193,10 +193,40 @@
    +
    -
    -
    +
    +
    +

    Tutorials

    + +
    +
    + Webinars & Courses +

    In-person courses, live webinars and webinar recordings +

    + +

    +
    + +
    + Tutorials +

    Tutorials: Training materials in HTML, PDF and video formats +

    +
    + +
    + Documentation +

    Online manuals, handbooks, fact sheets and FAQs +

    +
    + +
    +
    + +
    @@ -241,8 +271,22 @@
  • Time Machine: Full GN versions from 2009 to 2016 (mm9)
    • Cloud (EC2) - +
    +
    +
    +

    Affiliates

    + +
    + +
    diff --git a/wqflask/wqflask/templates/tutorials.html b/wqflask/wqflask/templates/tutorials.html index 8f3447e3..edec803c 100644 --- a/wqflask/wqflask/templates/tutorials.html +++ b/wqflask/wqflask/templates/tutorials.html @@ -56,9 +56,7 @@ -

    Webinar #01 - Introduction to Quantitative Trait Loci (QTL) Analysis

    -

    Friday, May 8th, 2020
    - 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT

    +

    Introduction to Quantitative Trait Loci (QTL) Analysis

    Goals of this webinar (trait variance to QTL):

    • Define quantitative trait locus (QTL)
      • @@ -76,10 +74,7 @@ University of Tennessee Health Science Center -

      Webinar #02 - Mapping Addiction and Behavioral Traits and Getting at Causal Gene Variants with GeneNetwork

      -

      Friday, May 22nd. 2020 - 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT -

      +

      Mapping Addiction and Behavioral Traits and Getting at Causal Gene Variants with GeneNetwork

      Goals of this webinar (QTL to gene variant):

      -- cgit v1.2.3 From fcfd7be522ce914b0aa11cd4555aeab2d2141428 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Fri, 19 Nov 2021 11:10:33 +0300 Subject: Feature/fix wgcna api path (#630) * add correct path gn3 api endpoint * remove ununsed dependencies;replace libraries with ones from guix * replace xterm cdn libs--- wqflask/wqflask/templates/wgcna_setup.html | 37 ++++++++---------------------- wqflask/wqflask/wgcna/gn3_wgcna.py | 10 +++----- 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/wqflask/wqflask/templates/wgcna_setup.html b/wqflask/wqflask/templates/wgcna_setup.html index 86d9fa10..d7acd5f2 100644 --- a/wqflask/wqflask/templates/wgcna_setup.html +++ b/wqflask/wqflask/templates/wgcna_setup.html @@ -9,7 +9,8 @@ } - + +
      @@ -80,19 +81,12 @@
      - - - - - - - + + - + {% endblock %} \ No newline at end of file diff --git a/wqflask/wqflask/wgcna/gn3_wgcna.py b/wqflask/wqflask/wgcna/gn3_wgcna.py index c4cc2e7f..15728f22 100644 --- a/wqflask/wqflask/wgcna/gn3_wgcna.py +++ b/wqflask/wqflask/wgcna/gn3_wgcna.py @@ -4,7 +4,9 @@ and process data to be rendered by datatables import requests from types import SimpleNamespace + from utility.helper_functions import get_trait_db_obs +from utility.tools import GN_SERVER_URL def fetch_trait_data(requestform): @@ -24,7 +26,6 @@ def process_dataset(trait_list): traits = [] strains = [] - # xtodo unique traits and strains for trait in trait_list: traits.append(trait[0].name) @@ -33,9 +34,6 @@ def process_dataset(trait_list): for strain in trait[0].data: strains.append(strain) input_data[trait[0].name][strain] = trait[0].data[strain].value - # "sample_names": list(set(strains)), - # "trait_names": form_traits, - # "trait_sample_data": form_strains, return { "input": input_data, @@ -77,9 +75,7 @@ def process_image(response): def run_wgcna(form_data): """function to run wgcna""" - GN3_URL = "http://127.0.0.1:8081" - - wgcna_api = f"{GN3_URL}/api/wgcna/run_wgcna" + wgcna_api = f"{GN_SERVER_URL}api/wgcna/run_wgcna" # parse form data -- cgit v1.2.3 From b7a4fa3007e2a3364e7a827b0bf4b3a54fcc272d Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 23 Nov 2021 13:38:36 +0300 Subject: merge commit :added some logic that takes into account corr_sample_group when determining which samples to use when getting sample_data --- wqflask/wqflask/correlation/correlation_gn3_api.py | 31 +++++++++++++++++----- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 635ef5ed..32a55b44 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -64,20 +64,27 @@ def test_process_data(this_trait, dataset, start_vars): return sample_data -def process_samples(start_vars, sample_names, excluded_samples=None): - """process samples""" +def process_samples(start_vars,sample_names = [],excluded_samples = []): + """code to fetch correct samples""" sample_data = {} - if not excluded_samples: - excluded_samples = () - sample_vals_dict = json.loads(start_vars["sample_vals"]) + sample_vals_dict = json.loads(start_vars["sample_vals"]) + if sample_names: for sample in sample_names: - if sample not in excluded_samples and sample in sample_vals_dict: + if sample in sample_vals_dict and sample not in excluded_samples: + val = sample_vals_dict[sample] + if not val.strip().lower() == "x": + sample_data[str(sample)] = float(val) + + else: + for sample in sample_vals_dict.keys(): + if sample not in excluded_samples: val = sample_vals_dict[sample] if not val.strip().lower() == "x": sample_data[str(sample)] = float(val) return sample_data + def merge_correlation_results(correlation_results, target_correlation_results): corr_dict = {} @@ -153,6 +160,18 @@ def lit_for_trait_list(corr_results, this_dataset, this_trait): def fetch_sample_data(start_vars, this_trait, this_dataset, target_dataset): + corr_samples_group = start_vars["corr_samples_group"] + if corr_samples_group == "samples_primary": + sample_data = process_samples( + start_vars, this_dataset.group.all_samples_ordered()) + + elif corr_samples_group == "samples_other": + sample_data = process_samples( + start_vars, excluded_samples = this_dataset.group.samplelist) + + else: + sample_data = process_samples(start_vars) + sample_data = process_samples( start_vars, this_dataset.group.all_samples_ordered()) -- cgit v1.2.3 From aa9a06d927bdc2b5221e58559f24921a0ff72cd8 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 23 Nov 2021 13:50:21 +0300 Subject: pep8 formatting remove dead variables --- wqflask/base/data_set.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 2687738d..4d75e7ee 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -758,7 +758,6 @@ class DataSet: chunk_size = 50 number_chunks = int(math.ceil(len(sample_ids) / chunk_size)) cached_results = fetch_cached_results(self.name, self.type) - # cached_results = None if cached_results is None: trait_sample_data = [] for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): -- cgit v1.2.3 From fffeb91789943a3c7db5a72d66405e2a0459ed44 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 23 Nov 2021 14:49:07 +0300 Subject: fix for overwriting file --- wqflask/wqflask/correlation/pre_computes.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index ad0bc6ef..975a53b8 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -28,8 +28,9 @@ def cache_new_traits_metadata(dataset_metadata: dict, new_traits_metadata, file_ if bool(new_traits_metadata): dataset_metadata.update(new_traits_metadata) - with open(file_path, "w+") as file_handler: - json.dump(dataset_metadata, file_handler) + + with open(file_path, "w+") as file_handler: + json.dump(dataset_metadata, file_handler) def generate_filename(*args, suffix="", file_ext="json"): -- cgit v1.2.3 From c398af6b98f89917065457153b0f7e55be28e835 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 23 Nov 2021 16:12:27 +0300 Subject: wqflask: views: Make error messages properly formatted --- wqflask/wqflask/views.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 2cbaffd6..55d9ebbe 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -131,9 +131,8 @@ def handle_generic_exceptions(e): time_str = now.strftime('%l:%M%p UTC %b %d, %Y') # get the stack trace and send it to the logger exc_type, exc_value, exc_traceback = sys.exc_info() - formatted_lines = {f"{request.url} ({time_str}) " - f" {traceback.format_exc().splitlines()}"} - + formatted_lines = (f"{request.url} ({time_str}) \n" + f"{traceback.format_exc()}") _message_templates = { werkzeug.exceptions.NotFound: ("404: Not Found: " f"{time_str}: {request.url}"), @@ -142,8 +141,8 @@ def handle_generic_exceptions(e): werkzeug.exceptions.RequestTimeout: ("408: Request Timeout: " f"{time_str}: {request.url}")} # Default to the lengthy stack trace! - logger.error(_message_templates.get(exc_type, - formatted_lines)) + app.logger.error(_message_templates.get(exc_type, + formatted_lines)) # Handle random animations # Use a cookie to have one animation on refresh animation = request.cookies.get(err_msg[:32]) @@ -152,7 +151,7 @@ def handle_generic_exceptions(e): "./wqflask/static/gif/error") if fn.endswith(".gif")]) resp = make_response(render_template("error.html", message=err_msg, - stack=formatted_lines, + stack={formatted_lines}, error_image=animation, version=GN_VERSION)) -- cgit v1.2.3 From 66960bdbd7cd313ebe5a7136568b7389dc7d7fed Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 23 Nov 2021 17:20:18 +0300 Subject: wqflask: metadata_edits: Remove "\n\n" check with something else In excel, "\n\n" is replaced with ",,,," and this leads to a value error. --- wqflask/wqflask/metadata_edits.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 572db080..21b73cb4 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -1,6 +1,7 @@ import datetime import json import os +import re from collections import namedtuple from itertools import groupby @@ -250,17 +251,21 @@ def update_phenotype(dataset_id: str, name: str): file_.save(new_file_name) publishdata_id = "" lines = [] + split_point = "" with open(new_file_name, "r") as f: lines = f.read() - first_line = lines.split('\n', 1)[0] - publishdata_id = first_line.split("Id:")[-1].strip() + for line in lines.split("\n"): + if "# Publish Data Id:" in line: + split_point = line + publishdata_id = re.findall(r'\b\d+\b', line)[0] + break with open(new_file_name, "w") as f: - f.write(lines.split("\n\n")[-1]) + f.write(lines.split(f"{split_point}\n")[-1].strip()) csv_ = get_trait_csv_sample_data(conn=conn, trait_name=str(name), phenotype_id=str(phenotype_id)) with open(uploaded_file_name, "w") as f_: - f_.write(csv_.split("\n\n")[-1]) + f_.write(csv_.split(str(publishdata_id))[-1].strip()) r = run_cmd(cmd=("csvdiff " f"'{uploaded_file_name}' '{new_file_name}' " "--format json")) -- cgit v1.2.3 From 41e742904ff4cf35abbd885eeb98902a05d3be80 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 23 Nov 2021 17:35:25 +0300 Subject: wqflask: metadata_edits: Sort files from most recent --- wqflask/wqflask/metadata_edits.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 21b73cb4..f87914d5 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -428,9 +428,15 @@ def list_diffs(): gn_proxy_url=current_app.config.get("GN2_PROXY")) return render_template( "display_files.html", - approved=files.get("approved"), - rejected=files.get("rejected"), - waiting=files.get("waiting")) + approved=sorted(files.get("approved"), + reverse=True, + key=lambda d: d.get("time_stamp")), + rejected=sorted(files.get("rejected"), + reverse=True, + key=lambda d: d.get("time_stamp")), + waiting=sorted(files.get("waiting"), + reverse=True, + key=lambda d: d.get("time_stamp"))) @metadata_edit.route("/diffs/") -- cgit v1.2.3 From 8a7894182a6b4d6a47f6a677c304d2a2256ca154 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 24 Nov 2021 13:00:49 +0300 Subject: Delete noisy logging --- wqflask/base/data_set.py | 2 -- wqflask/utility/tools.py | 3 --- wqflask/wqflask/views.py | 5 +---- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 49ece9dd..af248659 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -429,8 +429,6 @@ class DatasetGroup: if result is not None: self.samplelist = json.loads(result) else: - logger.debug("Cache not hit") - genotype_fn = locate_ignore_error(self.name + ".geno", 'genotype') if genotype_fn: self.samplelist = get_group_samplelists.get_samplelist( diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index f28961ec..db0b4320 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -194,7 +194,6 @@ def locate(name, subdir=None): if valid_path(base): lookfor = base + "/" + name if valid_file(lookfor): - logger.info("Found: file " + lookfor + "\n") return lookfor else: raise Exception("Can not locate " + lookfor) @@ -220,9 +219,7 @@ def locate_ignore_error(name, subdir=None): if valid_path(base): lookfor = base + "/" + name if valid_file(lookfor): - logger.debug("Found: file " + name + "\n") return lookfor - logger.info("WARNING: file " + name + " not found\n") return None diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 55d9ebbe..23b4e07a 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -398,12 +398,9 @@ def create_temp_trait(): @app.route('/export_trait_excel', methods=('POST',)) def export_trait_excel(): """Excel file consisting of the sample data from the trait data and analysis page""" - logger.info("In export_trait_excel") - logger.info("request.form:", request.form) - logger.info(request.url) trait_name, sample_data = export_trait_data.export_sample_table( request.form) - + app.logger.info(request.url) logger.info("sample_data - type: %s -- size: %s" % (type(sample_data), len(sample_data))) -- cgit v1.2.3 From 4d0fe8810b695ae883f02cad9ff4fc5716d8d3f0 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 25 Nov 2021 12:33:40 +0300 Subject: Remove useless comment --- wqflask/wqflask/metadata_edits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index f87914d5..76dd50d9 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -447,7 +447,7 @@ def show_diff(name): content = myfile.read() return Response(content, mimetype='text/json') -# http://localhost:5004/datasets/diffs + @metadata_edit.route("/diffs//reject") @edit_admins_access_required @login_required -- cgit v1.2.3 From eaf63573e1a78bf9e5c1051875f4e2c10d1dcb77 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 25 Nov 2021 21:02:44 +0300 Subject: wqflask: metadata_edits: Support deletions and insertions from csv --- wqflask/wqflask/metadata_edits.py | 95 ++++++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 40 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 76dd50d9..fa6cfea3 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -45,6 +45,8 @@ from gn3.db.phenotypes import PublishXRef from gn3.db.phenotypes import probeset_mapping from gn3.db.traits import get_trait_csv_sample_data from gn3.db.traits import update_sample_data +from gn3.db.traits import delete_sample_data +from gn3.db.traits import insert_sample_data metadata_edit = Blueprint('metadata_edit', __name__) @@ -249,23 +251,11 @@ def update_phenotype(dataset_id: str, name: str): TMPDIR, "sample-data/updated/", f"{_file_name}.csv.uploaded")) file_.save(new_file_name) - publishdata_id = "" - lines = [] - split_point = "" - with open(new_file_name, "r") as f: - lines = f.read() - for line in lines.split("\n"): - if "# Publish Data Id:" in line: - split_point = line - publishdata_id = re.findall(r'\b\d+\b', line)[0] - break - with open(new_file_name, "w") as f: - f.write(lines.split(f"{split_point}\n")[-1].strip()) - csv_ = get_trait_csv_sample_data(conn=conn, - trait_name=str(name), - phenotype_id=str(phenotype_id)) with open(uploaded_file_name, "w") as f_: - f_.write(csv_.split(str(publishdata_id))[-1].strip()) + f_.write(get_trait_csv_sample_data( + conn=conn, + trait_name=str(name), + phenotype_id=str(phenotype_id))) r = run_cmd(cmd=("csvdiff " f"'{uploaded_file_name}' '{new_file_name}' " "--format json")) @@ -274,9 +264,9 @@ def update_phenotype(dataset_id: str, name: str): with open(diff_output, "w") as f: dict_ = json.loads(r.get("output")) dict_.update({ + "trait_name": str(name), + "phenotype_id": str(phenotype_id), "author": author, - "publishdata_id": publishdata_id, - "dataset_id": name, "timestamp": datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S") }) @@ -324,11 +314,16 @@ def update_phenotype(dataset_id: str, name: str): k: data_.get(f"old_{k}") for k, v in publication_.items() if v is not None}, new=publication_)}) if diff_data: - diff_data.update({"dataset_id": name}) - diff_data.update({"resource_id": request.args.get('resource-id')}) - diff_data.update({"author": author}) - diff_data.update({"timestamp": datetime.datetime.now().strftime( - "%Y-%m-%d %H:%M:%S")}) + diff_data.update({ + "phenotype_id": str(phenotype_id), + "dataset_id": name, + "resource_id": request.args.get('resource-id'), + "author": author, + "timestamp": (datetime + .datetime + .now() + .strftime("%Y-%m-%d %H:%M:%S")), + }) insert(conn, table="metadata_audit", data=MetadataAudit(dataset_id=name, @@ -473,34 +468,54 @@ def approve_data(resource_id:str, file_name: str): with open(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), 'r') as myfile: sample_data = json.load(myfile) - modifications = [d for d in sample_data.get("Modifications")] - for modification in modifications: + for modification in ( + modifications := [d for d in sample_data.get("Modifications")]): if modification.get("Current"): - (strain_id, - strain_name, + (strain_name, value, se, count) = modification.get("Current").split(",") update_sample_data( conn=conn, + trait_name=sample_data.get("trait_name"), strain_name=strain_name, - strain_id=int(strain_id), - publish_data_id=int(sample_data.get("publishdata_id")), + phenotype_id=int(sample_data.get("phenotype_id")), value=value, error=se, - count=count - ) - insert(conn, - table="metadata_audit", - data=MetadataAudit( - dataset_id=sample_data.get("dataset_id"), - editor=sample_data.get("author"), - json_data=json.dumps(sample_data))) - if modifications: + count=count) + for deletion in (deletions := [d for d in sample_data.get("Deletions")]): + strain_name, _, _, _ = deletion.split(",") + delete_sample_data( + conn=conn, + trait_name=sample_data.get("trait_name"), + strain_name=strain_name, + phenotype_id=int(sample_data.get("phenotype_id"))) + for insertion in ( + insertions := [d for d in sample_data.get("Additions")]): + (strain_name, + value, se, count) = insertion.split(",") + insert_sample_data( + conn=conn, + trait_name=sample_data.get("trait_name"), + strain_name=strain_name, + phenotype_id=int(sample_data.get("phenotype_id")), + value=value, + error=se, + count=count) + + if any([any(modifications), any(deletions), any(insertions)]): + insert(conn, + table="metadata_audit", + data=MetadataAudit( + dataset_id=sample_data.get("trait_name"), + editor=sample_data.get("author"), + json_data=json.dumps(sample_data))) # Once data is approved, rename it! os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), os.path.join(f"{TMPDIR}/sample-data/diffs", f"{file_name}.approved")) - flash((f"Just updated data from: {file_name}; {len(modifications)} " - "row(s) modified!"), + flash((f"Just updated data from: {file_name};\n" + f"# Modifications: {len(modifications)}; " + f"# Additions: {len(insertions)}; " + f"# Deletions: {len(deletions)}"), "success") return redirect(url_for('metadata_edit.list_diffs')) -- cgit v1.2.3 From 98282951acf8bf5ad8a344a8b0f55606ed9e3931 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sat, 27 Nov 2021 09:14:18 +0300 Subject: display_files.html: Place "approved" before "rejected" section --- wqflask/wqflask/templates/display_files.html | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/wqflask/wqflask/templates/display_files.html b/wqflask/wqflask/templates/display_files.html index d5295fb4..2fd7d7ee 100644 --- a/wqflask/wqflask/templates/display_files.html +++ b/wqflask/wqflask/templates/display_files.html @@ -58,9 +58,9 @@
    {% endif %} - {% if rejected %} + {% if approved %}
    - Rejected Files: + Approved Data:
    @@ -69,7 +69,7 @@ - {% for data in rejected %} + {% for data in approved %} {% set file_url = url_for('metadata_edit.show_diff', name=data.get('file_name')) %} @@ -80,12 +80,12 @@
    TimeStamp
    {{ data.get("resource_id") }}
    -
    +
    {% endif %} - {% if approved %} + {% if rejected %}
    - Approved Data: + Rejected Files:
    @@ -94,7 +94,7 @@ - {% for data in approved %} + {% for data in rejected %} {% set file_url = url_for('metadata_edit.show_diff', name=data.get('file_name')) %} @@ -105,8 +105,9 @@
    TimeStamp
    {{ data.get("resource_id") }}
    -
    +
    {% endif %} +
    {%endblock%} -- cgit v1.2.3 From d9843928fe5140c953078b5238c2ddb8fb12a96c Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 1 Dec 2021 14:30:00 +0000 Subject: Introduce GN3_PYTHONPATH environment variable --- README.md | 6 ++++++ bin/genenetwork2 | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6921d299..62ead0bd 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,12 @@ startup script [./bin/genenetwork2](https://github.com/genenetwork/genenetwork2/ Also mariadb and redis need to be running, see [INSTALL](./doc/README.org). +## Development + +It may be useful to pull in the GN3 python modules locally. For this +use `GN3_PYTHONPATH` environment that gets injected in +the ./bin/genenetwork2 startup. + ## Testing To have tests pass, the redis and mariadb instance should be running, because of diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 5f714d2e..b5c940a8 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -147,7 +147,7 @@ if [ ! -d $R_LIBS_SITE ] ; then fi # We may change this one: -export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$PYTHONPATH +export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$GN3_PYTHONPATH:$PYTHONPATH # Our UNIX TMPDIR defaults to /tmp - change this on a shared server if [ -z $TMPDIR ]; then -- cgit v1.2.3 From 4bd10c0c4c2eddeb0e75de4da8fcbec222347fdc Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 1 Dec 2021 15:55:50 +0000 Subject: Figure out that we are using the wrong db_webqtl_s! --- README.md | 12 ++++++++++++ wqflask/wqflask/api/gen_menu.py | 12 ++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 62ead0bd..972d5c50 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,18 @@ alias runcmd="time env GN2_PROFILE=~/opt/gn-latest TMPDIR=//tmp SERVER_PORT=5004 Replace some of the env variables as per your use case. +### Troubleshooting + +If the menu does not pop up check your `GN2_BASE_URL`. E.g. + +``` +curl http://gn2-pjotr.genenetwork.org/api/v_pre1/gen_dropdown +``` + +check the logs. If there is ERROR 1054 (42S22): Unknown column +'InbredSet.Family' in 'field list' it may be you are trying the small +database. + ## Documentation User documentation can be found diff --git a/wqflask/wqflask/api/gen_menu.py b/wqflask/wqflask/api/gen_menu.py index a699a484..5d239343 100644 --- a/wqflask/wqflask/api/gen_menu.py +++ b/wqflask/wqflask/api/gen_menu.py @@ -1,4 +1,8 @@ from gn3.db.species import get_all_species + +import utility.logger +logger = utility.logger.getLogger(__name__) + def gen_dropdown_json(conn): """Generates and outputs (as json file) the data for the main dropdown menus on the home page @@ -19,16 +23,16 @@ def get_groups(species, conn): with conn.cursor() as cursor: for species_name, _species_full_name in species: groups[species_name] = [] - cursor.execute( - ("SELECT InbredSet.Name, InbredSet.FullName, " + query = ("SELECT InbredSet.Name, InbredSet.FullName, " "IFNULL(InbredSet.Family, 'None') " "FROM InbredSet, Species WHERE Species.Name = '{}' " "AND InbredSet.SpeciesId = Species.Id GROUP by " "InbredSet.Name ORDER BY IFNULL(InbredSet.FamilyOrder, " "InbredSet.FullName) ASC, IFNULL(InbredSet.Family, " "InbredSet.FullName) ASC, InbredSet.FullName ASC, " - "InbredSet.MenuOrderId ASC") - .format(species_name)) + "InbredSet.MenuOrderId ASC").format(species_name) + # logger.debug(query) + cursor.execute(query) results = cursor.fetchall() for result in results: family_name = "Family:" + str(result[2]) -- cgit v1.2.3 From 878afa280d5c2da36485fc73142961817d3c2d2f Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 2 Dec 2021 17:39:09 +0000 Subject: Increased mid-width for container element for mapping figure to prevent it from shrinking when the browser window is smaller (which prevents the links, etc, from overlapping properly with the image) --- wqflask/wqflask/templates/mapping_results.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index f2d11e89..c1ea1daf 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -12,7 +12,7 @@ {% endblock %} {% from "base_macro.html" import header %} {% block content %} -
    +
    {% if temp_trait is defined %} -- cgit v1.2.3 From 3d3fdf75628953e510528ed6fbd2c85c52d90350 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 2 Dec 2021 20:17:44 +0000 Subject: Increased min-width for showtrait-main-div to prevent mapping description text from overflowing --- wqflask/wqflask/static/new/css/show_trait.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/static/new/css/show_trait.css b/wqflask/wqflask/static/new/css/show_trait.css index f5e8c22a..3780a8f1 100644 --- a/wqflask/wqflask/static/new/css/show_trait.css +++ b/wqflask/wqflask/static/new/css/show_trait.css @@ -67,7 +67,7 @@ table.dataTable.cell-border tbody tr td:first-child { } .showtrait-main-div { - min-width: 1100px; + min-width: 1400px; } table.dataTable tbody td.column_name-Checkbox { -- cgit v1.2.3 From 337d02d00692c84053810067b4dcb033becbf80f Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 2 Dec 2021 20:49:05 +0000 Subject: Changed index pagee HTML to be more consistent, adding indentation, removed commented out HTML --- wqflask/wqflask/templates/index_page.html | 140 +++++++++++++----------------- 1 file changed, 60 insertions(+), 80 deletions(-) diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html index 3e2f424d..fce1c619 100755 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -40,12 +40,12 @@
    -
    -
    -
    -

    Tutorials

    - -
    -
    - Webinars & Courses -

    In-person courses, live webinars and webinar recordings -

    - -

    -
    - -
    - Tutorials -

    Tutorials: Training materials in HTML, PDF and video formats -

    -
    - -
    - Documentation -

    Online manuals, handbooks, fact sheets and FAQs -

    -
    - -
    -
    - - -
    -
    +
    -

    News

    -
    -
    - more news items... +

    Tutorials

    +
    +
    +
    + Webinars & Courses +

    In-person courses, live webinars and webinar recordings +

    +

    +
    +
    + Tutorials +

    Tutorials: Training materials in HTML, PDF and video formats +

    +
    +
    + Documentation +

    Online manuals, handbooks, fact sheets and FAQs +

    -
    -
    +
    +
    -

    Github

    - +

    News

    -
    +
    + +
    -

    Links

    +

    Links

    -

    GeneNetwork v2:

    - -

    GeneNetwork v1:

    + +
      +
    • GeneNetwork v1:
      • +
        +
      • Main website at UTHSC
        • +
      • Time Machine: Full GN versions from 2009 to 2016 (mm9)
        • Cloud (EC2) +
    -
    -
    -

    Affiliates

    - +
    +

    Affiliates

    - +
    -- cgit v1.2.3 From 69570e44f96a2385a36340d5c1985d5aab957061 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 2 Dec 2021 20:53:51 +0000 Subject: Made Affiliates a subcategory of Links --- wqflask/wqflask/templates/index_page.html | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html index fce1c619..6a697570 100755 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -253,20 +253,18 @@ Cloud (EC2) - -
    -
    -

    Affiliates

    -
    -
    +
    -- cgit v1.2.3 From 810cd25a4d205313eefdd18dbdf101bd84048705 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 2 Dec 2021 21:28:13 +0000 Subject: Removed colons from Links headers and removed margin-bottom for H2 elements --- wqflask/wqflask/templates/index_page.html | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html index 6a697570..507fcfc8 100755 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -4,14 +4,9 @@ {% endblock %} @@ -240,13 +238,13 @@
      -
    • GeneNetwork v1:
      • +
    • GeneNetwork v1
      • Main website at UTHSC
      • Time Machine: Full GN versions from 2009 to 2016 (mm9)
        • @@ -254,7 +252,7 @@
      -
    • Affiliates:
      • +
    • Affiliates
      • GeneNetwork 1 at UTHSC
      • Systems Genetics at EPFL
        • -- cgit v1.2.3 From 71b11e896c7edc1902def87f5913f05048c4d7f6 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 2 Dec 2021 21:29:09 +0000 Subject: Moved the UIkit CSS from base.html to index_page.html (btw Arthur, never add CSS files to base.html - it makes their styling apply to every single GN page) --- wqflask/wqflask/templates/base.html | 6 ------ wqflask/wqflask/templates/index_page.html | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index c17409d0..ad2e3744 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -22,12 +22,6 @@ - - - - - - {% block css %} diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html index 507fcfc8..6e9bf91a 100755 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -1,6 +1,12 @@ {% extends "base.html" %} {% block title %}GeneNetwork{% endblock %} {% block css %} + + + + + + {% endblock %} {% block content %} -- cgit v1.2.3 From 5d3119d48462f6da4f06c92f6132c516db7fdfd6 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 3 Dec 2021 19:06:49 +0000 Subject: Prevent points prior to the X axis from being displayed in mapping figure when zoomed into a sub-range on a chromosome --- wqflask/wqflask/marker_regression/display_mapping_results.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 6254b9b9..ead509d7 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -2580,6 +2580,8 @@ class DisplayMappingResults: if self.selectedChr != -1 and qtlresult['Mb'] > endMb: Xc = startPosX + endMb * plotXScale else: + if qtlresult['Mb'] - startMb < 0: + continue Xc = startPosX + (qtlresult['Mb'] - startMb) * plotXScale # updated by NL 06-18-2011: -- cgit v1.2.3 From d2532523023fd26730d377150016a4f13200d403 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 3 Dec 2021 20:12:57 +0000 Subject: Make sure mapping_scale is always physic when using GEMMA --- wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js | 1 + 1 file changed, 1 insertion(+) diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js index e42fe8c4..d0b41d04 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js +++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js @@ -177,6 +177,7 @@ $(".gemma-tab, #gemma_compute").on("click", (function(_this) { var form_data, url; url = "/loading"; $('input[name=method]').val("gemma"); + $('input[name=mapping_scale]').val('physic'); $('input[name=selected_chr]').val($('#chr_gemma').val()); $('input[name=num_perm]').val(0); $('input[name=genofile]').val($('#genofile_gemma').val()); -- cgit v1.2.3 From 4a757f3dbecd960bc8e84940b1eb46646fd3a4fb Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 3 Dec 2021 20:13:41 +0000 Subject: Properly set start position for cM mapping, to prevent overflow + remove some unnecessary commented out lines --- .../marker_regression/display_mapping_results.py | 24 ++++------------------ 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index ead509d7..920a8d30 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -2471,12 +2471,6 @@ class DisplayMappingResults: thisLRSColor = self.colorCollection[0] if qtlresult['chr'] != previous_chr and self.selectedChr == -1: if self.manhattan_plot != True: - # im_drawer.polygon( - # xy=LRSCoordXY, - # outline=thisLRSColor - # # , closed=0, edgeWidth=lrsEdgeWidth, - # # clipX=(xLeftOffset, xLeftOffset + plotWidth) - # ) draw_open_polygon(canvas, xy=LRSCoordXY, outline=thisLRSColor, width=lrsEdgeWidth) @@ -2497,25 +2491,21 @@ class DisplayMappingResults: im_drawer.line( xy=((Xc0, Yc0), (Xcm, yZero)), fill=plusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) im_drawer.line( xy=((Xcm, yZero), (Xc, yZero - (Yc - yZero))), fill=minusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) else: im_drawer.line( xy=((Xc0, yZero - (Yc0 - yZero)), (Xcm, yZero)), fill=minusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) im_drawer.line( xy=((Xcm, yZero), (Xc, Yc)), fill=plusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) elif (Yc0 - yZero) * (Yc - yZero) > 0: if Yc < yZero: @@ -2523,14 +2513,12 @@ class DisplayMappingResults: xy=((Xc0, Yc0), (Xc, Yc)), fill=plusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) else: im_drawer.line( xy=((Xc0, yZero - (Yc0 - yZero)), (Xc, yZero - (Yc - yZero))), fill=minusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) else: minYc = min(Yc - yZero, Yc0 - yZero) @@ -2538,14 +2526,12 @@ class DisplayMappingResults: im_drawer.line( xy=((Xc0, Yc0), (Xc, Yc)), fill=plusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) else: im_drawer.line( xy=((Xc0, yZero - (Yc0 - yZero)), (Xc, yZero - (Yc - yZero))), fill=minusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) LRSCoordXY = [] @@ -2558,22 +2544,21 @@ class DisplayMappingResults: startPosX += newStartPosX oldStartPosX = newStartPosX - # ZS: This is because the chromosome value stored in qtlresult['chr'] can be (for example) either X or 20 depending upon the mapping method/scale used + # This is because the chromosome value stored in qtlresult['chr'] can be (for example) either X or 20 depending upon the mapping method/scale used this_chr = str(self.ChrList[self.selectedChr][0]) if self.plotScale != "physic": this_chr = str(self.ChrList[self.selectedChr][1] + 1) if self.selectedChr == -1 or str(qtlresult['chr']) == this_chr: if self.plotScale != "physic" and self.mapping_method == "reaper" and not self.manhattan_plot: - Xc = startPosX + (qtlresult['cM'] - startMb) * plotXScale + start_cm = self.genotype[self.selectedChr - 1][0].cM + Xc = startPosX + (qtlresult['cM'] - start_cm) * plotXScale if hasattr(self.genotype, "filler"): if self.genotype.filler: if self.selectedChr != -1: - start_cm = self.genotype[self.selectedChr - 1][0].cM Xc = startPosX + \ (qtlresult['Mb'] - start_cm) * plotXScale else: - start_cm = self.genotype[previous_chr_as_int][0].cM Xc = startPosX + ((qtlresult['Mb'] - start_cm - startMb) * plotXScale) * ( ((qtlresult['Mb'] - start_cm - startMb) * plotXScale) / ((qtlresult['Mb'] - start_cm - startMb + self.GraphInterval) * plotXScale)) else: @@ -2648,9 +2633,8 @@ class DisplayMappingResults: AdditiveHeightThresh / additiveMax AdditiveCoordXY.append((Xc, Yc)) - if self.selectedChr != -1 and qtlresult['Mb'] > endMb: + if self.selectedChr != -1 and qtlresult['Mb'] > endMb and endMb != -1: break - m += 1 if self.manhattan_plot != True: -- cgit v1.2.3 From 9d87b302c03e4a9a8569b438bd0734abe3646e4a Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 3 Dec 2021 20:14:00 +0000 Subject: Add mapping_scale to mapping_results.html forum inputs --- wqflask/wqflask/templates/mapping_results.html | 1 + 1 file changed, 1 insertion(+) diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index c1ea1daf..f62d554d 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -39,6 +39,7 @@ + {% if manhattan_plot == True %} -- cgit v1.2.3 From 34f57439476959ca1fb8d149647ecb2b1720908c Mon Sep 17 00:00:00 2001 From: Arthur Centeno Date: Tue, 7 Dec 2021 18:11:32 +0000 Subject: Made some changes to the index page HTML --- wqflask/wqflask/templates/index_page.html | 51 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html index df975a0b..24464c3b 100755 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -213,36 +213,35 @@
    -
    -
    -

    Tutorials

    -
    -
    -
    - Webinars & Courses -

    In-person courses, live webinars and webinar recordings -

    -

    -
    -
    - Tutorials -

    Tutorials: Training materials in HTML, PDF and video formats -

    -
    -
    - Documentation -

    Online manuals, handbooks, fact sheets and FAQs -

    -
    -
    +
    +
    +

    Tutorials

    +
    +
    + Webinars & Courses
    + In-person courses, live webinars and webinar recordings
    + +
    +
    + Tutorials
    + Tutorials: Training materials in HTML, PDF and video formats
    + +
    +
    + Documentation
    + Online manuals, handbooks, fact sheets and FAQs
    + +
    +
    +

    News

    -
    -
    -
    -- cgit v1.2.3 From 99d9c213f1c825af5b5bf6007521894b29131ba8 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 6 Dec 2021 14:07:57 +0300 Subject: add new endpoint:for test gn3 wgcna --- wqflask/wqflask/views.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 23b4e07a..f2c539d1 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -339,9 +339,11 @@ def wcgna_setup(): return render_template("wgcna_setup.html", **request.form) -@app.route("/wgcna_results", methods=('POST',)) -def wcgna_results(): - """call the gn3 api to get wgcna response data""" + + +@app.route("/test/wgcna_results", methods=('POST',)) +def test_wcgna_results(): + """test call the gn3 api to get wgcna response data""" results = run_wgcna(dict(request.form)) return render_template("test_wgcna_results.html", **results) -- cgit v1.2.3 From 9cea1a8d757461ea2359f4c72afa96c7180e417f Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 6 Dec 2021 16:13:46 +0300 Subject: parse powers to use from form data --- wqflask/wqflask/wgcna/gn3_wgcna.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/wgcna/gn3_wgcna.py b/wqflask/wqflask/wgcna/gn3_wgcna.py index 15728f22..6692a890 100644 --- a/wqflask/wqflask/wgcna/gn3_wgcna.py +++ b/wqflask/wqflask/wgcna/gn3_wgcna.py @@ -6,7 +6,7 @@ import requests from types import SimpleNamespace from utility.helper_functions import get_trait_db_obs -from utility.tools import GN_SERVER_URL +from utility.tools import GN3_LOCAL_URL def fetch_trait_data(requestform): @@ -75,13 +75,21 @@ def process_image(response): def run_wgcna(form_data): """function to run wgcna""" - wgcna_api = f"{GN_SERVER_URL}api/wgcna/run_wgcna" + wgcna_api = f"{GN3_LOCAL_URL}/api/wgcna/run_wgcna" # parse form data trait_dataset = fetch_trait_data(form_data) form_data["minModuleSize"] = int(form_data["MinModuleSize"]) + + + + form_data["SoftThresholds"] = [int(threshold.strip()) + for threshold in form_data['SoftThresholds'].rstrip().split(",")] + + + response = requests.post(wgcna_api, json={ "sample_names": list(set(trait_dataset["sample_names"])), "trait_names": trait_dataset["trait_names"], @@ -91,6 +99,7 @@ def run_wgcna(form_data): } ).json() + return { "results": response, "data": process_wgcna_data(response["data"]), -- cgit v1.2.3 From c5e4153a9a7bb3ce07f406773bbbf95dc729b32b Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 6 Dec 2021 17:15:31 +0300 Subject: handle connection error & other exceptions --- wqflask/wqflask/wgcna/gn3_wgcna.py | 43 +++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/wqflask/wqflask/wgcna/gn3_wgcna.py b/wqflask/wqflask/wgcna/gn3_wgcna.py index 6692a890..17599711 100644 --- a/wqflask/wqflask/wgcna/gn3_wgcna.py +++ b/wqflask/wqflask/wgcna/gn3_wgcna.py @@ -26,7 +26,6 @@ def process_dataset(trait_list): traits = [] strains = [] - for trait in trait_list: traits.append(trait[0].name) @@ -82,26 +81,36 @@ def run_wgcna(form_data): trait_dataset = fetch_trait_data(form_data) form_data["minModuleSize"] = int(form_data["MinModuleSize"]) + form_data["SoftThresholds"] = [int(threshold.strip()) + for threshold in form_data['SoftThresholds'].rstrip().split(",")] + try: + response = requests.post(wgcna_api, json={ + "sample_names": list(set(trait_dataset["sample_names"])), + "trait_names": trait_dataset["trait_names"], + "trait_sample_data": list(trait_dataset["input"].values()), + **form_data - form_data["SoftThresholds"] = [int(threshold.strip()) - for threshold in form_data['SoftThresholds'].rstrip().split(",")] - - + } + ) - response = requests.post(wgcna_api, json={ - "sample_names": list(set(trait_dataset["sample_names"])), - "trait_names": trait_dataset["trait_names"], - "trait_sample_data": list(trait_dataset["input"].values()), - **form_data + status_code = response.status_code + response = response.json() - } - ).json() + if status_code != 200: + return { + "error": response + } + return { + "error": 'null', + "results": response, + "data": process_wgcna_data(response["data"]), + "image": process_image(response["data"]) + } - return { - "results": response, - "data": process_wgcna_data(response["data"]), - "image": process_image(response["data"]) - } + except requests.exceptions.ConnectionError: + return { + "error": "A connection error to perform computation occurred" + } -- cgit v1.2.3 From 744d7290d8b7eb32546692efe870bfc4e85d5e4a Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 6 Dec 2021 17:18:02 +0300 Subject: modify template to show error generated --- wqflask/wqflask/templates/test_wgcna_results.html | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/wqflask/wqflask/templates/test_wgcna_results.html b/wqflask/wqflask/templates/test_wgcna_results.html index 1dddd393..952f479e 100644 --- a/wqflask/wqflask/templates/test_wgcna_results.html +++ b/wqflask/wqflask/templates/test_wgcna_results.html @@ -70,6 +70,10 @@ }
    + {% if error!='null' %} +

    {{error}}

    + + {% else %}

    Soft Thresholds

    @@ -110,6 +114,9 @@
    + +{% endif %} +
    {% endblock %} -- cgit v1.2.3 From 3329584a1cfaa884b35bc35e6579d53c7edca84e Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 6 Dec 2021 17:19:02 +0300 Subject: modify endpoint for test wgcna --- wqflask/wqflask/views.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index f2c539d1..23b4e07a 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -339,11 +339,9 @@ def wcgna_setup(): return render_template("wgcna_setup.html", **request.form) - - -@app.route("/test/wgcna_results", methods=('POST',)) -def test_wcgna_results(): - """test call the gn3 api to get wgcna response data""" +@app.route("/wgcna_results", methods=('POST',)) +def wcgna_results(): + """call the gn3 api to get wgcna response data""" results = run_wgcna(dict(request.form)) return render_template("test_wgcna_results.html", **results) -- cgit v1.2.3 From 374ee9ff1e975dd14cea3a4a35a5950880db7225 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Fri, 10 Dec 2021 15:44:06 +0300 Subject: pep8 formatting --- wqflask/wqflask/wgcna/gn3_wgcna.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/wqflask/wqflask/wgcna/gn3_wgcna.py b/wqflask/wqflask/wgcna/gn3_wgcna.py index 17599711..7bf5c62b 100644 --- a/wqflask/wqflask/wgcna/gn3_wgcna.py +++ b/wqflask/wqflask/wgcna/gn3_wgcna.py @@ -98,12 +98,7 @@ def run_wgcna(form_data): status_code = response.status_code response = response.json() - if status_code != 200: - return { - "error": response - } - - return { + return {"error": response} if status_code != 200 else { "error": 'null', "results": response, "data": process_wgcna_data(response["data"]), -- cgit v1.2.3 From 522214bd2edf83e68a94ed5fd0d891548cacf34f Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 13 Dec 2021 21:18:57 +0000 Subject: Fixes issue that caused LRS searches that specify position to not work properly; the queries had to be changed after integrating the change that also pulls the Max LRS info in the original query --- wqflask/wqflask/do_search.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 6b8dfa41..99272ee3 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -515,10 +515,7 @@ class LrsSearch(DoSearch): self.search_term = converted_search_term - if len(self.search_term) > 2: - from_clause = ", Geno" - else: - from_clause = "" + from_clause = "" return from_clause -- cgit v1.2.3 From 92e555531e53630ce11e34a0b67bb111647b47c1 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 14 Dec 2021 18:20:48 +0000 Subject: Includes dataset name when creating SimpleNamespace dataset_ob, since it's needed during authentication --- wqflask/wqflask/gsearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index 31f3305c..53a124d0 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -119,7 +119,7 @@ class GSearch: this_trait['dataset_id'] = line[15] dataset_ob = SimpleNamespace( - id=this_trait["dataset_id"], type="ProbeSet", species=this_trait["species"]) + id=this_trait["dataset_id"], type="ProbeSet", name=this_trait["dataset"], species=this_trait["species"]) if dataset_ob.id not in dataset_to_permissions: permissions = check_resource_availability(dataset_ob) dataset_to_permissions[dataset_ob.id] = permissions -- cgit v1.2.3 From 7c6618d20aaf97bec7abcc52ca144997dde59b64 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 14 Dec 2021 22:21:49 +0000 Subject: Removed min-width from container div, since it seems to have preventd horizontal scrolling when the window is small --- wqflask/wqflask/templates/mapping_results.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index f62d554d..84db288c 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -12,7 +12,7 @@ {% endblock %} {% from "base_macro.html" import header %} {% block content %} -
    +
    {% if temp_trait is defined %} -- cgit v1.2.3 From 9b315a4037d2b7a6bfb35658d3cf514ac5e9ce6d Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 15 Dec 2021 17:36:02 +0000 Subject: Edited html for index page --- wqflask/wqflask/templates/index_page.html | 50 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html index 24464c3b..3a490658 100755 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -213,35 +213,35 @@
    -
    -
    -

    Tutorials

    -
    -
    - Webinars & Courses
    - In-person courses, live webinars and webinar recordings
    - -
    -
    - Tutorials
    - Tutorials: Training materials in HTML, PDF and video formats
    - -
    -
    - Documentation
    - Online manuals, handbooks, fact sheets and FAQs
    - -
    -
    -
    +
    +
    +

    Tutorials

    +
    +
    +
    + Webinars & Courses
    + In-person courses, live webinars and webinar recordings
    + +
    +
    + Tutorials
    + Tutorials: Training materials in HTML, PDF and video formats
    + +
    +
    + Documentation
    + Online manuals, handbooks, fact sheets and FAQs
    + +
    +

    News

    -
    - +
    +
    +
    -- cgit v1.2.3 From 69676ecd49a779350ef7bd3faa348c7f27602a20 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 17 Dec 2021 20:10:33 +0000 Subject: Some updates to the DOL genotypes conversion script --- scripts/convert_dol_genotypes.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/convert_dol_genotypes.py b/scripts/convert_dol_genotypes.py index 81b3bd6d..5cda2e9c 100644 --- a/scripts/convert_dol_genotypes.py +++ b/scripts/convert_dol_genotypes.py @@ -1,6 +1,8 @@ # This is just to convert the Rqtl2 format genotype files for DOL into a .geno file # Everything is hard-coded since I doubt this will be re-used and I just wanted to generate the file quickly +# This is to be used on the files generated as described by Karl Broman here - https://kbroman.org/qtl2/pages/prep_do_data.html + import os geno_dir = "/home/zas1024/gn2-zach/DO_genotypes/" @@ -30,9 +32,13 @@ for filename in os.listdir(geno_dir): if i < 3: continue elif not len(sample_names) and i == 3: - sample_names = [item.replace("TLB", "TB") for item in line_items[1:]] + sample_names_positions = [[item.replace("TLB", "TB").strip(), i] for i, item in enumerate(line_items[1:])] + sample_names_positions.sort(key = lambda x: x[0][2:]) + sample_names = [sample[0] for sample in sample_names_positions] elif i > 3: - marker_data[line_items[0]]['genotypes'] = ["X" if item.strip() == "-" else item.strip() for item in line_items[1:]] + genotypes = ["X" if item.strip() == "-" else item.strip() for item in line_items[1:]] + ordered_genotypes = [genotypes[i].strip() for i in [pos[1] for pos in sample_names_positions]] + marker_data[line_items[0]]['genotypes'] = ordered_genotypes # Generate list of marker obs to iterate through when writing to .geno file marker_list = [] @@ -46,6 +52,7 @@ for key, value in marker_data.items(): } marker_list.append(this_marker) + def sort_func(e): """For ensuring that X/Y chromosomes/mitochondria are sorted to the end correctly""" try: @@ -63,7 +70,7 @@ marker_list.sort(key=sort_func) # Write lines to .geno file with open(gn_geno_path, "w") as gn_geno_fh: - gn_geno_fh.write("\t".join((["Chr", "Locus", "cM", "Mb"] + sample_names))) + gn_geno_fh.write("\t".join((["Chr", "Locus", "cM", "Mb"] + sample_names)) + "\n") for marker in marker_list: row_contents = [ marker['chr'], @@ -72,3 +79,4 @@ with open(gn_geno_path, "w") as gn_geno_fh: marker['pos'] ] + marker['genotypes'] gn_geno_fh.write("\t".join(row_contents) + "\n") + -- cgit v1.2.3 From b8261fdf3775f2e6f9496f5b33ea002e30fc9f67 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 20 Dec 2021 20:48:39 +0000 Subject: This should fix ProbeSet trait descriptions in the search results, which previousluy weren't showing the proper description --- wqflask/wqflask/search_results.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 33f9319c..af0ce44f 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -120,7 +120,9 @@ class SearchResultPage: trait_dict['display_name'] = result[2] trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait_dict['display_name'], trait_dict['dataset'])) trait_dict['symbol'] = "N/A" if result[3] is None else result[3].strip() - description_text = "N/A" if result[4] is None or str(result[4]) == "" else trait_dict['symbol'] + description_text = "" + if result[4] is None or str(result[4]) != "": + description_text = result[4].decode('utf-8') target_string = result[5].decode('utf-8') if result[5] else "" description_display = description_text if target_string is None or str(target_string) == "" else description_text + "; " + str(target_string).strip() -- cgit v1.2.3 From 8892fd1826925a5d85ae8a966bb484365282d992 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 23 Dec 2021 18:54:16 +0000 Subject: Set new_traits_metadata so caching is used (previously was always an empty dict) --- wqflask/wqflask/correlation/show_corr_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 2c820658..4d2ac9ff 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -101,11 +101,11 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe target_trait = dataset_metadata.get(trait_name) if target_trait is None: - target_trait_ob = create_trait(dataset=target_dataset_ob, name=trait_name, get_qtl_info=True) target_trait = jsonable(target_trait_ob, target_dataset_ob) + new_traits_metadata[trait_name] = target_trait if target_trait['view'] == False: continue results_dict = {} -- cgit v1.2.3 From 64f421a2522ad4fbb295b2f962957ac0c8973e22 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 23 Dec 2021 19:00:00 +0000 Subject: If phenotype metadata is cached, authenticate for those traits (otherwise authentication happens in create_trait at line 105) --- wqflask/wqflask/correlation/show_corr_results.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 4d2ac9ff..1c391386 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -29,6 +29,7 @@ from base.webqtlConfig import TMPDIR from wqflask.correlation.pre_computes import fetch_all_cached_metadata from wqflask.correlation.pre_computes import cache_new_traits_metadata +from utility.authentication_tools import check_resource_availability from utility import hmac @@ -106,6 +107,12 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe get_qtl_info=True) target_trait = jsonable(target_trait_ob, target_dataset_ob) new_traits_metadata[trait_name] = target_trait + else: + if target_dataset['type'] == "Publish": + permissions = check_resource_availability(target_dataset_ob, trait_name) + if permissions['metadata'] == "no-access": + continue + if target_trait['view'] == False: continue results_dict = {} -- cgit v1.2.3 From cb8fc335ece8e7ceabab63dc73cb4464d0cef3d7 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 3 Jan 2022 19:13:51 +0000 Subject: Fixed wrong logic for checking for null or empty string trait descriptions in search results --- wqflask/wqflask/search_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index af0ce44f..dd360971 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -121,7 +121,7 @@ class SearchResultPage: trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait_dict['display_name'], trait_dict['dataset'])) trait_dict['symbol'] = "N/A" if result[3] is None else result[3].strip() description_text = "" - if result[4] is None or str(result[4]) != "": + if result[4] is not None and str(result[4]) != "": description_text = result[4].decode('utf-8') target_string = result[5].decode('utf-8') if result[5] else "" -- cgit v1.2.3 From c5c5de8aa9a8078ba5d955f2fc22dbc2c2494610 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 3 Jan 2022 20:02:19 +0000 Subject: Added something that should work as a 'universal' solution for decoding trait descriptions in search results, since there were still some traits with encoding issues --- wqflask/wqflask/search_results.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index dd360971..cf2905c9 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -5,6 +5,7 @@ import time import re import requests from types import SimpleNamespace +import unicodedata from pprint import pformat as pf @@ -122,7 +123,7 @@ class SearchResultPage: trait_dict['symbol'] = "N/A" if result[3] is None else result[3].strip() description_text = "" if result[4] is not None and str(result[4]) != "": - description_text = result[4].decode('utf-8') + description_text = unicodedata.normalize("NFKD", result[4].decode('latin1')) target_string = result[5].decode('utf-8') if result[5] else "" description_display = description_text if target_string is None or str(target_string) == "" else description_text + "; " + str(target_string).strip() -- cgit v1.2.3 From ee8b2bdbfcf0250664c96797029408a52be118da Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 22 Dec 2021 11:31:07 +0300 Subject: Use datatables to display approved and rejected files --- wqflask/wqflask/templates/display_files.html | 37 +++++++++++++++++----------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/wqflask/wqflask/templates/display_files.html b/wqflask/wqflask/templates/display_files.html index 2fd7d7ee..c8c446ba 100644 --- a/wqflask/wqflask/templates/display_files.html +++ b/wqflask/wqflask/templates/display_files.html @@ -1,5 +1,10 @@ {% extends "base.html" %} {% block title %}Trait Submission{% endblock %} + +{% block css %} + +{% endblock %} + {% block content %} {% with messages = get_flashed_messages(with_categories=true) %} @@ -12,14 +17,12 @@ {% endif %} {% endwith %} -
    - {% if waiting %}
    Files for approval: -
    - +
    +
    @@ -59,10 +62,10 @@ {% endif %} {% if approved %} +

    Approved Data:

    - Approved Data: -
    -
    Resource Id Author
    +
    +
    @@ -84,10 +87,10 @@ {% endif %} {% if rejected %} +

    Rejected Files:

    - Rejected Files: -
    -
    Resource Id Author
    +
    +
    @@ -105,16 +108,20 @@
    Resource Id Author
    -
    +
    {% endif %} -
    - {%endblock%} {% block js %} - + + {% endblock %} -- cgit v1.2.3 From 4beef23e2f554b0a290a2f913b87c05507a981fe Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 22 Dec 2021 12:41:21 +0300 Subject: Add interface for displaying diffs in a html page --- wqflask/wqflask/metadata_edits.py | 10 ++- wqflask/wqflask/templates/display_diffs.html | 96 ++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 wqflask/wqflask/templates/display_diffs.html diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index fa6cfea3..593605b5 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -440,7 +440,14 @@ def show_diff(name): with open(os.path.join(f"{TMPDIR}/sample-data/diffs", name), 'r') as myfile: content = myfile.read() - return Response(content, mimetype='text/json') + content = json.loads(content) + for data in content.get("Modifications"): + data["Diff"] = "\n".join(difflib.ndiff([data.get("Original")], + [data.get("Current")])) + return render_template( + "display_diffs.html", + diff=content + ) @metadata_edit.route("/diffs//reject") @@ -465,6 +472,7 @@ def approve_data(resource_id:str, file_name: str): passwd=current_app.config.get("DB_PASS"), host=current_app.config.get("DB_HOST")) TMPDIR = current_app.config.get("TMPDIR") + import pudb; pu.db with open(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), 'r') as myfile: sample_data = json.load(myfile) diff --git a/wqflask/wqflask/templates/display_diffs.html b/wqflask/wqflask/templates/display_diffs.html new file mode 100644 index 00000000..edd889b5 --- /dev/null +++ b/wqflask/wqflask/templates/display_diffs.html @@ -0,0 +1,96 @@ +{% extends "base.html" %} +{% block title %}Trait Submission{% endblock %} + +{% block css %} + +{% endblock %} + +{% block content %} + +
    + {% set additions = diff.get("Additions") %} + {% set modifications = diff.get("Modifications") %} + {% set deletions = diff.get("Deletions") %} + + {% if additions %} +

    Additions Data:

    +
    +
    + + + + + {% for data in additions %} + + + + {% endfor %} + +
    Added Data +
    {{ data }}
    +
    +
    + {% endif %} + + {% if modifications %} +

    Modified Data:

    + +
    +
    + + + + + {% for data in modifications %} + + + + + + {% endfor %} + +
    Original + Current + Diff +
    {{ data.get("Original") }}{{ data.get("Current") }}
    {{data.get("Diff")}}
    +
    +
    + {% endif %} + + {% if deletions %} +

    Deleted Data:

    +
    +
    + + + + + {% for data in deletions %} + + + + {% endfor %} + +
    Deleted Data +
    {{ data }}
    +
    +
    + {% endif %} + +
    +{%endblock%} + +{% block js %} + + + +{% endblock %} -- cgit v1.2.3 From 9944596f6873807c780e8939143f9d35b05d220b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 22 Dec 2021 12:41:55 +0300 Subject: Make "Files for approval:" text a "h2" --- wqflask/wqflask/templates/display_files.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/display_files.html b/wqflask/wqflask/templates/display_files.html index c8c446ba..5fad5d14 100644 --- a/wqflask/wqflask/templates/display_files.html +++ b/wqflask/wqflask/templates/display_files.html @@ -19,8 +19,8 @@
    {% if waiting %} +

    Files for approval:

    - Files for approval:
    -- cgit v1.2.3 From 9baebb774e3cf05feca09b0ffe44a73db17bd4d2 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 22 Dec 2021 12:46:20 +0300 Subject: display_diffs: Use datatables for every table --- wqflask/wqflask/templates/display_diffs.html | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/wqflask/wqflask/templates/display_diffs.html b/wqflask/wqflask/templates/display_diffs.html index edd889b5..c7921b3e 100644 --- a/wqflask/wqflask/templates/display_diffs.html +++ b/wqflask/wqflask/templates/display_diffs.html @@ -16,7 +16,7 @@

    Additions Data:

    -
    +
    @@ -37,7 +37,7 @@
    -
    Added Data
    +
    Original Current @@ -61,7 +61,7 @@

    Deleted Data:

    - +
    @@ -87,10 +87,9 @@ gn_server_url = "{{ gn_server_url }}"; $(document).ready( function() { - $('#table-approved').dataTable(); - }) - $(document).ready( function() { - $('#table-rejected').dataTable(); - }) + $('#table-additions').dataTable(); + $('#table-modifications').dataTable(); + $('#table-deletions').dataTable(); + }); {% endblock %} -- cgit v1.2.3 From 810d21845878afdffe5a3c7c35b0b88cb06fadbf Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 22 Dec 2021 13:39:03 +0300 Subject: Remove pudb line --- wqflask/wqflask/metadata_edits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 593605b5..e0ceaf03 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -472,7 +472,7 @@ def approve_data(resource_id:str, file_name: str): passwd=current_app.config.get("DB_PASS"), host=current_app.config.get("DB_HOST")) TMPDIR = current_app.config.get("TMPDIR") - import pudb; pu.db + # import pudb; pu.db with open(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), 'r') as myfile: sample_data = json.load(myfile) -- cgit v1.2.3 From a2941c93c724a1536e388fb45579524c61fb0441 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 22 Dec 2021 13:40:30 +0300 Subject: display_diffs: Remove in-line styling from "pre" tag --- wqflask/wqflask/templates/display_diffs.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/display_diffs.html b/wqflask/wqflask/templates/display_diffs.html index c7921b3e..e787e468 100644 --- a/wqflask/wqflask/templates/display_diffs.html +++ b/wqflask/wqflask/templates/display_diffs.html @@ -48,7 +48,7 @@ - + {% endfor %} -- cgit v1.2.3 From 4362e738f604e71e9482f832fa744631925fd1f6 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 22 Dec 2021 15:34:18 +0300 Subject: metadata_edits: Return early if uploaded csv has not been edited --- wqflask/wqflask/metadata_edits.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index e0ceaf03..c8e73eec 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -259,6 +259,13 @@ def update_phenotype(dataset_id: str, name: str): r = run_cmd(cmd=("csvdiff " f"'{uploaded_file_name}' '{new_file_name}' " "--format json")) + + # Edge case where the csv file has not been edited! + if not any(json.loads(r.get("output")).values()): + flash(f"You have not modified the csv file you downloaded!", + "warning") + return redirect(f"/datasets/{dataset_id}/traits/{name}" + f"?resource-id={request.args.get('resource-id')}") diff_output = (f"{TMPDIR}/sample-data/diffs/" f"{_file_name}.json") with open(diff_output, "w") as f: -- cgit v1.2.3 From a4e76455d79e1c683fcf3fa26d52463d0b54d55a Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 4 Jan 2022 13:21:15 +0300 Subject: Remove pudb debug statements --- wqflask/wqflask/metadata_edits.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index c8e73eec..3d450b36 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -479,7 +479,6 @@ def approve_data(resource_id:str, file_name: str): passwd=current_app.config.get("DB_PASS"), host=current_app.config.get("DB_HOST")) TMPDIR = current_app.config.get("TMPDIR") - # import pudb; pu.db with open(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), 'r') as myfile: sample_data = json.load(myfile) -- cgit v1.2.3 From 97ca71becd68ed24cd685f359dbe7fc10ffb5bda Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 4 Jan 2022 13:21:58 +0300 Subject: Check for any double-inserts or double-deletions Track any double-inserts or double-deletions --- wqflask/wqflask/metadata_edits.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 3d450b36..5b2b1e1d 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -495,18 +495,27 @@ def approve_data(resource_id:str, file_name: str): value=value, error=se, count=count) + + n_deletions = 0 for deletion in (deletions := [d for d in sample_data.get("Deletions")]): strain_name, _, _, _ = deletion.split(",") - delete_sample_data( + __deletions, _, _ = delete_sample_data( conn=conn, trait_name=sample_data.get("trait_name"), strain_name=strain_name, phenotype_id=int(sample_data.get("phenotype_id"))) + if __deletions: + n_deletions += 1 + # Remove any data that already exists from sample_data deletes + else: + sample_data.get("Deletions").remove(deletion) + + n_insertions = 0 for insertion in ( insertions := [d for d in sample_data.get("Additions")]): (strain_name, value, se, count) = insertion.split(",") - insert_sample_data( + __insertions, _, _ = insert_sample_data( conn=conn, trait_name=sample_data.get("trait_name"), strain_name=strain_name, @@ -516,6 +525,11 @@ def approve_data(resource_id:str, file_name: str): count=count) if any([any(modifications), any(deletions), any(insertions)]): + if __insertions: + n_insertions += 1 + # Remove any data that already exists from sample_data inserts + else: + sample_data.get("Additions").remove(insertion) insert(conn, table="metadata_audit", data=MetadataAudit( -- cgit v1.2.3 From 2824a0cb7f7ae3fab6b210d857c6c480fa9b5806 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 4 Jan 2022 13:24:24 +0300 Subject: Insert data to db by checking the tracked deletions, mods, or adds Prior to this, we only checked for the json files with contained the modifications. However, in the case of double deletions, and double inserts, the data would have been stored in the db anyway; which is a false representation of changes. --- wqflask/wqflask/metadata_edits.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 5b2b1e1d..1f8283f7 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -523,13 +523,14 @@ def approve_data(resource_id:str, file_name: str): value=value, error=se, count=count) - - if any([any(modifications), any(deletions), any(insertions)]): if __insertions: n_insertions += 1 # Remove any data that already exists from sample_data inserts else: sample_data.get("Additions").remove(insertion) + if any([sample_data.get("Additions"), + sample_data.get("Modifications"), + sample_data.get("Deletions")]): insert(conn, table="metadata_audit", data=MetadataAudit( -- cgit v1.2.3 From dd8a1678b4de4b5e280b2df1053fe287e2cc6e86 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 4 Jan 2022 13:26:19 +0300 Subject: Display correct message on the # of edits --- wqflask/wqflask/metadata_edits.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 1f8283f7..33e769ca 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -541,10 +541,19 @@ def approve_data(resource_id:str, file_name: str): os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), os.path.join(f"{TMPDIR}/sample-data/diffs", f"{file_name}.approved")) - flash((f"Just updated data from: {file_name};\n" - f"# Modifications: {len(modifications)}; " - f"# Additions: {len(insertions)}; " - f"# Deletions: {len(deletions)}"), - "success") + message = "" + if n_deletions: + flash(f"# Deletions: {n_deletions}", "success") + if n_insertions: + flash("# Additions: {len(modifications)", "success") + if len(modifications): + flash("# Modifications: {len(modifications)}", "success") + else: # Edge case where you need to automatically reject the file + os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), + os.path.join(f"{TMPDIR}/sample-data/diffs", + f"{file_name}.rejected")) + flash(("Automatically rejecting this file since no " + "changes could be applied."), "warning") + return redirect(url_for('metadata_edit.list_diffs')) -- cgit v1.2.3 From 2f0dacc477b2cd239b3da89078476845540de3e7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 4 Jan 2022 16:06:23 +0300 Subject: metadata_edits.py: Remove duplicate imports --- wqflask/wqflask/metadata_edits.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index 33e769ca..dc738f88 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -21,11 +21,9 @@ from flask import render_template from flask import request from flask import url_for -from wqflask.decorators import edit_access_required from wqflask.decorators import edit_access_required from wqflask.decorators import edit_admins_access_required from wqflask.decorators import login_required -from wqflask.decorators import login_required from gn3.authentication import AdminRole from gn3.authentication import DataRole -- cgit v1.2.3 From f4e336eb1ea526156e112cff97a3ec8137a2bc90 Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Mon, 3 Jan 2022 17:45:14 +0530 Subject: bin: Set GN2_PROFILE from GUIX_ENVIRONMENT. `guix shell' sets the profile path in an environment variable---GUIX_ENVIRONMENT. There is no need to pass it explicity as an input in GN2_PROFILE. * README.md: Remove all references to GN2_PROFILE. * bin/genenetwork2: Set GN2_PROFILE from GUIX_ENVIRONMENT. Do not check the value of GN2_PROFILE now that it is set automatically. Remove all references to GN2_PROFILE in example invocations. --- README.md | 17 +++++------- bin/genenetwork2 | 83 ++++++++++++++++++++------------------------------------ 2 files changed, 37 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 972d5c50..d5539a9f 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ genenetwork2 A quick example is ```sh -env GN2_PROFILE=~/opt/gn-latest SERVER_PORT=5300 \ +env SERVER_PORT=5300 \ GENENETWORK_FILES=~/data/gn2_data/ \ GN_PROXY_URL="http://localhost:8080"\ GN3_LOCAL_URL="http://localhost:8081"\ @@ -69,19 +69,16 @@ We are building 'Mechanical Rob' automated testing using Python which can be run with: ```sh -env GN2_PROFILE=~/opt/gn-latest \ - ./bin/genenetwork2 \ +env ./bin/genenetwork2 \ GN_PROXY_URL="http://localhost:8080" \ GN3_LOCAL_URL="http://localhost:8081 "\ ./etc/default_settings.py -c \ ../test/requests/test-website.py -a http://localhost:5003 ``` -The GN2_PROFILE is the Guix profile that contains all -dependencies. The ./bin/genenetwork2 script sets up the environment -and executes test-website.py in a Python interpreter. The -a switch -says to run all tests and the URL points to the running GN2 http -server. +The ./bin/genenetwork2 script sets up the environment and executes +test-website.py in a Python interpreter. The -a switch says to run all +tests and the URL points to the running GN2 http server. #### Unit tests @@ -102,9 +99,9 @@ runcmd coverage html The `runcmd` and `runpython` are shell aliases defined in the following way: ```sh -alias runpython="env GN2_PROFILE=~/opt/gn-latest TMPDIR=/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 +alias runpython="env TMPDIR=/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 -alias runcmd="time env GN2_PROFILE=~/opt/gn-latest TMPDIR=//tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 ./etc/default_settings.py -cli" +alias runcmd="time env TMPDIR=//tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 ./etc/default_settings.py -cli" ``` Replace some of the env variables as per your use case. diff --git a/bin/genenetwork2 b/bin/genenetwork2 index b5c940a8..93ab02f8 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -5,55 +5,45 @@ # # Typical usage # -# env GN2_PROFILE=~/opt/genenetwork2-phewas ./bin/genenetwork2 ~/my_settings.py -# -# Where GN2_PROFILE points to the GNU Guix profile used for deployment. +# ./bin/genenetwork2 ~/my_settings.py # # This will run the GN2 server (with default settings if none -# supplied). Typically you need a GNU Guix profile which is set with -# an environment variable (this profile is dictated by the -# installation path of genenetwork). Say your profile is in -# ~/opt/gn-latest-guix -# -# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 -# -# You can pass in your own settings file, e.g. -# -# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ~/my_settings.py +# supplied). # # To run a maintenance python script with settings (instead of the # webserver) run from the base-dir with settings file and add that # script with a -c switch, e.g. # -# env GN2_PROFILE=/usr/local/guix-profiles/gn-latest-20190905 TMPDIR=/export/local/home/zas1024/gn2-zach/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5002 GENENETWORK_FILES=/export/local/home/zas1024/gn2-zach/genotype_files SQL_URI=mysql://webqtlout:webqtlout@lily.uthsc.edu/db_webqtl ./bin/genenetwork2 ./etc/default_settings.py -c ./maintenance/gen_select_dataset.py +# env TMPDIR=/export/local/home/zas1024/gn2-zach/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5002 GENENETWORK_FILES=/export/local/home/zas1024/gn2-zach/genotype_files SQL_URI=mysql://webqtlout:webqtlout@lily.uthsc.edu/db_webqtl ./bin/genenetwork2 ./etc/default_settings.py -c ./maintenance/gen_select_dataset.py # # To run any script in the environment # -# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli echo "HELLO WORLD" +# ./bin/genenetwork2 ./etc/default_settings.py -cli echo "HELLO WORLD" # # To get a python REPL(!) # -# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli python +# ./bin/genenetwork2 ./etc/default_settings.py -cli python # # For development you may want to run # -# env GN2_PROFILE=~/opt/gn-latest WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2 +# env WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2 # # For staging and production we use gunicorn. Run with something like # (note you have to provide the server port). Provide a settings file! # -# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-prod +# env SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-prod # # For development use # -# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev +# env SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev # # For extra flexibility you can also provide gunicorn parameters yourself with something like # -# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi" +# ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi" SCRIPT=$(realpath "$0") echo SCRIPT=$SCRIPT +export GN2_PROFILE=$GUIX_ENVIRONMENT echo GN2_PROFILE=$GN2_PROFILE GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT")) GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR) @@ -101,40 +91,27 @@ fi export GN2_SETTINGS=$settings # Python echo GN2_SETTINGS=$settings -if [ -z $GN2_PROFILE ] ; then - echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" - export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) - if [ -d $GN2_PROFILE ]; then - echo "Best guess is $GN2_PROFILE" - fi - echo "ERROR: always set GN2_PROFILE" - exit 1 -fi -if [ -z $GN2_PROFILE ]; then - read -p "PRESS [ENTER] TO CONTINUE..." -else - export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH="$GN2_PROFILE/lib/python3.8/site-packages" # never inject another PYTHONPATH!! - export R_LIBS_SITE=$GN2_PROFILE/site-library - export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript - export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" - export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" - export XDG_DATA_DIRS="$GN2_PROFILE/share" - export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" - export LC_ALL=C # FIXME - export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" - export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" - export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" - if [ -z $GEMMA_WRAPPER_COMMAND ]; then - export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" - fi - while IFS=":" read -ra PPATH; do - for PPART in "${PPATH[@]}"; do - if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi - done - done <<< "$PYTHONPATH" - if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi +export PATH=$GN2_PROFILE/bin:$PATH +export PYTHONPATH="$GN2_PROFILE/lib/python3.8/site-packages" # never inject another PYTHONPATH!! +export R_LIBS_SITE=$GN2_PROFILE/site-library +export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript +export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" +export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" +export XDG_DATA_DIRS="$GN2_PROFILE/share" +export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" +export LC_ALL=C # FIXME +export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" +export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" +export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" +if [ -z $GEMMA_WRAPPER_COMMAND ]; then + export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" fi +while IFS=":" read -ra PPATH; do + for PPART in "${PPATH[@]}"; do + if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi + done +done <<< "$PYTHONPATH" +if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi if [ -z $PYTHONPATH ] ; then echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!" exit 1 -- cgit v1.2.3 From 344e428126b60932bff4c62c5ded8c36519155e8 Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Mon, 3 Jan 2022 17:51:04 +0530 Subject: bin: Do not set environment variables set by guix shell. The Guix python package accepts GUIX_PYTHONPATH instead of PYTHONPATH. `guix shell' sets GI_TYPELIB_PATH, GIO_EXTRA_MODULES, GUIX_GTK3_PATH, GUIX_PYTHONPATH, PATH, R_LIB_SITE and XDG_DATA_DIRS when necessary. There is no need to set these environment variables explicitly. * bin/genenetwork2: Do not set GI_TYPELIB_PATH, GIO_EXTRA_MODULES, GUIX_GTK3_PATH, PATH, PYTHONPATH, R_LIBS_SITE and XDG_DATA_DIRS. * README.md [Development]: Remove paragraph on injecting python modules locally. --- README.md | 4 ---- bin/genenetwork2 | 34 ---------------------------------- 2 files changed, 38 deletions(-) diff --git a/README.md b/README.md index d5539a9f..8dcbac9e 100644 --- a/README.md +++ b/README.md @@ -46,10 +46,6 @@ Also mariadb and redis need to be running, see ## Development -It may be useful to pull in the GN3 python modules locally. For this -use `GN3_PYTHONPATH` environment that gets injected in -the ./bin/genenetwork2 startup. - ## Testing To have tests pass, the redis and mariadb instance should be running, because of diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 93ab02f8..934a10d8 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -91,14 +91,7 @@ fi export GN2_SETTINGS=$settings # Python echo GN2_SETTINGS=$settings -export PATH=$GN2_PROFILE/bin:$PATH -export PYTHONPATH="$GN2_PROFILE/lib/python3.8/site-packages" # never inject another PYTHONPATH!! -export R_LIBS_SITE=$GN2_PROFILE/site-library export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript -export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" -export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" -export XDG_DATA_DIRS="$GN2_PROFILE/share" -export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" export LC_ALL=C # FIXME export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" @@ -106,25 +99,6 @@ export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" if [ -z $GEMMA_WRAPPER_COMMAND ]; then export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" fi -while IFS=":" read -ra PPATH; do - for PPART in "${PPATH[@]}"; do - if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi - done -done <<< "$PYTHONPATH" -if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi -if [ -z $PYTHONPATH ] ; then - echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!" - exit 1 -fi -if [ ! -d $R_LIBS_SITE ] ; then - echo "ERROR R_LIBS_SITE has not been set correctly (we only allow one path) - use GN2_PROFILE!" - echo "Paste into your shell the output of (for example)" - echo "guix package -p \$GN2_PROFILE --search-paths" - exit 1 -fi - -# We may change this one: -export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$GN3_PYTHONPATH:$PYTHONPATH # Our UNIX TMPDIR defaults to /tmp - change this on a shared server if [ -z $TMPDIR ]; then @@ -140,7 +114,6 @@ set|grep TMPDIR if [ "$1" = '-c' ] ; then cd $GN2_BASE_DIR/wqflask cmd=${2#wqflask/} - echo PYTHONPATH=$PYTHONPATH shift ; shift echo RUNNING COMMAND $cmd $* python $cmd $* @@ -151,7 +124,6 @@ fi if [ "$1" = "-cli" ] ; then cd $GN2_BASE_DIR/wqflask cmd=$2 - echo PYTHONPATH=$PYTHONPATH shift ; shift echo RUNNING COMMAND $cmd $* $cmd $* @@ -160,14 +132,12 @@ fi if [ "$1" = '-gunicorn' ] ; then cd $GN2_BASE_DIR/wqflask cmd=$2 - echo PYTHONPATH=$PYTHONPATH echo RUNNING gunicorn $cmd gunicorn $cmd exit $? fi if [ "$1" = '-gunicorn-dev' ] ; then cd $GN2_BASE_DIR/wqflask - echo PYTHONPATH=$PYTHONPATH if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --timeout 180 --reload wsgi" echo RUNNING gunicorn $cmd @@ -176,7 +146,6 @@ if [ "$1" = '-gunicorn-dev' ] ; then fi if [ "$1" = '-gunicorn-prod' ] ; then cd $GN2_BASE_DIR/wqflask - echo PYTHONPATH=$PYTHONPATH if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi PID=$TMPDIR/gunicorn.$USER.pid cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID --workers 20 --keep-alive 6000 --max-requests 100 --max-requests-jitter 30 --timeout 1200 wsgi" @@ -190,9 +159,6 @@ echo -n "dir $TMPDIR dbfilename gn2.rdb " | redis-server - & -# Overrides for packages that are not yet public (currently r-auwerx) -# export R_LIBS_SITE=$R_LIBS_SITE:$HOME/.Rlibs/das1i1pm54dj6lbdcsw5w0sdwhccyj1a-r-3.3.2/lib/R/lib - # Start the flask server running GN2 cd $GN2_BASE_DIR/wqflask echo "Starting with $settings" -- cgit v1.2.3 From 80003d3f892846c4475a1365c5c5129424970def Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Tue, 4 Jan 2022 11:29:27 +0530 Subject: bin: Set shebang to sh instead of bash. /bin/sh is present on Guix System and other distros while /bin/bash is present only on other distros. * bin/genenetwork2: Set shebang to sh instead of bash. --- bin/genenetwork2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 934a10d8..eaf3f6ca 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,4 +1,4 @@ -#! /bin/bash +#! /bin/sh # # This is the startup script for GN2. It sets the environment variables to pick # up a Guix profile and allows for overriding parameters. -- cgit v1.2.3 From 339e4d084bc8400f86b0fb3cf83bb3049f6b39ed Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Tue, 4 Jan 2022 11:32:40 +0530 Subject: bin: Stop execution on error. * bin/genenetwork2: Pass -e flag to sh. --- bin/genenetwork2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index eaf3f6ca..ce3678e4 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,4 +1,4 @@ -#! /bin/sh +#! /bin/sh -e # # This is the startup script for GN2. It sets the environment variables to pick # up a Guix profile and allows for overriding parameters. -- cgit v1.2.3 From ff9a6968af326c4c5480b1245e56f6d574b52ef7 Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Tue, 4 Jan 2022 11:37:52 +0530 Subject: README: Fix typo. * README.md: Change clinians to clinicians. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8dcbac9e..2b41a415 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This repository contains the current source code for GeneNetwork (GN) (https://www.genenetwork.org/ (version 2). GN2 is a Web 2.0-style framework that includes data and computational tools for online genetics and genomic analysis of many different populations and many types of molecular, cellular, and physiological data. -The system is used by scientists and clinians in the field of precision health care and systems genetics. +The system is used by scientists and clinicians in the field of precision health care and systems genetics. GN and its predecessors have been in operation since Jan 1994, making it one of the longest-lived web services in biomedical research (https://en.wikipedia.org/wiki/GeneNetwork, and see a partial list of publications using GN and its predecessor, WebQTL (https://genenetwork.org/references/). ## Install -- cgit v1.2.3 From 1bcbdfb6a659acf888fee40cb4033752e59e4b2c Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Tue, 4 Jan 2022 11:54:29 +0530 Subject: guix.scm: Add guix.scm. * guix.scm: New file. --- guix.scm | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 guix.scm diff --git a/guix.scm b/guix.scm new file mode 100644 index 00000000..9352c7c5 --- /dev/null +++ b/guix.scm @@ -0,0 +1,24 @@ +;; Make sure you have the +;; https://git.genenetwork.org/guix-bioinformatics/guix-bioinformatics +;; channel set up. +;; +;; To drop into a development environment, run +;; +;; guix shell -Df guix.scm +;; +;; To get a development environment in a container, run +;; +;; guix shell -C -Df guix.scm + +(use-modules (gn packages genenetwork) + (guix gexp) + (guix git-download) + (guix packages)) + +(define %source-dir (dirname (current-filename))) + +(package + (inherit genenetwork3) + (source (local-file %source-dir "genenetwork3-checkout" + #:recursive? #t + #:select? (git-predicate %source-dir)))) -- cgit v1.2.3 From e0a260b0de55fbea0c507eb0ca5fbdc3c1a825c0 Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Tue, 4 Jan 2022 12:08:27 +0530 Subject: README: Document `guix shell' development process. * README.md: Clean up and document `guix shell' development process. --- README.md | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 2b41a415..ea473699 100644 --- a/README.md +++ b/README.md @@ -11,24 +11,27 @@ many different populations and many types of molecular, cellular, and physiologi The system is used by scientists and clinicians in the field of precision health care and systems genetics. GN and its predecessors have been in operation since Jan 1994, making it one of the longest-lived web services in biomedical research (https://en.wikipedia.org/wiki/GeneNetwork, and see a partial list of publications using GN and its predecessor, WebQTL (https://genenetwork.org/references/). -## Install - -The recommended installation is with GNU Guix which allows you to -deploy GN2 and dependencies as a self contained unit on any machine. -The database can be run separately as well as the source tree (for -developers). See the [installation docs](doc/README.org). - ## Run -Once having installed GN2 it can be run through a browser -interface +We recommend you use GNU Guix. GNU Guix allows you to deploy +GeneNetwork2 and dependencies as a self contained unit on any machine. +The database can be run separately as well as the source tree (for +developers). +Make sure you have the +[guix-bioinformatics](https://git.genenetwork.org/guix-bioinformatics/guix-bioinformatics) +channel set up. Then, to drop into a development environment with all +dependencies, run ```sh -genenetwork2 +guix shell -Df guix.scm +``` +Or, to drop into a development environment in a container, run +``` +guix shell -C --network -Df guix.scm ``` -A quick example is - +In the development environment, start GeneNetwork2 by running, for +example, ```sh env SERVER_PORT=5300 \ GENENETWORK_FILES=~/data/gn2_data/ \ @@ -39,13 +42,12 @@ env SERVER_PORT=5300 \ For full examples (you may need to set a number of environment variables), including running scripts and a Python REPL, also see the -startup script [./bin/genenetwork2](https://github.com/genenetwork/genenetwork2/blob/testing/bin/genenetwork2). +startup script +[./bin/genenetwork2](https://github.com/genenetwork/genenetwork2/blob/testing/bin/genenetwork2). Also mariadb and redis need to be running, see [INSTALL](./doc/README.org). -## Development - ## Testing To have tests pass, the redis and mariadb instance should be running, because of -- cgit v1.2.3 From 9ab0c3b6cc146e1711f1478242d4198eed720e4c Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 11 Jan 2022 22:17:36 +0000 Subject: Removed the deferRender setting, since it appears to revent DataTables API from being able to access all rows (which caused functions like Select All to only select the first 100-200 rows) --- wqflask/wqflask/templates/search_result_page.html | 1 - 1 file changed, 1 deletion(-) diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index dade6ba5..f73cba17 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -446,7 +446,6 @@ "sDom": "iti", "destroy": true, "autoWidth": false, - "deferRender": true, "bSortClasses": false, "scrollY": "500px", "scrollCollapse": true, -- cgit v1.2.3
    Deleted Data
    {{ data.get("Original") }} {{ data.get("Current") }}
    {{data.get("Diff")}}
    {{data.get("Diff")}}