From b0ccb12682fed83bf72d22ff42f1f442a8e6176e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 19 Apr 2021 14:43:16 +0300 Subject: Remove stale comments --- wqflask/base/data_set.py | 11 ---- wqflask/utility/helper_functions.py | 4 -- wqflask/wqflask/show_trait/show_trait.py | 72 +++++++++++++++----------- wqflask/wqflask/templates/index_page_orig.html | 10 ---- wqflask/wqflask/templates/submit_trait.html | 12 ----- wqflask/wqflask/views.py | 23 ++------ 6 files changed, 46 insertions(+), 86 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 178234fe..cc5a428c 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -262,8 +262,6 @@ class Markers(object): elif isinstance(p_values, dict): filtered_markers = [] for marker in self.markers: - #logger.debug("marker[name]", marker['name']) - #logger.debug("p_values:", p_values) if marker['name'] in p_values: #logger.debug("marker {} IS in p_values".format(i)) marker['p_value'] = p_values[marker['name']] @@ -276,10 +274,6 @@ class Markers(object): marker['lrs_value'] = - \ math.log10(marker['p_value']) * 4.61 filtered_markers.append(marker) - # else: - #logger.debug("marker {} NOT in p_values".format(i)) - # self.markers.remove(marker) - #del self.markers[i] self.markers = filtered_markers @@ -306,7 +300,6 @@ class HumanMarkers(Markers): marker['Mb'] = float(splat[3]) / 1000000 self.markers.append(marker) - #logger.debug("markers is: ", pf(self.markers)) def add_pvalues(self, p_values): super(HumanMarkers, self).add_pvalues(p_values) @@ -520,7 +513,6 @@ def datasets(group_name, this_group=None): break if tissue_already_exists: - #logger.debug("dataset_menu:", dataset_menu[i]['datasets']) dataset_menu[i]['datasets'].append((dataset, dataset_short)) else: dataset_menu.append(dict(tissue=tissue_name, @@ -735,9 +727,6 @@ class PhenotypeDataSet(DataSet): DS_NAME_MAP['Publish'] = 'PhenotypeDataSet' def setup(self): - - #logger.debug("IS A PHENOTYPEDATASET") - # Fields in the database table self.search_fields = ['Phenotype.Post_publication_description', 'Phenotype.Pre_publication_description', diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index 7eb7f013..15d5b3ab 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -10,7 +10,6 @@ import logging logger = logging.getLogger(__name__ ) def get_species_dataset_trait(self, start_vars): - #assert type(read_genotype) == type(bool()), "Expecting boolean value for read_genotype" if "temp_trait" in list(start_vars.keys()): if start_vars['temp_trait'] == "True": self.dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = start_vars['group']) @@ -27,9 +26,6 @@ def get_species_dataset_trait(self, start_vars): get_qtl_info=True) logger.debug("After creating trait") - #if read_genotype: - #self.dataset.group.read_genotype_file() - #self.genotype = self.dataset.group.genotype def get_trait_db_obs(self, trait_db_list): if isinstance(trait_db_list, str): diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 6892f02b..ed55d473 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -138,17 +138,12 @@ class ShowTrait(object): self.ncbi_summary = get_ncbi_summary(self.this_trait) - #Get nearest marker for composite mapping + # Get nearest marker for composite mapping if not self.temp_trait: if check_if_attr_exists(self.this_trait, 'locus_chr') and self.dataset.type != "Geno" and self.dataset.type != "Publish": self.nearest_marker = get_nearest_marker(self.this_trait, self.dataset) - #self.nearest_marker1 = get_nearest_marker(self.this_trait, self.dataset)[0] - #self.nearest_marker2 = get_nearest_marker(self.this_trait, self.dataset)[1] else: self.nearest_marker = "" - #self.nearest_marker1 = "" - #self.nearest_marker2 = "" - self.make_sample_lists() @@ -168,16 +163,19 @@ class ShowTrait(object): categorical_var_list = [] if not self.temp_trait: - categorical_var_list = get_categorical_variables(self.this_trait, self.sample_groups[0]) #ZS: Only using first samplelist, since I think mapping only uses those samples + # ZS: Only using first samplelist, since I think mapping only uses those samples + categorical_var_list = get_categorical_variables(self.this_trait, self.sample_groups[0]) - #ZS: Get list of chromosomes to select for mapping + # ZS: Get list of chromosomes to select for mapping self.chr_list = [["All", -1]] for i, this_chr in enumerate(self.dataset.species.chromosomes.chromosomes): self.chr_list.append([self.dataset.species.chromosomes.chromosomes[this_chr].name, i]) self.genofiles = self.dataset.group.get_genofiles() - if "QTLReaper" or "R/qtl" in dataset.group.mapping_names: #ZS: No need to grab scales from .geno file unless it's using a mapping method that reads .geno files + # ZS: No need to grab scales from .geno file unless it's using + # a mapping method that reads .geno files + if "QTLReaper" or "R/qtl" in dataset.group.mapping_names: if self.genofiles: self.scales_in_geno = get_genotype_scales(self.genofiles) else: @@ -187,10 +185,15 @@ class ShowTrait(object): self.has_num_cases = has_num_cases(self.this_trait) - #ZS: Needed to know whether to display bar chart + get max sample name length in order to set table column width + # ZS: Needed to know whether to display bar chart + get max + # sample name length in order to set table column width self.num_values = 0 - self.binary = "true" #ZS: So it knows whether to display the Binary R/qtl mapping method, which doesn't work unless all values are 0 or 1 - self.negative_vals_exist = "false" #ZS: Since we don't want to show log2 transform option for situations where it doesn't make sense + # ZS: So it knows whether to display the Binary R/qtl mapping + # method, which doesn't work unless all values are 0 or 1 + self.binary = "true" + # ZS: Since we don't want to show log2 transform option for + # situations where it doesn't make sense + self.negative_vals_exist = "false" max_samplename_width = 1 for group in self.sample_groups: for sample in group.sample_list: @@ -203,7 +206,8 @@ class ShowTrait(object): if sample.value < 0: self.negative_vals_exist = "true" - #ZS: Check whether any attributes have few enough distinct values to show the "Block samples by group" option + # ZS: Check whether any attributes have few enough distinct + # values to show the "Block samples by group" option self.categorical_attr_exists = "false" for attribute in self.sample_groups[0].attributes: if len(self.sample_groups[0].attributes[attribute].distinct_values) <= 10: @@ -258,7 +262,6 @@ class ShowTrait(object): if not self.temp_trait: if hasattr(self.this_trait, 'locus_chr') and self.this_trait.locus_chr != "" and self.dataset.type != "Geno" and self.dataset.type != "Publish": hddn['control_marker'] = self.nearest_marker - #hddn['control_marker'] = self.nearest_marker1+","+self.nearest_marker2 hddn['do_control'] = False hddn['maf'] = 0.05 hddn['mapping_scale'] = "physic" @@ -268,7 +271,8 @@ class ShowTrait(object): if len(self.scales_in_geno) < 2: hddn['mapping_scale'] = self.scales_in_geno[list(self.scales_in_geno.keys())[0]][0][0] - # We'll need access to this_trait and hddn in the Jinja2 Template, so we put it inside self + # We'll need access to this_trait and hddn in the Jinja2 + # Template, so we put it inside self self.hddn = hddn js_data = dict(trait_id = self.trait_id, @@ -294,7 +298,8 @@ class ShowTrait(object): self.js_data = js_data def get_external_links(self): - #ZS: There's some weirdness here because some fields don't exist while others are empty strings + # ZS: There's some weirdness here because some fields don't + # exist while others are empty strings self.pubmed_link = webqtlConfig.PUBMEDLINK_URL % self.this_trait.pubmed_id if check_if_attr_exists(self.this_trait, 'pubmed_id') else None self.ncbi_gene_link = webqtlConfig.NCBI_LOCUSID % self.this_trait.geneid if check_if_attr_exists(self.this_trait, 'geneid') else None self.omim_link = webqtlConfig.OMIM_ID % self.this_trait.omim if check_if_attr_exists(self.this_trait, 'omim') else None @@ -320,7 +325,6 @@ class ShowTrait(object): self.panther_link = webqtlConfig.PANTHER_URL % self.this_trait.symbol self.ebi_gwas_link = webqtlConfig.EBIGWAS_URL % self.this_trait.symbol self.protein_atlas_link = webqtlConfig.PROTEIN_ATLAS_URL % self.this_trait.symbol - #self.open_targets_link = webqtlConfig.OPEN_TARGETS_URL % self.this_trait.symbol if self.dataset.group.species == "mouse" or self.dataset.group.species == "human": self.rgd_link = webqtlConfig.RGD_URL % (self.this_trait.symbol, self.dataset.group.species.capitalize()) @@ -429,7 +433,9 @@ class ShowTrait(object): all_samples_ordered.append(sample) other_sample_names.append(sample) - #ZS: CFW is here because the .geno file doesn't properly contain its full list of samples. This should probably be fixed. + # ZS: CFW is here because the .geno file doesn't properly + # contain its full list of samples. This should probably + # be fixed. if self.dataset.group.species == "human" or (set(primary_sample_names) == set(parent_f1_samples)) or self.dataset.group.name == "CFW": primary_sample_names += other_sample_names other_sample_names = [] @@ -445,7 +451,8 @@ class ShowTrait(object): sample_group_type='primary', header=primary_header) - #if other_sample_names and self.dataset.group.species != "human" and self.dataset.group.name != "CFW": + # if other_sample_names and self.dataset.group.species != + # "human" and self.dataset.group.name != "CFW": if len(other_sample_names) > 0: other_sample_names.sort() #Sort other samples if parent_f1_samples: @@ -539,7 +546,8 @@ def get_z_scores(sample_groups): def get_nearest_marker(this_trait, this_db): this_chr = this_trait.locus_chr this_mb = this_trait.locus_mb - #One option is to take flanking markers, another is to take the two (or one) closest + # One option is to take flanking markers, another is to take the + # two (or one) closest query = """SELECT Geno.Name FROM Geno, GenoXRef, GenoFreeze WHERE Geno.Chr = '{}' AND @@ -552,7 +560,6 @@ def get_nearest_marker(this_trait, this_db): if result == []: return "" - #return "", "" else: return result[0][0] @@ -617,7 +624,8 @@ def check_if_attr_exists(the_trait, id_type): def get_ncbi_summary(this_trait): if check_if_attr_exists(this_trait, 'geneid'): - #ZS: Need to switch this try/except to something that checks the output later + # ZS: Need to switch this try/except to something that checks + # the output later try: response = requests.get("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=gene&id=%s&retmode=json" % this_trait.geneid) summary = json.loads(response.content)['result'][this_trait.geneid]['summary'] @@ -661,8 +669,8 @@ def get_genotype_scales(genofiles): def get_scales_from_genofile(file_location): geno_path = locate_ignore_error(file_location, 'genotype') - - if not geno_path: #ZS: This is just to allow the code to run when + # ZS: This is just to allow the code to run when + if not geno_path: return [["physic", "Mb"]] cm_and_mb_cols_exist = True cm_column = None @@ -670,7 +678,9 @@ def get_scales_from_genofile(file_location): with open(geno_path, "r") as geno_fh: for i, line in enumerate(geno_fh): if line[0] == "#" or line[0] == "@": - if "@scale" in line: #ZS: If the scale is made explicit in the metadata, use that + # ZS: If the scale is made explicit in the metadata, + # use that + if "@scale" in line: scale = line.split(":")[1].strip() if scale == "morgan": return [["morgan", "cM"]] @@ -690,12 +700,16 @@ def get_scales_from_genofile(file_location): mb_column = 3 break - #ZS: This attempts to check whether the cM and Mb columns are 'real', since some .geno files have one column be a copy of the other column, or have one column that is all 0s + # ZS: This attempts to check whether the cM and Mb columns are + # 'real', since some .geno files have one column be a copy of + # the other column, or have one column that is all 0s cm_all_zero = True mb_all_zero = True cm_mb_all_equal = True for i, line in enumerate(geno_fh): - if first_marker_line <= i < first_marker_line + 10: #ZS: I'm assuming there won't be more than 10 markers where the position is listed as 0 + # ZS: I'm assuming there won't be more than 10 markers + # where the position is listed as 0 + if first_marker_line <= i < first_marker_line + 10: if cm_column: cm_val = line.split("\t")[cm_column].strip() if cm_val != "0": @@ -711,8 +725,8 @@ def get_scales_from_genofile(file_location): if i > first_marker_line + 10: break - - #ZS: This assumes that both won't be all zero, since if that's the case mapping shouldn't be an option to begin with + # ZS: This assumes that both won't be all zero, since if that's + # the case mapping shouldn't be an option to begin with if mb_all_zero: return [["morgan", "cM"]] elif cm_mb_all_equal: diff --git a/wqflask/wqflask/templates/index_page_orig.html b/wqflask/wqflask/templates/index_page_orig.html index 7f82b35c..87cf1b45 100755 --- a/wqflask/wqflask/templates/index_page_orig.html +++ b/wqflask/wqflask/templates/index_page_orig.html @@ -7,16 +7,6 @@ {% endblock %} {% block content %} - - -
{{ flash_me() }} diff --git a/wqflask/wqflask/templates/submit_trait.html b/wqflask/wqflask/templates/submit_trait.html index 68b06f55..334a608d 100644 --- a/wqflask/wqflask/templates/submit_trait.html +++ b/wqflask/wqflask/templates/submit_trait.html @@ -61,18 +61,6 @@
-

Paste or Type Multiple Values: You can enter data by pasting a series of numbers representing trait values into this area. diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 2c0ba586..c4b510d4 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -156,12 +156,6 @@ def index_page(): import_collections = params['import_collections'] if import_collections == "true": g.user_session.import_traits_to_user(params['anon_id']) - #if USE_GN_SERVER: - # # The menu is generated using GN_SERVER - # return render_template("index_page.html", gn_server_url = GN_SERVER_URL, version=GN_VERSION) - #else: - - # Old style static menu (OBSOLETE) return render_template("index_page_orig.html", version=GN_VERSION) @@ -343,14 +337,10 @@ def intro(): @app.route("/tutorials") def tutorials(): - #doc = Docs("links", request.args) - #return render_template("docs.html", **doc.__dict__) return render_template("tutorials.html") @app.route("/credits") def credits(): - #doc = Docs("links", request.args) - #return render_template("docs.html", **doc.__dict__) return render_template("credits.html") @app.route("/update_text", methods=('POST',)) @@ -368,12 +358,9 @@ def submit_trait_form(): @app.route("/create_temp_trait", methods=('POST',)) def create_temp_trait(): logger.info(request.url) - - #template_vars = submit_trait.SubmitTrait(request.form) - doc = Docs("links") return render_template("links.html", **doc.__dict__) - #return render_template("show_trait.html", **template_vars.__dict__) + @app.route('/export_trait_excel', methods=('POST',)) def export_trait_excel(): @@ -487,21 +474,17 @@ def export_perm_data(): mimetype='text/csv', headers={"Content-Disposition":"attachment;filename=" + file_name + ".csv"}) + @app.route("/show_temp_trait", methods=('POST',)) def show_temp_trait_page(): logger.info(request.url) template_vars = show_trait.ShowTrait(request.form) - #logger.info("js_data before dump:", template_vars.js_data) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, indent=" ") - # Sorting the keys messes up the ordered dictionary, so don't do that - #sort_keys=True) - - #logger.info("js_data after dump:", template_vars.js_data) - #logger.info("show_trait template_vars:", pf(template_vars.__dict__)) return render_template("show_trait.html", **template_vars.__dict__) + @app.route("/show_trait") def show_trait_page(): logger.info(request.url) -- cgit v1.2.3 From 4534daa6fb07c23b90e024560ca64091fc330eed Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 19 Apr 2021 17:46:38 +0300 Subject: Move looped sql query into one statement in "get_species_groups" It's in-efficient to have a sql query executed in a loop. As data grows, the query becomes slower. It's better to let sql handle such queries. --- wqflask/utility/helper_functions.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index 15d5b3ab..4ba92ed5 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -47,19 +47,18 @@ def get_trait_db_obs(self, trait_db_list): if trait_ob: self.trait_list.append((trait_ob, dataset_ob)) -def get_species_groups(): - - species_query = "SELECT SpeciesId, MenuName FROM Species" - species_ids_and_names = g.db.execute(species_query).fetchall() - - species_and_groups = [] - for species_id, species_name in species_ids_and_names: - this_species_groups = {} - this_species_groups['species'] = species_name - groups_query = "SELECT InbredSetName FROM InbredSet WHERE SpeciesId = %s" % (species_id) - groups = [group[0] for group in g.db.execute(groups_query).fetchall()] - this_species_groups['groups'] = groups - species_and_groups.append(this_species_groups) - - return species_and_groups +def get_species_groups(): + """Group each species into a group""" + _menu = {} + for species, group_name in g.db.execute( + "SELECT s.MenuName, i.InbredSetName FROM InbredSet i " + "INNER JOIN Species s ON s.SpeciesId = i.SpeciesId " + "ORDER BY i.SpeciesId ASC, i.Name ASC").fetchall(): + if _menu.get(species): + _menu = _menu[species].append(group_name) + else: + _menu[species] = [group_name] + return [{"species": key, + "groups": value} for key, value in + list(_menu.items())] -- cgit v1.2.3 From d2e2046a3ce1af0ca97ea1b6d9ccb3a4c9aecf7c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 23 Apr 2021 17:21:12 +0300 Subject: Add full link to genetic data collected as part of WebQTL project --- wqflask/wqflask/templates/submit_trait.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/submit_trait.html b/wqflask/wqflask/templates/submit_trait.html index 334a608d..2cc18240 100644 --- a/wqflask/wqflask/templates/submit_trait.html +++ b/wqflask/wqflask/templates/submit_trait.html @@ -14,7 +14,7 @@

Introduction


The trait values that you enter are statistically compared with verified genotypes collected at a set of microsatellite markers in each RI set. The markers are drawn from a set of over 750, but for each set redundant markers have been removed, preferentially retaining those that are most informative.

-

These error-checked RI mapping data match theoretical expectations for RI strain sets. The cumulative adjusted length of the RI maps are approximately 1400 cM, a value that matches those of both MIT maps and Chromosome Committee Report maps. See our full description of the genetic data collected as part of the WebQTL project.

+

These error-checked RI mapping data match theoretical expectations for RI strain sets. The cumulative adjusted length of the RI maps are approximately 1400 cM, a value that matches those of both MIT maps and Chromosome Committee Report maps. See our full description of the genetic data collected as part of the WebQTL project.


-- cgit v1.2.3 From 2114ad9e84ad7778e048b52cf865b5f031ceab88 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 23 Apr 2021 17:27:31 +0300 Subject: Rename index_page_orig to index_page --- wqflask/wqflask/templates/index_page.html | 222 +++++++++------- wqflask/wqflask/templates/index_page_orig.html | 339 ------------------------- wqflask/wqflask/views.py | 2 +- 3 files changed, 136 insertions(+), 427 deletions(-) mode change 100644 => 100755 wqflask/wqflask/templates/index_page.html delete mode 100755 wqflask/wqflask/templates/index_page_orig.html diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html old mode 100644 new mode 100755 index 31846f87..87cf1b45 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -1,78 +1,73 @@ {% extends "base.html" %} {% block title %}GeneNetwork{% endblock %} +{% block css %} + +{% endblock %} {% block content %} - - - -
+
{{ flash_me() }} -
+
-
+
- + + + + -->
-

Websites affiliated with GeneNetwork

-
    -
  • Genome Browser at UTHSC
  • - -
  • Galaxy at - UTHSC
  • - -
  • GeneNetwork 1 at Amazon - Cloud (EC2)
  • - -
  • GeneNetwork 1 Source Code at SourceForge
  • - -
  • GeneNetwork 2 Source Code at GitHub
  • -
-

GN1 Mirror and development sites

- +

GeneNetwork v2:

+ +

GeneNetwork v1:

-
+ + @@ -268,11 +303,7 @@ {%endblock%} {% block js %} - - - + + + + + + {% endblock %} diff --git a/wqflask/wqflask/templates/index_page_orig.html b/wqflask/wqflask/templates/index_page_orig.html deleted file mode 100755 index 87cf1b45..00000000 --- a/wqflask/wqflask/templates/index_page_orig.html +++ /dev/null @@ -1,339 +0,0 @@ -{% extends "base.html" %} -{% block title %}GeneNetwork{% endblock %} -{% block css %} - -{% endblock %} -{% block content %} - -
- - {{ flash_me() }} - -
- -
- -
- - -

You can also use advanced commands. Copy these simple examples into the Get Any field for single term searches and Combined for searches with multiple terms:

- -
    -
  • POSITION=(chr1 25 30) finds genes, markers, or transcripts on - chromosome 1 between 25 and 30 Mb.
  • - -
  • MEAN=(15 16) in the Combined field finds - highly expressed genes (15 to 16 log2 units)
  • - -
  • RANGE=(1.5 2.5) in the Any field finds traits with values with a specified fold-range (minimum = 1). - Useful for finding "housekeeping genes" (1.0 1.2) or highly variable molecular assays (10 100).
  • - -
  • LRS=(15 1000) or LOD=(2 8) finds all traits with peak LRS or LOD scores between lower and upper limits.
  • - -
  • LRS=(9 999 Chr4 122 155) finds all traits on Chr 4 from 122 and 155 Mb with LRS scores between 9 and 999.
  • - -
  • cisLRS=(15 1000 5) or cisLOD=(2 8 5) finds all cis eQTLs with peak LRS or LOD scores between lower and upper limits, - with an inclusion zone of 5 Mb around the parent gene.
  • - -
  • transLRS=(15 1000 5) or transLOD=(2 8 5) finds all trans eQTLs with peak LRS or LOD scores between lower and upper limits, - with an exclusion zone of 5 Mb around the parent gene. You can also add a fourth term specifying which chromosome you want the transLRS to be on - (for example transLRS=(15 1000 5 7) would find all trans eQTLs with peak LRS on chromosome 7 that is also a trans eQTL with exclusionary zone of 5Mb).
  • - -
  • POSITION=(Chr4 122 130) cisLRS=(9 999 10) - finds all traits on Chr 4 from 122 and 155 Mb with cisLRS scores - between 9 and 999 and an inclusion zone of 10 Mb.
  • - -
  • RIF=mitochondrial searches RNA databases for - GeneRIF links.
  • - -
  • WIKI=nicotine searches - GeneWiki for genes that you or other users have annotated - with the word nicotine.
  • - -
  • GO:0045202 searches for synapse-associated genes listed in the - - Gene Ontology.
  • - -
  • RIF=diabetes LRS=(9 999 Chr2 100 105) transLRS=(9 999 10) - finds diabetes-associated transcripts with peak - trans eQTLs on Chr 2 between 100 and 105 Mb with LRS - scores between 9 and 999.
  • -
-
-
- -
-
- -
-
- -
-
- -
- -
- -

GeneNetwork v2:

- -

GeneNetwork v1:

-
    -
  • Main website at UTHSC
  • -
  • Time Machine: Full GN versions from 2009 to 2016 (mm9)
  • - Cloud (EC2) -
- -
- - -
-
-
- -{%endblock%} - -{% block js %} - - - - - - - - - -{% endblock %} diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index c4b510d4..156b4772 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -156,7 +156,7 @@ def index_page(): import_collections = params['import_collections'] if import_collections == "true": g.user_session.import_traits_to_user(params['anon_id']) - return render_template("index_page_orig.html", version=GN_VERSION) + return render_template("index_page.html", version=GN_VERSION) @app.route("/tmp/") -- cgit v1.2.3 From 8bad987466203baed948ba761d248e24f04ca49c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 23 Apr 2021 17:32:35 +0300 Subject: Rewrite For Loop in a more Pythonic way --- wqflask/wqflask/api/gen_menu.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/wqflask/wqflask/api/gen_menu.py b/wqflask/wqflask/api/gen_menu.py index 18afc5ad..eaddecd7 100644 --- a/wqflask/wqflask/api/gen_menu.py +++ b/wqflask/wqflask/api/gen_menu.py @@ -23,12 +23,7 @@ def get_species(): """Build species list""" results = g.db.execute( "SELECT Name, MenuName FROM Species ORDER BY OrderId").fetchall() - - species = [] - for result in results: - species.append([str(result[0]), str(result[1])]) - - return species + return [[name, menu_name] for name, menu_name in results] def get_groups(species): -- cgit v1.2.3 From d66a8d2ccbf10b53b31ef094835c5ba8ec672a68 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 23 Apr 2021 17:35:54 +0300 Subject: Apply PEP-8 --- wqflask/utility/helper_functions.py | 28 +++++++++++++++++----------- wqflask/wqflask/views.py | 11 +++++++++-- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index 4ba92ed5..e0ae3414 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -4,19 +4,23 @@ from base.species import TheSpecies from utility import hmac -from flask import Flask, g +from flask import g import logging -logger = logging.getLogger(__name__ ) +logger = logging.getLogger(__name__) + def get_species_dataset_trait(self, start_vars): if "temp_trait" in list(start_vars.keys()): - if start_vars['temp_trait'] == "True": - self.dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = start_vars['group']) - else: - self.dataset = data_set.create_dataset(start_vars['dataset']) + if start_vars['temp_trait'] == "True": + self.dataset = data_set.create_dataset( + dataset_name="Temp", + dataset_type="Temp", + group_name=start_vars['group']) + else: + self.dataset = data_set.create_dataset(start_vars['dataset']) else: - self.dataset = data_set.create_dataset(start_vars['dataset']) + self.dataset = data_set.create_dataset(start_vars['dataset']) logger.debug("After creating dataset") self.species = TheSpecies(dataset=self.dataset) logger.debug("After creating species") @@ -35,15 +39,17 @@ def get_trait_db_obs(self, trait_db_list): for trait in trait_db_list: data, _separator, hmac_string = trait.rpartition(':') data = data.strip() - assert hmac_string==hmac.hmac_creation(data), "Data tampering?" + assert hmac_string == hmac.hmac_creation(data), "Data tampering?" trait_name, dataset_name = data.split(":") if dataset_name == "Temp": - dataset_ob = data_set.create_dataset(dataset_name=dataset_name, dataset_type="Temp", group_name=trait_name.split("_")[2]) + dataset_ob = data_set.create_dataset( + dataset_name=dataset_name, dataset_type="Temp", + group_name=trait_name.split("_")[2]) else: dataset_ob = data_set.create_dataset(dataset_name) trait_ob = create_trait(dataset=dataset_ob, - name=trait_name, - cellid=None) + name=trait_name, + cellid=None) if trait_ob: self.trait_list.append((trait_ob, dataset_ob)) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 156b4772..36033d80 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -334,26 +334,33 @@ def intro(): return render_template("docs.html", **doc.__dict__) - @app.route("/tutorials") def tutorials(): return render_template("tutorials.html") + @app.route("/credits") def credits(): return render_template("credits.html") + @app.route("/update_text", methods=('POST',)) def update_page(): update_text(request.form) doc = Docs(request.form['entry_type'], request.form) return render_template("docs.html", **doc.__dict__) + @app.route("/submit_trait") def submit_trait_form(): logger.info(request.url) species_and_groups = get_species_groups() - return render_template("submit_trait.html", **{'species_and_groups' : species_and_groups, 'gn_server_url' : GN_SERVER_URL, 'version' : GN_VERSION}) + return render_template( + "submit_trait.html", + **{'species_and_groups': species_and_groups, + 'gn_server_url': GN_SERVER_URL, + 'version': GN_VERSION}) + @app.route("/create_temp_trait", methods=('POST',)) def create_temp_trait(): -- cgit v1.2.3 From f97c298c1653c538be1753c87336f9f75c3a754c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 23 Apr 2021 20:19:37 +0300 Subject: Remove stale comments --- wqflask/wqflask/templates/index_page.html | 77 ------------------------------- wqflask/wqflask/views.py | 7 +-- 2 files changed, 1 insertion(+), 83 deletions(-) diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html index 87cf1b45..942776e7 100755 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -65,12 +65,6 @@
- - - - - -
@@ -116,12 +110,10 @@
-
-
@@ -207,40 +199,10 @@
-
- -
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 36033d80..baaece2f 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -496,17 +496,12 @@ def show_temp_trait_page(): def show_trait_page(): logger.info(request.url) template_vars = show_trait.ShowTrait(request.args) - #logger.info("js_data before dump:", template_vars.js_data) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, indent=" ") - # Sorting the keys messes up the ordered dictionary, so don't do that - #sort_keys=True) - - #logger.info("js_data after dump:", template_vars.js_data) - #logger.info("show_trait template_vars:", pf(template_vars.__dict__)) return render_template("show_trait.html", **template_vars.__dict__) + @app.route("/heatmap", methods=('POST',)) def heatmap_page(): logger.info("In heatmap, request.form is:", pf(request.form)) -- cgit v1.2.3 From 243f884b0f7e5e42aaab7d7c8076479818be8578 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 27 Apr 2021 14:35:37 +0300 Subject: templates: submit_trait.html: Add field for inputting trait name --- wqflask/wqflask/templates/submit_trait.html | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/submit_trait.html b/wqflask/wqflask/templates/submit_trait.html index 2cc18240..3572b0a9 100644 --- a/wqflask/wqflask/templates/submit_trait.html +++ b/wqflask/wqflask/templates/submit_trait.html @@ -53,7 +53,7 @@ -
+

2. Enter Trait Data:

File uploading isn't enabled yet, but is coming soon.


@@ -77,6 +77,24 @@
+
+

3. Enable Use of Trait Variance:

+
+ +
+
+
+

+ Name Your Trait: (optional) +

+ +
+
+
+ + +
+
-- cgit v1.2.3 From f799477b49ef8c2bd8d8408b0c1b1ee6094cb8ca Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 28 Apr 2021 20:53:13 +0300 Subject: wqflask: views.py: Clean up imports section Remove unused imports and break up long imports into shorter lines. --- wqflask/wqflask/views.py | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index baaece2f..91fb8df2 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1,20 +1,17 @@ """Main routing table for GN2""" -import traceback # for error page +import traceback # for error page import os # for error gifs import random # for random error gif import datetime # for errors -import time # for errors import sys import csv import simplejson as json -import yaml import xlsxwriter import io # Todo: Use cStringIO? from zipfile import ZipFile, ZIP_DEFLATED -import gc import numpy as np import pickle as pickle import uuid @@ -24,23 +21,27 @@ import base64 import array import sqlalchemy from wqflask import app -from flask import g, Response, request, make_response, render_template, send_from_directory, jsonify, redirect, url_for, send_file +from flask import g +from flask import Response +from flask import request +from flask import make_response +from flask import render_template +from flask import send_from_directory +from flask import redirect +from flask import url_for +from flask import send_file -from wqflask import group_manager -from wqflask import resource_manager from wqflask import search_results -from wqflask import export_traits -from wqflask import gsearch -from wqflask import update_search_results -from wqflask import docs from wqflask import news from wqflask import server_side from wqflask.submit_bnw import get_bnw_input -from base.data_set import create_dataset, DataSet # Used by YAML in marker_regression +from base.data_set import create_dataset # Used by YAML in marker_regression from wqflask.show_trait import show_trait from wqflask.show_trait import export_trait_data from wqflask.heatmap import heatmap -from wqflask.external_tools import send_to_bnw, send_to_webgestalt, send_to_geneweaver +from wqflask.external_tools import send_to_bnw +from wqflask.external_tools import send_to_webgestalt +from wqflask.external_tools import send_to_geneweaver from wqflask.comparison_bar_chart import comparison_bar_chart from wqflask.marker_regression import run_mapping from wqflask.marker_regression import display_mapping_results @@ -59,24 +60,31 @@ from wqflask.docs import Docs, update_text from wqflask.db_info import InfoPage from utility import temp_data -from utility.tools import SQL_URI, TEMPDIR, USE_REDIS, USE_GN_SERVER, GN_SERVER_URL, GN_VERSION, JS_TWITTER_POST_FETCHER_PATH, JS_GUIX_PATH, CSS_PATH +from utility.tools import SQL_URI +from utility.tools import TEMPDIR +from utility.tools import USE_REDIS +from utility.tools import GN_SERVER_URL +from utility.tools import GN_VERSION +from utility.tools import JS_TWITTER_POST_FETCHER_PATH +from utility.tools import JS_GUIX_PATH from utility.helper_functions import get_species_groups from utility.authentication_tools import check_resource_availability from utility.redis_tools import get_redis_conn -Redis = get_redis_conn() + from base.webqtlConfig import GENERATED_IMAGE_DIR, DEFAULT_PRIVILEGES from utility.benchmark import Bench from pprint import pformat as pf -from wqflask import collect from wqflask.database import db_session -import werkzeug import utility.logger -logger = utility.logger.getLogger(__name__ ) + +Redis = get_redis_conn() + +logger = utility.logger.getLogger(__name__) @app.before_request -- cgit v1.2.3 From 5ae817e71a7232a747c988cd9f6abfa25a35971c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 28 Apr 2021 21:17:01 +0300 Subject: wqflask: views: Remove news import This name clashes with "def news" which is defined later. --- wqflask/wqflask/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 91fb8df2..3aafd7d3 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -32,7 +32,6 @@ from flask import url_for from flask import send_file from wqflask import search_results -from wqflask import news from wqflask import server_side from wqflask.submit_bnw import get_bnw_input from base.data_set import create_dataset # Used by YAML in marker_regression -- cgit v1.2.3 From ed434e8fba9f807d94892d06d6191f5de6670bd9 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 28 Apr 2021 21:20:32 +0300 Subject: Remove local variables that are unused --- wqflask/wqflask/views.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 3aafd7d3..b1c2ed68 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -97,7 +97,6 @@ def connect_db(): @app.before_request def check_access_permissions(): logger.debug("@app.before_request check_access_permissions") - available = True if 'dataset' in request.args: permissions = DEFAULT_PRIVILEGES if request.args['dataset'] != "Temp": @@ -656,7 +655,6 @@ def loading_page(): dataset = create_dataset(start_vars['dataset'], group_name = start_vars['group']) else: dataset = create_dataset(start_vars['dataset']) - genofile_samplelist = [] samples = start_vars['primary_samples'].split(",") if 'genofile' in start_vars: if start_vars['genofile'] != "": @@ -846,7 +844,6 @@ def export_pdf(): svg_xml = request.form.get("data", "Invalid data") logger.info("svg_xml:", svg_xml) filename = request.form.get("filename", "interval_map_pdf") - filepath = GENERATED_IMAGE_DIR+filename pdf_file = cairosvg.svg2pdf(bytestring=svg_xml) response = Response(pdf_file, mimetype="application/pdf") response.headers["Content-Disposition"] = "attachment; filename=%s"%filename @@ -942,7 +939,6 @@ def security_tutorial_page(): @app.route("/submit_bnw", methods=('POST',)) def submit_bnw(): logger.info(request.url) - template_vars = get_bnw_input(request.form) return render_template("empty_collection.html", **{'tool':'Correlation Matrix'}) # Take this out or secure it before putting into production -- cgit v1.2.3 From 41791eb9eb809c11b5bd953235ae98c4b4d82156 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 28 Apr 2021 21:20:54 +0300 Subject: Remove stale comments --- wqflask/wqflask/views.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index b1c2ed68..31ba8df0 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -972,9 +972,6 @@ def json_default_handler(obj): # Handle custom objects if hasattr(obj, '__dict__'): return obj.__dict__ - #elif type(obj) == "Dataset": - # logger.info("Not going to serialize Dataset") - # return None else: raise TypeError('Object of type %s with value of %s is not JSON serializable' % ( type(obj), repr(obj))) -- cgit v1.2.3 From ca1bc2a22f93591ac9c120bca1ba554ae891c9ab Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 28 Apr 2021 21:21:03 +0300 Subject: Use right block level comments --- wqflask/wqflask/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 31ba8df0..a33c64f1 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -962,7 +962,7 @@ def browser_inputs(): ########################################################################## def json_default_handler(obj): - '''Based on http://stackoverflow.com/a/2680060/1175849''' + """Based on http://stackoverflow.com/a/2680060/1175849""" # Handle datestamps if hasattr(obj, 'isoformat'): return obj.isoformat() -- cgit v1.2.3 From f23a198951e3aa3040b6ae8c28aba0fd34514ebd Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 28 Apr 2021 21:24:04 +0300 Subject: wqflask: views: Apply pep-8 --- wqflask/wqflask/views.py | 222 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 154 insertions(+), 68 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index a33c64f1..319f1270 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -91,9 +91,11 @@ def connect_db(): logger.info("@app.before_request connect_db") db = getattr(g, '_database', None) if db is None: - g.db = g._database = sqlalchemy.create_engine(SQL_URI, encoding="latin1") + g.db = g._database = sqlalchemy.create_engine( + SQL_URI, encoding="latin1") logger.debug(g.db) + @app.before_request def check_access_permissions(): logger.debug("@app.before_request check_access_permissions") @@ -105,7 +107,8 @@ def check_access_permissions(): if dataset.type == "Temp": permissions = DEFAULT_PRIVILEGES elif 'trait_id' in request.args: - permissions = check_resource_availability(dataset, request.args['trait_id']) + permissions = check_resource_availability( + dataset, request.args['trait_id']) elif dataset.type != "Publish": permissions = check_resource_availability(dataset) @@ -116,6 +119,7 @@ def check_access_permissions(): if permissions['data'] == 'no-access': return redirect(url_for("no_access_page")) + @app.teardown_appcontext def shutdown_session(exception=None): db = getattr(g, '_database', None) @@ -124,6 +128,7 @@ def shutdown_session(exception=None): db_session.remove() g.db = None + @app.errorhandler(Exception) def handle_bad_request(e): err_msg = str(e) @@ -134,25 +139,30 @@ def handle_bad_request(e): logger.error(traceback.format_exc()) now = datetime.datetime.utcnow() time_str = now.strftime('%l:%M%p UTC %b %d, %Y') - formatted_lines = [request.url + " ("+time_str+")"]+traceback.format_exc().splitlines() + formatted_lines = [request.url + + " ("+time_str+")"]+traceback.format_exc().splitlines() # Handle random animations # Use a cookie to have one animation on refresh animation = request.cookies.get(err_msg[:32]) if not animation: - list = [fn for fn in os.listdir("./wqflask/static/gif/error") if fn.endswith(".gif") ] + list = [fn for fn in os.listdir( + "./wqflask/static/gif/error") if fn.endswith(".gif")] animation = random.choice(list) - resp = make_response(render_template("error.html", message=err_msg, stack=formatted_lines, error_image=animation, version=GN_VERSION)) + resp = make_response(render_template("error.html", message=err_msg, + stack=formatted_lines, error_image=animation, version=GN_VERSION)) # logger.error("Set cookie %s with %s" % (err_msg, animation)) resp.set_cookie(err_msg[:32], animation) return resp + @app.route("/authentication_needed") def no_access_page(): return render_template("new_security/not_authenticated.html") + @app.route("/") def index_page(): logger.info("Sending index_page") @@ -177,7 +187,7 @@ def tmp_page(img_path): imgB64 = base64.b64encode(imgdata) bytesarray = array.array('B', imgB64) return render_template("show_image.html", - img_base64 = bytesarray ) + img_base64=bytesarray) @app.route("/js/") @@ -189,6 +199,7 @@ def js(filename): name = name.replace('js_alt/', '') return send_from_directory(js_path, name) + @app.route("/css/") def css(filename): js_path = JS_GUIX_PATH @@ -198,10 +209,12 @@ def css(filename): name = name.replace('js_alt/', '') return send_from_directory(js_path, name) + @app.route("/twitter/") def twitter(filename): return send_from_directory(JS_TWITTER_POST_FETCHER_PATH, filename) + @app.route("/search", methods=('GET',)) def search_page(): logger.info("in search_page") @@ -209,7 +222,8 @@ def search_page(): result = None if USE_REDIS: with Bench("Trying Redis cache"): - key = "search_results:v1:" + json.dumps(request.args, sort_keys=True) + key = "search_results:v1:" + \ + json.dumps(request.args, sort_keys=True) logger.debug("key is:", pf(key)) result = Redis.get(key) if result: @@ -232,6 +246,7 @@ def search_page(): else: return render_template("search_error.html") + @app.route("/search_table", methods=('GET',)) def search_page_table(): logger.info("in search_page table") @@ -242,7 +257,7 @@ def search_page_table(): logger.info(type(the_search.trait_list)) logger.info(the_search.trait_list) - + current_page = server_side.ServerSideTable( len(the_search.trait_list), the_search.trait_list, @@ -252,6 +267,7 @@ def search_page_table(): return flask.jsonify(current_page) + @app.route("/gsearch", methods=('GET',)) def gsearchact(): logger.info(request.url) @@ -262,6 +278,7 @@ def gsearchact(): elif type == "phenotype": return render_template("gsearch_pheno.html", **result) + @app.route("/gsearch_updating", methods=('POST',)) def gsearch_updating(): logger.info("REQUEST ARGS:", request.values) @@ -292,41 +309,59 @@ def generated_file(filename): logger.info(request.url) return send_from_directory(GENERATED_IMAGE_DIR, filename) + @app.route("/help") def help(): logger.info(request.url) doc = Docs("help", request.args) return render_template("docs.html", **doc.__dict__) + @app.route("/wgcna_setup", methods=('POST',)) def wcgna_setup(): - logger.info("In wgcna, request.form is:", request.form) # We are going to get additional user input for the analysis + # We are going to get additional user input for the analysis + logger.info("In wgcna, request.form is:", request.form) logger.info(request.url) - return render_template("wgcna_setup.html", **request.form) # Display them using the template + # Display them using the template + return render_template("wgcna_setup.html", **request.form) + @app.route("/wgcna_results", methods=('POST',)) def wcgna_results(): logger.info("In wgcna, request.form is:", request.form) logger.info(request.url) - wgcna = wgcna_analysis.WGCNA() # Start R, load the package and pointers and create the analysis - wgcnaA = wgcna.run_analysis(request.form) # Start the analysis, a wgcnaA object should be a separate long running thread - result = wgcna.process_results(wgcnaA) # After the analysis is finished store the result - return render_template("wgcna_results.html", **result) # Display them using the template + # Start R, load the package and pointers and create the analysis + wgcna = wgcna_analysis.WGCNA() + # Start the analysis, a wgcnaA object should be a separate long running thread + wgcnaA = wgcna.run_analysis(request.form) + # After the analysis is finished store the result + result = wgcna.process_results(wgcnaA) + # Display them using the template + return render_template("wgcna_results.html", **result) + @app.route("/ctl_setup", methods=('POST',)) def ctl_setup(): - logger.info("In ctl, request.form is:", request.form) # We are going to get additional user input for the analysis + # We are going to get additional user input for the analysis + logger.info("In ctl, request.form is:", request.form) logger.info(request.url) - return render_template("ctl_setup.html", **request.form) # Display them using the template + # Display them using the template + return render_template("ctl_setup.html", **request.form) + @app.route("/ctl_results", methods=('POST',)) def ctl_results(): logger.info("In ctl, request.form is:", request.form) logger.info(request.url) - ctl = ctl_analysis.CTL() # Start R, load the package and pointers and create the analysis - ctlA = ctl.run_analysis(request.form) # Start the analysis, a ctlA object should be a separate long running thread - result = ctl.process_results(ctlA) # After the analysis is finished store the result - return render_template("ctl_results.html", **result) # Display them using the template + # Start R, load the package and pointers and create the analysis + ctl = ctl_analysis.CTL() + # Start the analysis, a ctlA object should be a separate long running thread + ctlA = ctl.run_analysis(request.form) + # After the analysis is finished store the result + result = ctl.process_results(ctlA) + # Display them using the template + return render_template("ctl_results.html", **result) + @app.route("/news") def news(): @@ -381,9 +416,11 @@ def export_trait_excel(): logger.info("In export_trait_excel") logger.info("request.form:", request.form) logger.info(request.url) - trait_name, sample_data = export_trait_data.export_sample_table(request.form) + trait_name, sample_data = export_trait_data.export_sample_table( + request.form) - logger.info("sample_data - type: %s -- size: %s" % (type(sample_data), len(sample_data))) + logger.info("sample_data - type: %s -- size: %s" % + (type(sample_data), len(sample_data))) buff = io.BytesIO() workbook = xlsxwriter.Workbook(buff, {'in_memory': True}) @@ -397,7 +434,8 @@ def export_trait_excel(): return Response(excel_data, mimetype='application/vnd.ms-excel', - headers={"Content-Disposition":"attachment;filename="+ trait_name + ".xlsx"}) + headers={"Content-Disposition": "attachment;filename=" + trait_name + ".xlsx"}) + @app.route('/export_trait_csv', methods=('POST',)) def export_trait_csv(): @@ -405,9 +443,11 @@ def export_trait_csv(): logger.info("In export_trait_csv") logger.info("request.form:", request.form) logger.info(request.url) - trait_name, sample_data = export_trait_data.export_sample_table(request.form) + trait_name, sample_data = export_trait_data.export_sample_table( + request.form) - logger.info("sample_data - type: %s -- size: %s" % (type(sample_data), len(sample_data))) + logger.info("sample_data - type: %s -- size: %s" % + (type(sample_data), len(sample_data))) buff = io.StringIO() writer = csv.writer(buff) @@ -418,7 +458,8 @@ def export_trait_csv(): return Response(csv_data, mimetype='text/csv', - headers={"Content-Disposition":"attachment;filename="+ trait_name + ".csv"}) + headers={"Content-Disposition": "attachment;filename=" + trait_name + ".csv"}) + @app.route('/export_traits_csv', methods=('POST',)) def export_traits_csv(): @@ -443,7 +484,8 @@ def export_traits_csv(): else: return Response(file_list[0][1], mimetype='text/csv', - headers={"Content-Disposition":"attachment;filename=" + file_list[0][0]}) + headers={"Content-Disposition": "attachment;filename=" + file_list[0][0]}) + @app.route('/export_perm_data', methods=('POST',)) def export_perm_data(): @@ -454,7 +496,8 @@ def export_perm_data(): now = datetime.datetime.now() time_str = now.strftime('%H:%M_%d%B%Y') - file_name = "Permutation_" + perm_info['num_perm'] + "_" + perm_info['trait_name'] + "_" + time_str + file_name = "Permutation_" + \ + perm_info['num_perm'] + "_" + perm_info['trait_name'] + "_" + time_str the_rows = [ ["#Permutation Test"], @@ -468,10 +511,14 @@ def export_perm_data(): ["#N_genotypes: " + str(perm_info['n_genotypes'])], ["#Genotype_file: " + perm_info['genofile']], ["#Units_linkage: " + perm_info['units_linkage']], - ["#Permutation_stratified_by: " + ", ".join([ str(cofactor) for cofactor in perm_info['strat_cofactors']])], - ["#RESULTS_1: Suggestive LRS(p=0.63) = " + str(np.percentile(np.array(perm_info['perm_data']), 67))], - ["#RESULTS_2: Significant LRS(p=0.05) = " + str(np.percentile(np.array(perm_info['perm_data']), 95))], - ["#RESULTS_3: Highly Significant LRS(p=0.01) = " + str(np.percentile(np.array(perm_info['perm_data']), 99))], + ["#Permutation_stratified_by: " + + ", ".join([str(cofactor) for cofactor in perm_info['strat_cofactors']])], + ["#RESULTS_1: Suggestive LRS(p=0.63) = " + + str(np.percentile(np.array(perm_info['perm_data']), 67))], + ["#RESULTS_2: Significant LRS(p=0.05) = " + str( + np.percentile(np.array(perm_info['perm_data']), 95))], + ["#RESULTS_3: Highly Significant LRS(p=0.01) = " + str( + np.percentile(np.array(perm_info['perm_data']), 99))], ["#Comment: Results sorted from low to high peak linkage"] ] @@ -485,7 +532,7 @@ def export_perm_data(): return Response(csv_data, mimetype='text/csv', - headers={"Content-Disposition":"attachment;filename=" + file_name + ".csv"}) + headers={"Content-Disposition": "attachment;filename=" + file_name + ".csv"}) @app.route("/show_temp_trait", methods=('POST',)) @@ -519,7 +566,8 @@ def heatmap_page(): traits = [trait.strip() for trait in start_vars['trait_list'].split(',')] if traits[0] != "": version = "v5" - key = "heatmap:{}:".format(version) + json.dumps(start_vars, sort_keys=True) + key = "heatmap:{}:".format( + version) + json.dumps(start_vars, sort_keys=True) logger.info("key is:", pf(key)) with Bench("Loading cache"): result = Redis.get(key) @@ -540,7 +588,8 @@ def heatmap_page(): result = template_vars.__dict__ for item in list(template_vars.__dict__.keys()): - logger.info(" ---**--- {}: {}".format(type(template_vars.__dict__[item]), item)) + logger.info( + " ---**--- {}: {}".format(type(template_vars.__dict__[item]), item)) pickled_result = pickle.dumps(result, pickle.HIGHEST_PROTOCOL) logger.info("pickled result length:", len(pickled_result)) @@ -551,10 +600,12 @@ def heatmap_page(): rendered_template = render_template("heatmap.html", **result) else: - rendered_template = render_template("empty_collection.html", **{'tool':'Heatmap'}) + rendered_template = render_template( + "empty_collection.html", **{'tool': 'Heatmap'}) return rendered_template + @app.route("/bnw_page", methods=('POST',)) def bnw_page(): logger.info("In run BNW, request.form is:", pf(request.form)) @@ -569,10 +620,12 @@ def bnw_page(): result = template_vars.__dict__ rendered_template = render_template("bnw_page.html", **result) else: - rendered_template = render_template("empty_collection.html", **{'tool':'BNW'}) + rendered_template = render_template( + "empty_collection.html", **{'tool': 'BNW'}) return rendered_template + @app.route("/webgestalt_page", methods=('POST',)) def webgestalt_page(): logger.info("In run WebGestalt, request.form is:", pf(request.form)) @@ -587,10 +640,12 @@ def webgestalt_page(): result = template_vars.__dict__ rendered_template = render_template("webgestalt_page.html", **result) else: - rendered_template = render_template("empty_collection.html", **{'tool':'WebGestalt'}) + rendered_template = render_template( + "empty_collection.html", **{'tool': 'WebGestalt'}) return rendered_template + @app.route("/geneweaver_page", methods=('POST',)) def geneweaver_page(): logger.info("In run WebGestalt, request.form is:", pf(request.form)) @@ -605,10 +660,12 @@ def geneweaver_page(): result = template_vars.__dict__ rendered_template = render_template("geneweaver_page.html", **result) else: - rendered_template = render_template("empty_collection.html", **{'tool':'GeneWeaver'}) + rendered_template = render_template( + "empty_collection.html", **{'tool': 'GeneWeaver'}) return rendered_template + @app.route("/comparison_bar_chart", methods=('POST',)) def comp_bar_chart_page(): logger.info("In comp bar chart, request.form is:", pf(request.form)) @@ -620,26 +677,30 @@ def comp_bar_chart_page(): if traits[0] != "": template_vars = comparison_bar_chart.ComparisonBarChart(request.form) template_vars.js_data = json.dumps(template_vars.js_data, - default=json_default_handler, - indent=" ") + default=json_default_handler, + indent=" ") result = template_vars.__dict__ - rendered_template = render_template("comparison_bar_chart.html", **result) + rendered_template = render_template( + "comparison_bar_chart.html", **result) else: - rendered_template = render_template("empty_collection.html", **{'tool':'Comparison Bar Chart'}) + rendered_template = render_template( + "empty_collection.html", **{'tool': 'Comparison Bar Chart'}) return rendered_template + @app.route("/mapping_results_container") def mapping_results_container_page(): return render_template("mapping_results_container.html") + @app.route("/loading", methods=('POST',)) def loading_page(): logger.info(request.url) initial_start_vars = request.form start_vars_container = {} - n_samples = 0 #ZS: So it can be displayed on loading page + n_samples = 0 # ZS: So it can be displayed on loading page if 'wanted_inputs' in initial_start_vars: wanted = initial_start_vars['wanted_inputs'].split(",") start_vars = {} @@ -652,7 +713,8 @@ def loading_page(): else: sample_vals_dict = json.loads(start_vars['sample_vals']) if 'group' in start_vars: - dataset = create_dataset(start_vars['dataset'], group_name = start_vars['group']) + dataset = create_dataset( + start_vars['dataset'], group_name=start_vars['group']) else: dataset = create_dataset(start_vars['dataset']) samples = start_vars['primary_samples'].split(",") @@ -660,7 +722,8 @@ def loading_page(): if start_vars['genofile'] != "": genofile_string = start_vars['genofile'] dataset.group.genofile = genofile_string.split(":")[0] - genofile_samples = run_mapping.get_genofile_samplelist(dataset) + genofile_samples = run_mapping.get_genofile_samplelist( + dataset) if len(genofile_samples) > 1: samples = genofile_samples @@ -680,6 +743,7 @@ def loading_page(): return rendered_template + @app.route("/run_mapping", methods=('POST',)) def mapping_results_page(): initial_start_vars = request.form @@ -750,9 +814,10 @@ def mapping_results_page(): start_vars[key] = value version = "v3" - key = "mapping_results:{}:".format(version) + json.dumps(start_vars, sort_keys=True) + key = "mapping_results:{}:".format( + version) + json.dumps(start_vars, sort_keys=True) with Bench("Loading cache"): - result = None # Just for testing + result = None # Just for testing #result = Redis.get(key) #logger.info("************************ Starting result *****************") @@ -772,12 +837,12 @@ def mapping_results_page(): rendered_template = render_template("mapping_error.html") return rendered_template except: - rendered_template = render_template("mapping_error.html") - return rendered_template + rendered_template = render_template("mapping_error.html") + return rendered_template template_vars.js_data = json.dumps(template_vars.js_data, - default=json_default_handler, - indent=" ") + default=json_default_handler, + indent=" ") result = template_vars.__dict__ @@ -792,18 +857,22 @@ def mapping_results_page(): imgB64 = base64.b64encode(imgdata) bytesarray = array.array('B', imgB64) result['pair_scan_array'] = bytesarray - rendered_template = render_template("pair_scan_results.html", **result) + rendered_template = render_template( + "pair_scan_results.html", **result) else: - gn1_template_vars = display_mapping_results.DisplayMappingResults(result).__dict__ + gn1_template_vars = display_mapping_results.DisplayMappingResults( + result).__dict__ with Bench("Rendering template"): #if (gn1_template_vars['mapping_method'] == "gemma") or (gn1_template_vars['mapping_method'] == "plink"): #gn1_template_vars.pop('qtlresults', None) - rendered_template = render_template("mapping_results.html", **gn1_template_vars) + rendered_template = render_template( + "mapping_results.html", **gn1_template_vars) return rendered_template -@app.route("/export_mapping_results", methods = ('POST',)) + +@app.route("/export_mapping_results", methods=('POST',)) def export_mapping_results(): logger.info("request.form:", request.form) logger.info(request.url) @@ -811,32 +880,35 @@ def export_mapping_results(): results_csv = open(file_path, "r").read() response = Response(results_csv, mimetype='text/csv', - headers={"Content-Disposition":"attachment;filename=mapping_results.csv"}) + headers={"Content-Disposition": "attachment;filename=mapping_results.csv"}) return response -@app.route("/export_corr_matrix", methods = ('POST',)) + +@app.route("/export_corr_matrix", methods=('POST',)) def export_corr_matrix(): file_path = request.form.get("export_filepath") file_name = request.form.get("export_filename") results_csv = open(file_path, "r").read() response = Response(results_csv, mimetype='text/csv', - headers={"Content-Disposition":"attachment;filename=" + file_name + ".csv"}) + headers={"Content-Disposition": "attachment;filename=" + file_name + ".csv"}) return response -@app.route("/export", methods = ('POST',)) + +@app.route("/export", methods=('POST',)) def export(): logger.info("request.form:", request.form) logger.info(request.url) svg_xml = request.form.get("data", "Invalid data") filename = request.form.get("filename", "manhattan_plot_snp") response = Response(svg_xml, mimetype="image/svg+xml") - response.headers["Content-Disposition"] = "attachment; filename=%s"%filename + response.headers["Content-Disposition"] = "attachment; filename=%s" % filename return response -@app.route("/export_pdf", methods = ('POST',)) + +@app.route("/export_pdf", methods=('POST',)) def export_pdf(): import cairosvg logger.info("request.form:", request.form) @@ -846,9 +918,10 @@ def export_pdf(): filename = request.form.get("filename", "interval_map_pdf") pdf_file = cairosvg.svg2pdf(bytestring=svg_xml) response = Response(pdf_file, mimetype="application/pdf") - response.headers["Content-Disposition"] = "attachment; filename=%s"%filename + response.headers["Content-Disposition"] = "attachment; filename=%s" % filename return response + @app.route("/network_graph", methods=('POST',)) def network_graph_page(): logger.info("In network_graph, request.form is:", pf(request.form)) @@ -863,7 +936,8 @@ def network_graph_page(): return render_template("network_graph.html", **template_vars.__dict__) else: - return render_template("empty_collection.html", **{'tool':'Network Graph'}) + return render_template("empty_collection.html", **{'tool': 'Network Graph'}) + @app.route("/corr_compute", methods=('POST',)) def corr_compute_page(): @@ -872,6 +946,7 @@ def corr_compute_page(): template_vars = show_corr_results.CorrelationResults(request.form) return render_template("correlation_page.html", **template_vars.__dict__) + @app.route("/corr_matrix", methods=('POST',)) def corr_matrix_page(): logger.info("In corr_matrix, request.form is:", pf(request.form)) @@ -887,7 +962,8 @@ def corr_matrix_page(): return render_template("correlation_matrix.html", **template_vars.__dict__) else: - return render_template("empty_collection.html", **{'tool':'Correlation Matrix'}) + return render_template("empty_collection.html", **{'tool': 'Correlation Matrix'}) + @app.route("/corr_scatter_plot") def corr_scatter_plot_page(): @@ -898,6 +974,7 @@ def corr_scatter_plot_page(): indent=" ") return render_template("corr_scatterplot.html", **template_vars.__dict__) + @app.route("/snp_browser", methods=('GET',)) def snp_browser_page(): logger.info(request.url) @@ -905,12 +982,14 @@ def snp_browser_page(): return render_template("snp_browser.html", **template_vars.__dict__) + @app.route("/db_info", methods=('GET',)) def db_info_page(): template_vars = InfoPage(request.args) return render_template("info_page.html", **template_vars.__dict__) + @app.route("/snp_browser_table", methods=('GET',)) def snp_browser_table(): logger.info(request.url) @@ -924,30 +1003,36 @@ def snp_browser_table(): return flask.jsonify(current_page) + @app.route("/tutorial/WebQTLTour", methods=('GET',)) def tutorial_page(): - #ZS: Currently just links to GN1 + # ZS: Currently just links to GN1 logger.info(request.url) return redirect("http://gn1.genenetwork.org/tutorial/WebQTLTour/") + @app.route("/tutorial/security", methods=('GET',)) def security_tutorial_page(): - #ZS: Currently just links to GN1 + # ZS: Currently just links to GN1 logger.info(request.url) return render_template("admin/security_help.html") + @app.route("/submit_bnw", methods=('POST',)) def submit_bnw(): logger.info(request.url) - return render_template("empty_collection.html", **{'tool':'Correlation Matrix'}) + return render_template("empty_collection.html", **{'tool': 'Correlation Matrix'}) # Take this out or secure it before putting into production + + @app.route("/get_temp_data") def get_temp_data(): logger.info(request.url) temp_uuid = request.args['key'] return flask.jsonify(temp_data.TempData(temp_uuid).get_all()) + @app.route("/browser_input", methods=('GET',)) def browser_inputs(): """ Returns JSON from tmp directory for the purescript genome browser""" @@ -961,6 +1046,7 @@ def browser_inputs(): ########################################################################## + def json_default_handler(obj): """Based on http://stackoverflow.com/a/2680060/1175849""" # Handle datestamps -- cgit v1.2.3 From a68795a639a731e2bdae0a977bccb01f291fc9f7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 28 Apr 2021 21:25:01 +0300 Subject: wqflask: views: Delete stale comments --- wqflask/wqflask/views.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 319f1270..28ab630f 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -285,11 +285,7 @@ def gsearch_updating(): logger.info(request.url) result = UpdateGSearch(request.args).__dict__ return result['results'] - # type = request.args['type'] - # if type == "gene": - # return render_template("gsearch_gene_updating.html", **result) - # elif type == "phenotype": - # return render_template("gsearch_pheno.html", **result) + @app.route("/docedit") def docedit(): @@ -864,8 +860,6 @@ def mapping_results_page(): result).__dict__ with Bench("Rendering template"): - #if (gn1_template_vars['mapping_method'] == "gemma") or (gn1_template_vars['mapping_method'] == "plink"): - #gn1_template_vars.pop('qtlresults', None) rendered_template = render_template( "mapping_results.html", **gn1_template_vars) -- cgit v1.2.3 From 6e8afe5a6871a680778bfaed6b79c934b33d0307 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 28 Apr 2021 19:40:58 +0000 Subject: Account for temp traits when adding covariates with GEMMA mapping --- wqflask/wqflask/marker_regression/gemma_mapping.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index 83ebcdf9..06c9300a 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -146,7 +146,12 @@ def gen_covariates_file(this_dataset, covariates, samples): for covariate in covariate_list: this_covariate_data = [] trait_name = covariate.split(":")[0] - dataset_ob = create_dataset(covariate.split(":")[1]) + dataset_name = covariate.split(":")[1] + if dataset_name == "Temp": + temp_group = trait_name.split("_")[2] + dataset_ob = create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = temp_group) + else: + dataset_ob = create_dataset(covariate.split(":")[1]) trait_ob = create_trait(dataset=dataset_ob, name=trait_name, cellid=None) -- cgit v1.2.3 From 65a2c8c1f455c5fad48fbac1a3e75310d49ed844 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 28 Apr 2021 19:45:37 +0000 Subject: Added something to jsonable in trait.py to account for temp traits (this is necessary for them to show up correctly when selecting traits from collections in pop-up windows, like when selecting cofactors for mapping) --- wqflask/base/trait.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index df96d46e..b4b4452a 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -337,6 +337,10 @@ def jsonable(trait): dataset_name=dataset.shortname, location=trait.location_repr ) + elif dataset.name == "Temp": + return dict(name=trait.name, + dataset="Temp", + dataset_name="Temp") else: return dict() -- cgit v1.2.3 From 90a6a950fba0bb463cff9669f04a87cab8efb36c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Apr 2021 11:49:42 +0300 Subject: wqflask: views: Replace dict de-structuring with kw arguments * wqflask/wqflask/views.py (submit_trait_form): Use kw arguments when passing variables to the template. This is more readable. --- wqflask/wqflask/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 28ab630f..38c56b71 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -394,9 +394,9 @@ def submit_trait_form(): species_and_groups = get_species_groups() return render_template( "submit_trait.html", - **{'species_and_groups': species_and_groups, - 'gn_server_url': GN_SERVER_URL, - 'version': GN_VERSION}) + species_and_groups=species_and_groups, + gn_server_url=GN_SERVER_URL, + version=GN_VERSION) @app.route("/create_temp_trait", methods=('POST',)) -- cgit v1.2.3 From e048c6321b37a1c05adbfb6754513d7920b10dd8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Apr 2021 12:45:28 +0300 Subject: templates: edit_trait.html: New file Copy of submit_trait.html. This is a copy; that'll form the basis of the new edit page. --- wqflask/wqflask/templates/edit_trait.html | 111 ++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 wqflask/wqflask/templates/edit_trait.html diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html new file mode 100644 index 00000000..3572b0a9 --- /dev/null +++ b/wqflask/wqflask/templates/edit_trait.html @@ -0,0 +1,111 @@ +{% extends "base.html" %} +{% block title %}Trait Submission{% endblock %} +{% block content %} + +
+
+ + {{ flash_me() }} + +
+
+
+
+

Introduction

+
+

The trait values that you enter are statistically compared with verified genotypes collected at a set of microsatellite markers in each RI set. The markers are drawn from a set of over 750, but for each set redundant markers have been removed, preferentially retaining those that are most informative.

+

These error-checked RI mapping data match theoretical expectations for RI strain sets. The cumulative adjusted length of the RI maps are approximately 1400 cM, a value that matches those of both MIT maps and Chromosome Committee Report maps. See our full description of the genetic data collected as part of the WebQTL project.

+
+
+
+
+
+

About Your Data

+
+

You can open a separate window giving the number of strains for each data set and sample data.

+

None of your submitted data is copied or stored by this system except during the actual processing of your submission. By the time the reply page displays in your browser, your submission has been cleared from this system.

+
+
+
+
+
+
+

Trait Submission Form

+
+
+

1. Choose Species and Group:

+
+
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+
+
+

2. Enter Trait Data:

+

File uploading isn't enabled yet, but is coming soon.

+
+
+ +
+
+
+

+ Paste or Type Multiple Values: You can enter data by pasting a series of numbers representing trait values into this area. + The values can be on one line separated by spaces or tabs, or they can be on separate lines. Include one value for each individual + or line. Use an "x" for missing values. If you have chosen a set of inbred strains, then your data will be displayed in a form in + which you can confirm and/or edit. If you enter a file name in the previous section, + any data that you paste here will be ignored. Check sample data for the correct format. +

+ +
+
+
+ + +
+
+
+

3. Enable Use of Trait Variance:

+
+ +
+
+
+

+ Name Your Trait: (optional) +

+ +
+
+
+ + +
+
+
+
+
+
+
+ +{%endblock%} + +{% block js %} + + +{% endblock %} -- cgit v1.2.3 From 442fc67ba1d66dd7931e20d161b40451b4e4b8c8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Apr 2021 13:17:34 +0300 Subject: wqflask: views: Add new function for selecting a dataset * wqflask/wqflask/views.py(edit_trait_page): New function. --- wqflask/wqflask/views.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 38c56b71..81ccaf7d 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -399,6 +399,16 @@ def submit_trait_form(): version=GN_VERSION) +@app.route("/edit_trait_form") +def edit_trait_page(): + species_and_groups = get_species_groups() + return render_template( + "edit_trait.html", + species_and_groups=species_and_groups, + gn_server_url=GN_SERVER_URL, + version=GN_VERSION) + + @app.route("/create_temp_trait", methods=('POST',)) def create_temp_trait(): logger.info(request.url) -- cgit v1.2.3 From 1e2e5b23ec30937f200778524f446515a5f83fb3 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Apr 2021 13:18:50 +0300 Subject: templates: edit_trait.html: Add comment headers The comment headers mark different sections. --- wqflask/wqflask/templates/edit_trait.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html index 3572b0a9..82d7120d 100644 --- a/wqflask/wqflask/templates/edit_trait.html +++ b/wqflask/wqflask/templates/edit_trait.html @@ -39,12 +39,14 @@
+
+
-- cgit v1.2.3 From ba8cd37b5cd7e038e31c879fbbc38939fa093acf Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Apr 2021 13:19:29 +0300 Subject: templates: edit_trait.html: Add extra fields for the form Add form elements for selecting group, type, and dataset. This mimics the home page. --- wqflask/wqflask/templates/edit_trait.html | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html index 82d7120d..4c9b3ba1 100644 --- a/wqflask/wqflask/templates/edit_trait.html +++ b/wqflask/wqflask/templates/edit_trait.html @@ -53,6 +53,20 @@
+ +
+ +
+ +
+
+ +
+ +
+ +
+
-- cgit v1.2.3 From 43ed9d7e5c22de2a457457e75b8153f6c2e1488c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Apr 2021 14:57:03 +0300 Subject: templates: edit_trait.html: Trim form to only contain selection menu --- wqflask/wqflask/templates/edit_trait.html | 45 ++----------------------------- 1 file changed, 2 insertions(+), 43 deletions(-) diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html index 4c9b3ba1..14f44698 100644 --- a/wqflask/wqflask/templates/edit_trait.html +++ b/wqflask/wqflask/templates/edit_trait.html @@ -30,14 +30,11 @@
-

Trait Submission Form

+

Edit Trait Form


-

1. Choose Species and Group:

+

Choose Dataset to Edit:


-
- -
@@ -68,44 +65,6 @@
-
-
-

2. Enter Trait Data:

-

File uploading isn't enabled yet, but is coming soon.

-
-
- -
-
-
-

- Paste or Type Multiple Values: You can enter data by pasting a series of numbers representing trait values into this area. - The values can be on one line separated by spaces or tabs, or they can be on separate lines. Include one value for each individual - or line. Use an "x" for missing values. If you have chosen a set of inbred strains, then your data will be displayed in a form in - which you can confirm and/or edit. If you enter a file name in the previous section, - any data that you paste here will be ignored. Check sample data for the correct format. -

- -
-
-
- - -
-
-
-

3. Enable Use of Trait Variance:

-
- -
-
-
-

- Name Your Trait: (optional) -

- -
-
-- cgit v1.2.3 From 99f8349b8cda4a3b17380c913207fc8395481653 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Apr 2021 15:25:18 +0300 Subject: wqflask: Add new js file for processing menu entries * wqflask/wqflask/static/new/javascript/dataset_select_menu_edit_trait.js: New file. Copy of dataset_select_menu_orig.js that will later modified... * wqflask/wqflask/templates/edit_trait.html: Use :point_up:. --- .../javascript/dataset_select_menu_edit_trait.js | 253 +++++++++++++++++++++ wqflask/wqflask/templates/edit_trait.html | 4 +- 2 files changed, 255 insertions(+), 2 deletions(-) create mode 100644 wqflask/wqflask/static/new/javascript/dataset_select_menu_edit_trait.js diff --git a/wqflask/wqflask/static/new/javascript/dataset_select_menu_edit_trait.js b/wqflask/wqflask/static/new/javascript/dataset_select_menu_edit_trait.js new file mode 100644 index 00000000..1d4a94d9 --- /dev/null +++ b/wqflask/wqflask/static/new/javascript/dataset_select_menu_edit_trait.js @@ -0,0 +1,253 @@ +var apply_default, check_search_term, dataset_info, group_info, make_default, open_window, populate_dataset, populate_group, populate_species, populate_type, process_json, redo_dropdown; +process_json = function(data) { + window.jdata = data; + populate_species(); + if ($('#type').length > 0) { //This is to determine if it's the index page or the submit_trait page (which only has species and group selection and no make default option) + return apply_default(); + } +}; + +$.ajax('/api/v_pre1/gen_dropdown', { + dataType: 'json', + success: process_json +}); + +populate_species = function() { + var species_list; + species_list = this.jdata.species; + redo_dropdown($('#species'), species_list); + return populate_group(); +}; +window.populate_species = populate_species; +populate_group = function() { + var group_list, species; + console.log("in populate group"); + species = $('#species').val(); + group_list = this.jdata.groups[species]; + for (_i = 0, _len = group_list.length; _i < (_len - 1); _i++) { + if (group_list[_i][0] == "BXD300"){ + group_list.splice(_i, 1) + } + } + redo_dropdown($('#group'), group_list); + if ($('#type').length > 0) { //This is to determine if it's the index page or the submit_trait page (which only has species and group selection and no make default option) + return populate_type(); + } +}; +window.populate_group = populate_group; +populate_type = function() { + var group, species, type_list; + console.log("in populate type"); + species = $('#species').val(); + group = $('#group').val(); + type_list = this.jdata.types[species][group]; + redo_dropdown($('#type'), type_list); + return populate_dataset(); +}; +window.populate_type = populate_type; +populate_dataset = function() { + var dataset_list, group, species, type; + console.log("in populate dataset"); + species = $('#species').val(); + group = $('#group').val(); + type = $('#type').val(); + console.log("sgt:", species, group, type); + dataset_list = this.jdata.datasets[species][group][type]; + console.log("pop_dataset:", dataset_list); + return redo_dropdown($('#dataset'), dataset_list); +}; +window.populate_dataset = populate_dataset; +redo_dropdown = function(dropdown, items) { + var item, _i, _len, _results; + console.log("in redo:", dropdown, items); + dropdown.empty(); + _results = []; + + if (dropdown.attr('id') == "group"){ + group_family_list = []; + for (_i = 0, _len = items.length; _i < _len; _i++) { + item = items[_i]; + group_family = item[2].toString().split(":")[1] + group_family_list.push([item[0], item[1], group_family]) + } + + current_family = "" + this_opt_group = null + for (_i = 0, _len = group_family_list.length; _i < _len; _i++) { + item = group_family_list[_i]; + if (item[2] != "None" && current_family == ""){ + current_family = item[2] + this_opt_group = $("") + this_opt_group.append($("") + this_opt_group.append($("") + this_opt_group.append($("") + this_opt_group.append($("
@@ -247,6 +247,7 @@ {% if geno_db_exists == "True" %}{% endif %} +

@@ -612,7 +613,7 @@ return $('#marker_regression_form').submit(); } - $('#export_mapping_results').click(export_mapping_results); + $('.export_mapping_results').click(export_mapping_results); $('#browser_tab').click(function() { $('#gn1_map_options').css("display", "none") -- cgit v1.2.3 From 1e11d0ea0fd68a7b16dfb208217a442ad95909e9 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 5 May 2021 18:18:20 +0000 Subject: Added a number of new metadata items to the mapping loading page, such as covariates and permutation/bootstrap numbers --- wqflask/wqflask/templates/loading.html | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 926f258d..69610f34 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -15,12 +15,31 @@ n = {{ start_vars.n_samples }}
Method = {% if start_vars.method == "gemma" %}GEMMA{% else %}{{ start_vars.method }}{% endif %} + {% if start_vars.num_perm | int > 0 %}
+ # Permutations = {{ start_vars.num_perm }} + {% endif %} + {% if start_vars.num_bootstrap | int > 0 %} +
+ # Bootstrap = {{ start_vars.num_bootstrap }} + {% endif %} {% if start_vars.transform != "" %} - transform = {{ start_vars.transform }}
+ transform = {{ start_vars.transform }} {% endif %} + {% if start_vars.maf != "" and start_vars.method != "reaper" %} +
MAF >= {{ start_vars.maf }} + {% endif %} + {% if start_vars.covariates != "" and start_vars.method != "reaper" %} +
+ {% set covariate_list = start_vars.covariates.split(",") %} + Trait Covariates: {% for covariate in covariate_list %}{% set this_covariate = covariate.split(":")[0] %}{{ this_covariate }}{% if not loop.last %}, {% endif %}{% endfor %} + {% endif %} + {% if start_vars.control_marker != "" and start_vars.do_control == "true" and start_vars.method != "gemma" %} +
+ Marker Covariate: {{ start_vars.control_marker }} + {% endif %} {% else %}

Loading {{ start_vars.tool_used }} Results...

{% endif %} -- cgit v1.2.3 From 6919762a2e0bce1afea0e979cb60a5c4b1dceacf Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 5 May 2021 18:19:41 +0000 Subject: Removed an unused item from mapping_input_list --- wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js | 2 +- wqflask/wqflask/templates/mapping_results.html | 2 +- wqflask/wqflask/views.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js index 53bcd1f6..09e9d024 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js +++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js @@ -143,7 +143,7 @@ $('input[name=display_all]').change((function(_this) { //ZS: This is a list of inputs to be passed to the loading page, since not all inputs on the trait page are relevant to mapping var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form_url', 'method', 'transform', 'trimmed_markers', 'selected_chr', 'chromosomes', 'mapping_scale', 'sample_vals', 'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars', 'num_bootstrap', 'bootCheck', 'bootstrap_results', - 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker', 'control_marker_db', 'do_control', 'genofile', + 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker', 'do_control', 'genofile', 'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', 'haplotypeAnalystCheck', 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'reaper_version', 'primary_samples'] diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index edc30164..f054506c 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -481,7 +481,7 @@ var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form_url', 'method', 'transform', 'trimmed_markers', 'selected_chr', 'chromosomes', 'mapping_scale', 'sample_vals', 'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars', 'num_bootstrap', 'bootCheck', 'bootstrap_results', - 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'color_scheme', 'manhattan_single_color', 'control_marker', 'control_marker_db', 'do_control', 'genofile', + 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'color_scheme', 'manhattan_single_color', 'control_marker', 'do_control', 'genofile', 'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', 'haplotypeAnalystCheck', 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'reaper_version', 'primary_samples', 'n_samples'] diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index f9b8f310..276d3019 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -794,7 +794,6 @@ def mapping_results_page(): 'color_scheme', 'manhattan_single_color', 'control_marker', - 'control_marker_db', 'do_control', 'genofile', 'genofile_string', -- cgit v1.2.3 From 1dd80ec2767c1de0028ab720c7ef4caae834a99c Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 5 May 2021 18:23:09 +0000 Subject: Added genotype file to mapping loading page --- wqflask/wqflask/templates/loading.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 69610f34..7344b918 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -15,6 +15,9 @@ n = {{ start_vars.n_samples }}
Method = {% if start_vars.method == "gemma" %}GEMMA{% else %}{{ start_vars.method }}{% endif %} +
+ {% set genofile_desc = start_vars.genofile.split(":")[1] %} + Genotype File = {{ genofile_desc }} {% if start_vars.num_perm | int > 0 %}
# Permutations = {{ start_vars.num_perm }} -- cgit v1.2.3 From a25ca626b7a6e1ca75d1fa0a830705e1df4d1e6c Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 5 May 2021 18:32:18 +0000 Subject: Added whether GEMMA is using LOCO to mapping loading page --- wqflask/wqflask/templates/loading.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 7344b918..124f0608 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -14,7 +14,7 @@
n = {{ start_vars.n_samples }}
- Method = {% if start_vars.method == "gemma" %}GEMMA{% else %}{{ start_vars.method }}{% endif %} + Method = {% if start_vars.method == "gemma" %}GEMMA {% if start_vars.use_loco == "True" %}using LOCO {% endif %}{% else %}{{ start_vars.method }}{% endif %}
{% set genofile_desc = start_vars.genofile.split(":")[1] %} Genotype File = {{ genofile_desc }} -- cgit v1.2.3 From 6202ef9c4016d08cecf75fd8217c9b53d18ed201 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 5 May 2021 15:58:41 +0300 Subject: wqflask: database: Remove logging --- wqflask/wqflask/database.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py index e743c4b3..42fa1594 100644 --- a/wqflask/wqflask/database.py +++ b/wqflask/wqflask/database.py @@ -5,9 +5,6 @@ from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.ext.declarative import declarative_base from utility.tools import SQL_URI -import utility.logger -logger = utility.logger.getLogger(__name__) - engine = create_engine(SQL_URI, encoding="latin1") @@ -23,10 +20,8 @@ def init_db(): # they will be registered properly on the metadata. Otherwise # you will have to import them first before calling init_db() #import yourapplication.models - logger.info("Initializing database connection") import wqflask.model Base.metadata.create_all(bind=engine) - logger.info("Done creating all model metadata") init_db() -- cgit v1.2.3 From 773c758efb1a48f97f85ac1e43e91355f67bde72 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 5 May 2021 16:25:41 +0300 Subject: wqflask: user_manager: Delete usermanager.py and it's references This module is not used anywhere. --- test/requests/test_forgot_password.py | 4 +- test/requests/test_login_local.py | 3 - wqflask/wqflask/user_manager.py | 1095 --------------------------------- 3 files changed, 1 insertion(+), 1101 deletions(-) delete mode 100644 wqflask/wqflask/user_manager.py diff --git a/test/requests/test_forgot_password.py b/test/requests/test_forgot_password.py index 2bf34c5c..346524bc 100644 --- a/test/requests/test_forgot_password.py +++ b/test/requests/test_forgot_password.py @@ -1,5 +1,4 @@ import requests -from wqflask import user_manager from utility.elasticsearch_tools import get_user_by_unique_column from parameterized import parameterized from parametrized_test import ParametrizedTest @@ -27,8 +26,7 @@ class TestForgotPassword(ParametrizedTest): "password": "test_password", "password_confirm": "test_password" } - user_manager.basic_info = lambda : { "basic_info": "basic" } - user_manager.RegisterUser(data) + def testWithoutEmail(self): data = {"email_address": ""} diff --git a/test/requests/test_login_local.py b/test/requests/test_login_local.py index 808649ca..6691d135 100644 --- a/test/requests/test_login_local.py +++ b/test/requests/test_login_local.py @@ -1,5 +1,4 @@ import requests -from wqflask import user_manager from parameterized import parameterized from parametrized_test import ParametrizedTest @@ -19,8 +18,6 @@ class TestLoginLocal(ParametrizedTest): "password": "test_password", "password_confirm": "test_password" } - user_manager.basic_info = lambda : { "basic_info": "basic" } - user_manager.RegisterUser(data) @parameterized.expand([ diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py deleted file mode 100644 index cf84ea73..00000000 --- a/wqflask/wqflask/user_manager.py +++ /dev/null @@ -1,1095 +0,0 @@ -import os -import hashlib -import datetime -import time -import uuid -import hmac -import base64 -import redis # used for collections -import simplejson as json -import requests - -from base.data_set import create_datasets_list - -from flask import g -from flask import render_template -from flask import url_for -from flask import request -from flask import make_response -from flask import redirect -from flask import flash - -from wqflask import app -from wqflask import pbkdf2 # password hashing -from wqflask.database import db_session -from wqflask import model - -from smtplib import SMTP - -from pprint import pformat as pf - -from utility import Bunch -from utility import Struct -from utility.logger import getLogger - -from utility.redis_tools import get_user_id -from utility.redis_tools import get_user_by_unique_column -from utility.redis_tools import set_user_attribute -from utility.redis_tools import save_user -from utility.redis_tools import save_verification_code -from utility.redis_tools import check_verification_code -from utility.redis_tools import get_user_collections -from utility.redis_tools import save_collections - -from utility.tools import SMTP_CONNECT -from utility.tools import SMTP_USERNAME -from utility.tools import SMTP_PASSWORD - - -logger = getLogger(__name__) - - -Redis = redis.StrictRedis() - -THREE_DAYS = 60 * 60 * 24 * 3 - - -def timestamp(): - return datetime.datetime.utcnow().isoformat() - - -class AnonUser: - """Anonymous user handling""" - cookie_name = 'anon_user_v1' - - def __init__(self): - self.cookie = request.cookies.get(self.cookie_name) - if self.cookie: - logger.debug("ANON COOKIE ALREADY EXISTS") - self.anon_id = verify_cookie(self.cookie) - else: - logger.debug("CREATING NEW ANON COOKIE") - self.anon_id, self.cookie = create_signed_cookie() - - self.key = "anon_collection:v1:{}".format(self.anon_id) - - def add_collection(self, new_collection): - collection_dict = dict(name=new_collection.name, - created_timestamp=datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), - changed_timestamp=datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), - num_members=new_collection.num_members, - members=new_collection.get_members()) - - Redis.set(self.key, json.dumps(collection_dict)) - Redis.expire(self.key, 60 * 60 * 24 * 365) - - def delete_collection(self, collection_name): - existing_collections = self.get_collections() - updated_collections = [] - for i, collection in enumerate(existing_collections): - if collection['name'] == collection_name: - continue - else: - this_collection = {} - this_collection['id'] = collection['id'] - this_collection['name'] = collection['name'] - this_collection['created_timestamp'] = collection['created_timestamp'].strftime( - '%b %d %Y %I:%M%p') - this_collection['changed_timestamp'] = collection['changed_timestamp'].strftime( - '%b %d %Y %I:%M%p') - this_collection['num_members'] = collection['num_members'] - this_collection['members'] = collection['members'] - updated_collections.append(this_collection) - - Redis.set(self.key, json.dumps(updated_collections)) - - def get_collections(self): - json_collections = Redis.get(self.key) - if json_collections == None or json_collections == "None": - return [] - else: - collections = json.loads(json_collections) - for collection in collections: - collection['created_timestamp'] = datetime.datetime.strptime( - collection['created_timestamp'], '%b %d %Y %I:%M%p') - collection['changed_timestamp'] = datetime.datetime.strptime( - collection['changed_timestamp'], '%b %d %Y %I:%M%p') - - collections = sorted( - collections, key=lambda i: i['changed_timestamp'], reverse=True) - return collections - - def import_traits_to_user(self): - result = Redis.get(self.key) - collections_list = json.loads(result if result else "[]") - for collection in collections_list: - collection_exists = g.user_session.get_collection_by_name( - collection['name']) - if collection_exists: - continue - else: - g.user_session.add_collection( - collection['name'], collection['members']) - - def display_num_collections(self): - """ - Returns the number of collections or a blank string if there are zero. - - Because this is so unimportant...we wrap the whole thing in a try/expect...last thing we - want is a webpage not to be displayed because of an error here - - Importand TODO: use redis to cache this, don't want to be constantly computing it - """ - try: - num = len(self.get_collections()) - if num > 0: - return num - else: - return "" - except Exception as why: - print("Couldn't display_num_collections:", why) - return "" - - -def verify_cookie(cookie): - the_uuid, separator, the_signature = cookie.partition(':') - assert len(the_uuid) == 36, "Is session_id a uuid?" - assert separator == ":", "Expected a : here" - assert the_signature == actual_hmac_creation( - the_uuid), "Uh-oh, someone tampering with the cookie?" - return the_uuid - - -def create_signed_cookie(): - the_uuid = str(uuid.uuid4()) - signature = actual_hmac_creation(the_uuid) - uuid_signed = the_uuid + ":" + signature - logger.debug("uuid_signed:", uuid_signed) - return the_uuid, uuid_signed - - -class UserSession: - """Logged in user handling""" - - cookie_name = 'session_id_v1' - - def __init__(self): - cookie = request.cookies.get(self.cookie_name) - if not cookie: - logger.debug("NO USER COOKIE") - self.logged_in = False - return - else: - session_id = verify_cookie(cookie) - - self.redis_key = self.cookie_name + ":" + session_id - logger.debug("self.redis_key is:", self.redis_key) - self.session_id = session_id - self.record = Redis.hgetall(self.redis_key) - - if not self.record: - # This will occur, for example, when the browser has been left open over a long - # weekend and the site hasn't been visited by the user - self.logged_in = False - - # Grrr...this won't work because of the way flask handles cookies - # Delete the cookie - #response = make_response(redirect(url_for('login'))) - #response.set_cookie(self.cookie_name, '', expires=0) - # flash( - # "Due to inactivity your session has expired. If you'd like please login again.") - # return response - return - - if Redis.ttl(self.redis_key) < THREE_DAYS: - # (Almost) everytime the user does something we extend the session_id in Redis... - logger.debug("Extending ttl...") - Redis.expire(self.redis_key, THREE_DAYS) - - logger.debug("record is:", self.record) - self.logged_in = True - - @property - def user_id(self): - """Shortcut to the user_id""" - if 'user_id' in self.record: - return self.record['user_id'] - else: - return '' - - @property - def redis_user_id(self): - """User id from ElasticSearch (need to check if this is the same as the id stored in self.records)""" - - user_email = self.record['user_email_address'] - - # ZS: Get user's collections if they exist - user_id = None - user_id = get_user_id("email_address", user_email) - return user_id - - @property - def user_name(self): - """Shortcut to the user_name""" - if 'user_name' in self.record: - return self.record['user_name'] - else: - return '' - - @property - def user_collections(self): - """List of user's collections""" - - # ZS: Get user's collections if they exist - collections = get_user_collections(self.redis_user_id) - return collections - - @property - def num_collections(self): - """Number of user's collections""" - - return len(self.user_collections) - - def add_collection(self, collection_name, traits): - """Add collection into ElasticSearch""" - - collection_dict = {'id': str(uuid.uuid4()), - 'name': collection_name, - 'created_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), - 'changed_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), - 'num_members': len(traits), - 'members': list(traits)} - - current_collections = self.user_collections - current_collections.append(collection_dict) - self.update_collections(current_collections) - - return collection_dict['id'] - - def delete_collection(self, collection_id): - """Remove collection with given ID""" - - updated_collections = [] - for collection in self.user_collections: - if collection['id'] == collection_id: - continue - else: - updated_collections.append(collection) - - self.update_collections(updated_collections) - - return collection['name'] - - def add_traits_to_collection(self, collection_id, traits_to_add): - """Add specified traits to a collection""" - - this_collection = self.get_collection_by_id(collection_id) - - updated_collection = this_collection - updated_traits = this_collection['members'] + traits_to_add - - updated_collection['members'] = updated_traits - updated_collection['num_members'] = len(updated_traits) - updated_collection['changed_timestamp'] = datetime.datetime.utcnow().strftime( - '%b %d %Y %I:%M%p') - - updated_collections = [] - for collection in self.user_collections: - if collection['id'] == collection_id: - updated_collections.append(updated_collection) - else: - updated_collections.append(collection) - - self.update_collections(updated_collections) - - def remove_traits_from_collection(self, collection_id, traits_to_remove): - """Remove specified traits from a collection""" - - this_collection = self.get_collection_by_id(collection_id) - - updated_collection = this_collection - updated_traits = [] - for trait in this_collection['members']: - if trait in traits_to_remove: - continue - else: - updated_traits.append(trait) - - updated_collection['members'] = updated_traits - updated_collection['num_members'] = len(updated_traits) - updated_collection['changed_timestamp'] = datetime.datetime.utcnow().strftime( - '%b %d %Y %I:%M%p') - - updated_collections = [] - for collection in self.user_collections: - if collection['id'] == collection_id: - updated_collections.append(updated_collection) - else: - updated_collections.append(collection) - - self.update_collections(updated_collections) - - return updated_traits - - def get_collection_by_id(self, collection_id): - for collection in self.user_collections: - if collection['id'] == collection_id: - return collection - - def get_collection_by_name(self, collection_name): - for collection in self.user_collections: - if collection['name'] == collection_name: - return collection - - return None - - def update_collections(self, updated_collections): - collection_body = json.dumps(updated_collections) - - save_collections(self.redis_user_id, collection_body) - - def delete_session(self): - # And more importantly delete the redis record - Redis.delete(self.cookie_name) - logger.debug("At end of delete_session") - - -@app.before_request -def get_cookie(): - logger.info("@app.before_request get cookie") - g.user_session = UserSession() - g.cookie_session = AnonUser() - -# @app.after_request - - -def set_cookie(response): - if not request.cookies.get(g.cookie_session.cookie_name): - response.set_cookie(g.cookie_session.cookie_name, - g.cookie_session.cookie) - return response - - -class UsersManager: - def __init__(self): - self.users = model.User.query.all() - logger.debug("Users are:", self.users) - - -class UserManager: - def __init__(self, kw): - self.user_id = kw['user_id'] - logger.debug("In UserManager locals are:", pf(locals())) - #self.user = model.User.get(user_id) - #logger.debug("user is:", user) - self.user = model.User.query.get(self.user_id) - logger.debug("user is:", self.user) - datasets = create_datasets_list() - for dataset in datasets: - if not dataset.check_confidentiality(): - continue - logger.debug("\n Name:", dataset.name) - logger.debug(" Type:", dataset.type) - logger.debug(" ID:", dataset.id) - logger.debug(" Confidential:", dataset.check_confidentiality()) - #logger.debug(" ---> self.datasets:", self.datasets) - - -class RegisterUser: - def __init__(self, kw): - self.thank_you_mode = False - self.errors = [] - self.user = Bunch() - - self.user.email_address = kw.get( - 'email_address', '').encode("utf-8").strip() - if not (5 <= len(self.user.email_address) <= 50): - self.errors.append( - 'Email Address needs to be between 5 and 50 characters.') - else: - email_exists = get_user_by_unique_column( - "email_address", self.user.email_address) - #email_exists = get_user_by_unique_column(es, "email_address", self.user.email_address) - if email_exists: - self.errors.append('User already exists with that email') - - self.user.full_name = kw.get('full_name', '').encode("utf-8").strip() - if not (5 <= len(self.user.full_name) <= 50): - self.errors.append( - 'Full Name needs to be between 5 and 50 characters.') - - self.user.organization = kw.get( - 'organization', '').encode("utf-8").strip() - if self.user.organization and not (5 <= len(self.user.organization) <= 50): - self.errors.append( - 'Organization needs to be empty or between 5 and 50 characters.') - - password = str(kw.get('password', '')) - if not (6 <= len(password)): - self.errors.append('Password needs to be at least 6 characters.') - - if kw.get('password_confirm') != password: - self.errors.append("Passwords don't match.") - - if self.errors: - return - - logger.debug("No errors!") - - set_password(password, self.user) - self.user.user_id = str(uuid.uuid4()) - self.user.confirmed = 1 - - self.user.registration_info = json.dumps(basic_info(), sort_keys=True) - save_user(self.user.__dict__, self.user.user_id) - - -def set_password(password, user): - pwfields = Bunch() - - pwfields.algorithm = "pbkdf2" - pwfields.hashfunc = "sha256" - #hashfunc = getattr(hashlib, pwfields.hashfunc) - - # Encoding it to base64 makes storing it in json much easier - pwfields.salt = base64.b64encode(os.urandom(32)) - - # https://forums.lastpass.com/viewtopic.php?t=84104 - pwfields.iterations = 100000 - pwfields.keylength = 32 - - pwfields.created_ts = timestamp() - # One more check on password length - assert len(password) >= 6, "Password shouldn't be so short here" - - logger.debug("pwfields:", vars(pwfields)) - logger.debug("locals:", locals()) - - enc_password = Password(password, - pwfields.salt, - pwfields.iterations, - pwfields.keylength, - pwfields.hashfunc) - - pwfields.password = enc_password.password - pwfields.encrypt_time = enc_password.encrypt_time - - user.password = json.dumps(pwfields.__dict__, - sort_keys=True, - ) - - -class VerificationEmail: - template_name = "email/verification.txt" - key_prefix = "verification_code" - subject = "GeneNetwork email verification" - - def __init__(self, user): - verification_code = str(uuid.uuid4()) - key = self.key_prefix + ":" + verification_code - - data = json.dumps(dict(id=user.user_id, - timestamp=timestamp()) - ) - - Redis.set(key, data) - #two_days = 60 * 60 * 24 * 2 - Redis.expire(key, THREE_DAYS) - to = user.email_address - subject = self.subject - body = render_template(self.template_name, - verification_code=verification_code) - send_email(to, subject, body) - - -class ForgotPasswordEmail(VerificationEmail): - template_name = "email/forgot_password.txt" - key_prefix = "forgot_password_code" - subject = "GeneNetwork password reset" - fromaddr = "no-reply@genenetwork.org" - - def __init__(self, toaddr): - from email.MIMEMultipart import MIMEMultipart - from email.MIMEText import MIMEText - verification_code = str(uuid.uuid4()) - key = self.key_prefix + ":" + verification_code - - data = { - "verification_code": verification_code, - "email_address": toaddr, - "timestamp": timestamp() - } - - save_verification_code(toaddr, verification_code) - - subject = self.subject - body = render_template( - self.template_name, - verification_code=verification_code) - - msg = MIMEMultipart() - msg["To"] = toaddr - msg["Subject"] = self.subject - msg["From"] = self.fromaddr - msg.attach(MIMEText(body, "plain")) - - send_email(toaddr, msg.as_string()) - - -class Password: - def __init__(self, unencrypted_password, salt, iterations, keylength, hashfunc): - hashfunc = getattr(hashlib, hashfunc) - logger.debug("hashfunc is:", hashfunc) - # On our computer it takes around 1.4 seconds in 2013 - start_time = time.time() - salt = base64.b64decode(salt) - self.password = pbkdf2.pbkdf2_hex(str(unencrypted_password), - salt, iterations, keylength, hashfunc) - self.encrypt_time = round(time.time() - start_time, 3) - logger.debug("Creating password took:", self.encrypt_time) - - -def basic_info(): - return dict(timestamp=timestamp(), - ip_address=request.remote_addr, - user_agent=request.headers.get('User-Agent')) - -# @app.route("/manage/verify_email") - - -def verify_email(): - user = DecodeUser(VerificationEmail.key_prefix).user - user.confirmed = json.dumps(basic_info(), sort_keys=True) - db_session.commit() - - # As long as they have access to the email account - # We might as well log them in - - session_id_signed = LoginUser().successful_login(user) - response = make_response(render_template("new_security/thank_you.html")) - response.set_cookie(UserSession.cookie_name, session_id_signed) - return response - -# @app.route("/n/password_reset", methods=['GET']) - - -def password_reset(): - """Entry point after user clicks link in E-mail""" - logger.debug("in password_reset request.url is:", request.url) - # We do this mainly just to assert that it's in proper form for displaying next page - # Really not necessary but doesn't hurt - # user_encode = DecodeUser(ForgotPasswordEmail.key_prefix).reencode_standalone() - verification_code = request.args.get('code') - hmac = request.args.get('hm') - - if verification_code: - user_email = check_verification_code(verification_code) - if user_email: - user_details = get_user_by_unique_column( - 'email_address', user_email) - if user_details: - return render_template( - "new_security/password_reset.html", user_encode=user_details["user_id"]) - else: - flash("Invalid code: User no longer exists!", "error") - else: - flash( - "Invalid code: Password reset code does not exist or might have expired!", "error") - else: - return redirect(url_for("login")) - -# @app.route("/n/password_reset_step2", methods=('POST',)) - - -def password_reset_step2(): - """Handle confirmation E-mail for password reset""" - logger.debug("in password_reset request.url is:", request.url) - - errors = [] - user_id = request.form['user_encode'] - - logger.debug("locals are:", locals()) - - user = Bunch() - password = request.form['password'] - set_password(password, user) - - set_user_attribute(user_id, "password", user.__dict__.get("password")) - - flash("Password changed successfully. You can now sign in.", "alert-info") - response = make_response(redirect(url_for('login'))) - - return response - - -class DecodeUser: - - def __init__(self, code_prefix): - verify_url_hmac(request.url) - - #params = urlparse.parse_qs(url) - - self.verification_code = request.args['code'] - self.user = self.actual_get_user(code_prefix, self.verification_code) - - def reencode_standalone(self): - hmac = actual_hmac_creation(self.verification_code) - return self.verification_code + ":" + hmac - - @staticmethod - def actual_get_user(code_prefix, verification_code): - data = Redis.get(code_prefix + ":" + verification_code) - logger.debug("in get_coded_user, data is:", data) - data = json.loads(data) - logger.debug("data is:", data) - return model.User.query.get(data['id']) - -# @app.route("/n/login", methods=('GET', 'POST')) - - -def login(): - lu = LoginUser() - login_type = request.args.get("type") - if login_type: - uid = request.args.get("uid") - return lu.oauth2_login(login_type, uid) - else: - return lu.standard_login() - -# @app.route("/n/login/github_oauth2", methods=('GET', 'POST')) - - -def github_oauth2(): - from utility.tools import GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET - code = request.args.get("code") - data = { - "client_id": GITHUB_CLIENT_ID, - "client_secret": GITHUB_CLIENT_SECRET, - "code": code - } - result = requests.post( - "https://github.com/login/oauth/access_token", json=data) - result_dict = {arr[0]: arr[1] for arr in [tok.split( - "=") for tok in [token.encode("utf-8") for token in result.text.split("&")]]} - - github_user = get_github_user_details(result_dict["access_token"]) - - user_details = get_user_by_unique_column("github_id", github_user["id"]) - if user_details == None: - user_details = { - "user_id": str(uuid.uuid4()), "name": github_user["name"].encode("utf-8"), "github_id": github_user["id"], "user_url": github_user["html_url"].encode("utf-8"), "login_type": "github", "organization": "", "active": 1, "confirmed": 1 - } - save_user(user_details, user_details["user_id"]) - - url = "/n/login?type=github&uid=" + user_details["user_id"] - return redirect(url) - -# @app.route("/n/login/orcid_oauth2", methods=('GET', 'POST')) - - -def orcid_oauth2(): - from uuid import uuid4 - from utility.tools import ORCID_CLIENT_ID, ORCID_CLIENT_SECRET, ORCID_TOKEN_URL, ORCID_AUTH_URL - code = request.args.get("code") - error = request.args.get("error") - url = "/n/login" - if code: - data = { - "client_id": ORCID_CLIENT_ID, "client_secret": ORCID_CLIENT_SECRET, "grant_type": "authorization_code", "code": code - } - result = requests.post(ORCID_TOKEN_URL, data=data) - result_dict = json.loads(result.text.encode("utf-8")) - - user_details = get_user_by_unique_column("orcid", result_dict["orcid"]) - if user_details == None: - user_details = { - "user_id": str(uuid4()), "name": result_dict["name"], "orcid": result_dict["orcid"], "user_url": "%s/%s" % ( - "/".join(ORCID_AUTH_URL.split("/")[:-2]), - result_dict["orcid"]), "login_type": "orcid", "organization": "", "active": 1, "confirmed": 1 - } - save_user(user_details, user_details["user_id"]) - - url = "/n/login?type=orcid&uid=" + user_details["user_id"] - else: - flash("There was an error getting code from ORCID") - return redirect(url) - - -def get_github_user_details(access_token): - from utility.tools import GITHUB_API_URL - result = requests.get(GITHUB_API_URL, params={ - "access_token": access_token}) - return result.json() - - -class LoginUser: - remember_time = 60 * 60 * 24 * 30 # One month in seconds - - def __init__(self): - self.remember_me = False - self.logged_in = False - - def oauth2_login(self, login_type, user_id): - """Login via an OAuth2 provider""" - - user_details = get_user_by_unique_column("user_id", user_id) - if user_details: - user = model.User() - user.id = user_details["user_id"] if user_details["user_id"] == None else "N/A" - user.full_name = user_details["name"] - user.login_type = user_details["login_type"] - return self.actual_login(user) - else: - flash("Error logging in via OAuth2") - return make_response(redirect(url_for('login'))) - - def standard_login(self): - """Login through the normal form""" - params = request.form if request.form else request.args - logger.debug("in login params are:", params) - - if not params: - from utility.tools import GITHUB_AUTH_URL, GITHUB_CLIENT_ID, ORCID_AUTH_URL, ORCID_CLIENT_ID - external_login = {} - if GITHUB_AUTH_URL and GITHUB_CLIENT_ID != 'UNKNOWN': - external_login["github"] = GITHUB_AUTH_URL - if ORCID_AUTH_URL and ORCID_CLIENT_ID != 'UNKNOWN': - external_login["orcid"] = ORCID_AUTH_URL - - return render_template( - "new_security/login_user.html", external_login=external_login, redis_is_available=is_redis_available()) - else: - user_details = get_user_by_unique_column( - "email_address", params["email_address"]) - #user_details = get_user_by_unique_column(es, "email_address", params["email_address"]) - user = None - valid = None - if user_details: - user = model.User() - for key in user_details: - user.__dict__[key] = user_details[key] - valid = False - - submitted_password = params['password'] - pwfields = Struct(json.loads(user.password)) - encrypted = Password( - submitted_password, - pwfields.salt, - pwfields.iterations, - pwfields.keylength, - pwfields.hashfunc) - logger.debug("\n\nComparing:\n{}\n{}\n".format( - encrypted.password, pwfields.password)) - valid = pbkdf2.safe_str_cmp( - encrypted.password, pwfields.password) - logger.debug("valid is:", valid) - - if valid and not user.confirmed: - VerificationEmail(user) - return render_template("new_security/verification_still_needed.html", - subject=VerificationEmail.subject) - if valid: - if params.get('remember'): - logger.debug("I will remember you") - self.remember_me = True - - if 'import_collections' in params: - import_col = "true" - else: - import_col = "false" - - # g.cookie_session.import_traits_to_user() - - self.logged_in = True - - return self.actual_login(user, import_collections=import_col) - - else: - if user: - self.unsuccessful_login(user) - flash("Invalid email-address or password. Please try again.", - "alert-danger") - response = make_response(redirect(url_for('login'))) - - return response - - def actual_login(self, user, assumed_by=None, import_collections=None): - """The meat of the logging in process""" - session_id_signed = self.successful_login(user, assumed_by) - flash("Thank you for logging in {}.".format( - user.full_name), "alert-success") - response = make_response( - redirect(url_for('index_page', import_collections=import_collections))) - if self.remember_me: - max_age = self.remember_time - else: - max_age = None - - response.set_cookie(UserSession.cookie_name, - session_id_signed, max_age=max_age) - return response - - def successful_login(self, user, assumed_by=None): - login_rec = model.Login(user) - login_rec.successful = True - login_rec.session_id = str(uuid.uuid4()) - login_rec.assumed_by = assumed_by - #session_id = "session_id:{}".format(login_rec.session_id) - session_id_signature = actual_hmac_creation(login_rec.session_id) - session_id_signed = login_rec.session_id + ":" + session_id_signature - logger.debug("session_id_signed:", session_id_signed) - - if not user.id: - user.id = '' - - session = dict(login_time=time.time(), - user_id=user.id, - user_name=user.full_name, - user_email_address=user.email_address) - - key = UserSession.cookie_name + ":" + login_rec.session_id - logger.debug("Key when signing:", key) - Redis.hmset(key, session) - if self.remember_me: - expire_time = self.remember_time - else: - expire_time = THREE_DAYS - Redis.expire(key, expire_time) - - return session_id_signed - - def unsuccessful_login(self, user): - login_rec = model.Login(user) - login_rec.successful = False - db_session.add(login_rec) - db_session.commit() - -# @app.route("/n/logout") - - -def logout(): - logger.debug("Logging out...") - UserSession().delete_session() - flash("You are now logged out. We hope you come back soon!") - response = make_response(redirect(url_for('index_page'))) - # Delete the cookie - response.set_cookie(UserSession.cookie_name, '', expires=0) - return response - - -# @app.route("/n/forgot_password", methods=['GET']) -def forgot_password(): - """Entry point for forgotten password""" - print("ARGS: ", request.args) - errors = {"no-email": request.args.get("no-email")} - print("ERRORS: ", errors) - return render_template("new_security/forgot_password.html", errors=errors) - -# @app.route("/n/forgot_password_submit", methods=('POST',)) - - -def forgot_password_submit(): - """When a forgotten password form is submitted we get here""" - params = request.form - email_address = params['email_address'] - next_page = None - if email_address != "": - logger.debug("Wants to send password E-mail to ", email_address) - user_details = get_user_by_unique_column( - "email_address", email_address) - if user_details: - ForgotPasswordEmail(user_details["email_address"]) - return render_template("new_security/forgot_password_step2.html", - subject=ForgotPasswordEmail.subject) - else: - flash("The e-mail entered is not associated with an account.", - "alert-danger") - return redirect(url_for("forgot_password")) - - else: - flash("You MUST provide an email", "alert-danger") - return redirect(url_for("forgot_password")) - - -@app.errorhandler(401) -def unauthorized(error): - return redirect(url_for('login')) - - -def is_redis_available(): - try: - Redis.ping() - except: - return False - return True - -### -# ZS: The following 6 functions require the old MySQL User accounts; I'm leaving them commented out just in case we decide to reimplement them using ElasticSearch -### -# def super_only(): -# try: -# superuser = g.user_session.user_ob.superuser -# except AttributeError: -# superuser = False -# if not superuser: -# flash("You must be a superuser to access that page.", "alert-error") -# abort(401) - -# @app.route("/manage/users") -# def manage_users(): -# super_only() -# template_vars = UsersManager() -# return render_template("admin/user_manager.html", **template_vars.__dict__) - -# @app.route("/manage/user") -# def manage_user(): -# super_only() -# template_vars = UserManager(request.args) -# return render_template("admin/ind_user_manager.html", **template_vars.__dict__) - -# @app.route("/manage/groups") -# def manage_groups(): -# super_only() -# template_vars = GroupsManager(request.args) -# return render_template("admin/group_manager.html", **template_vars.__dict__) - -# @app.route("/manage/make_superuser") -# def make_superuser(): -# super_only() -# params = request.args -# user_id = params['user_id'] -# user = model.User.query.get(user_id) -# superuser_info = basic_info() -# superuser_info['crowned_by'] = g.user_session.user_id -# user.superuser = json.dumps(superuser_info, sort_keys=True) -# db_session.commit() -# flash("We've made {} a superuser!".format(user.name_and_org)) -# return redirect(url_for("manage_users")) - -# @app.route("/manage/assume_identity") -# def assume_identity(): -# super_only() -# params = request.args -# user_id = params['user_id'] -# user = model.User.query.get(user_id) -# assumed_by = g.user_session.user_id -# return LoginUser().actual_login(user, assumed_by=assumed_by) - - -# @app.route("/n/register", methods=('GET', 'POST')) -def register(): - params = None - errors = None - - params = request.form if request.form else request.args - params = params.to_dict(flat=True) - - if params: - logger.debug("Attempting to register the user...") - result = RegisterUser(params) - errors = result.errors - - if len(errors) == 0: - flash( - "Registration successful. You may login with your new account", "alert-info") - return redirect(url_for("login")) - - return render_template("new_security/register_user.html", values=params, errors=errors) - - -################################# Sign and unsign ##################################### - -def url_for_hmac(endpoint, **values): - """Like url_for but adds an hmac at the end to insure the url hasn't been tampered with""" - - url = url_for(endpoint, **values) - - hm = actual_hmac_creation(url) - if '?' in url: - combiner = "&" - else: - combiner = "?" - return url + combiner + "hm=" + hm - - -def data_hmac(stringy): - """Takes arbitray data string and appends :hmac so we know data hasn't been tampered with""" - return stringy + ":" + actual_hmac_creation(stringy) - - -def verify_url_hmac(url): - """Pass in a url that was created with url_hmac and this assures it hasn't been tampered with""" - logger.debug("url passed in to verify is:", url) - # Verify parts are correct at the end - we expect to see &hm= or ?hm= followed by an hmac - assert url[-23:-20] == "hm=", "Unexpected url (stage 1)" - assert url[-24] in ["?", "&"], "Unexpected url (stage 2)" - hmac = url[-20:] - url = url[:-24] # Url without any of the hmac stuff - - #logger.debug("before urlsplit, url is:", url) - #url = divide_up_url(url)[1] - #logger.debug("after urlsplit, url is:", url) - - hm = actual_hmac_creation(url) - - assert hm == hmac, "Unexpected url (stage 3)" - - -def actual_hmac_creation(stringy): - """Helper function to create the actual hmac""" - - secret = app.config['SECRET_HMAC_CODE'] - - hmaced = hmac.new(secret, stringy, hashlib.sha1) - hm = hmaced.hexdigest() - # "Conventional wisdom is that you don't lose much in terms of security if you throw away up to half of the output." - # http://www.w3.org/QA/2009/07/hmac_truncation_in_xml_signatu.html - hm = hm[:20] - return hm - - -app.jinja_env.globals.update(url_for_hmac=url_for_hmac, - data_hmac=data_hmac) - -####################################################################################### - -# def send_email(to, subject, body): -# msg = json.dumps(dict(From="no-reply@genenetwork.org", -# To=to, -# Subject=subject, -# Body=body)) -# Redis.rpush("mail_queue", msg) - - -def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"): - """Send an E-mail through SMTP_CONNECT host. If SMTP_USERNAME is not - 'UNKNOWN' TLS is used - - """ - if SMTP_USERNAME == 'UNKNOWN': - logger.debug("SMTP: connecting with host " + SMTP_CONNECT) - server = SMTP(SMTP_CONNECT) - server.sendmail(fromaddr, toaddr, msg) - else: - logger.debug("SMTP: connecting TLS with host " + SMTP_CONNECT) - server = SMTP(SMTP_CONNECT) - server.starttls() - logger.debug("SMTP: login with user " + SMTP_USERNAME) - server.login(SMTP_USERNAME, SMTP_PASSWORD) - logger.debug("SMTP: " + fromaddr) - logger.debug("SMTP: " + toaddr) - logger.debug("SMTP: " + msg) - server.sendmail(fromaddr, toaddr, msg) - server.quit() - logger.info("Successfully sent email to " + toaddr) - - -class GroupsManager: - def __init__(self, kw): - self.datasets = create_datasets_list() - - -class RolesManager: - def __init__(self): - self.roles = model.Role.query.all() - logger.debug("Roles are:", self.roles) -- cgit v1.2.3 From 5c4da47e6f4c7c61666b80dc5133aa584db38e5d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 5 May 2021 16:26:09 +0300 Subject: Delete model.py and it's references * wqflask/wqflask/database.py (init_db): Remove references to "wqflask.model" * wqflask/wqflask/model.py: Delete it. --- wqflask/wqflask/database.py | 5 -- wqflask/wqflask/model.py | 182 -------------------------------------------- 2 files changed, 187 deletions(-) delete mode 100644 wqflask/wqflask/model.py diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py index 42fa1594..38c37d2c 100644 --- a/wqflask/wqflask/database.py +++ b/wqflask/wqflask/database.py @@ -16,11 +16,6 @@ Base.query = db_session.query_property() def init_db(): - # import all modules here that might define models so that - # they will be registered properly on the metadata. Otherwise - # you will have to import them first before calling init_db() - #import yourapplication.models - import wqflask.model Base.metadata.create_all(bind=engine) diff --git a/wqflask/wqflask/model.py b/wqflask/wqflask/model.py deleted file mode 100644 index a222b87c..00000000 --- a/wqflask/wqflask/model.py +++ /dev/null @@ -1,182 +0,0 @@ -import uuid -import datetime - -import simplejson as json - -from flask import request - -from wqflask import app - -import sqlalchemy -from sqlalchemy import (Column, ForeignKey, Unicode, Boolean, DateTime, - Text, Index) -from sqlalchemy.orm import relationship - -from wqflask.database import Base, init_db - - -class User(Base): - __tablename__ = "user" - id = Column(Unicode(36), primary_key=True, - default=lambda: str(uuid.uuid4())) - email_address = Column(Unicode(50), unique=True, nullable=False) - - # Todo: Turn on strict mode for Mysql - password = Column(Text, nullable=False) - - full_name = Column(Unicode(50)) - organization = Column(Unicode(50)) - - active = Column(Boolean(), nullable=False, default=True) - - # json detailing when they were registered, etc. - registration_info = Column(Text) - - confirmed = Column(Text) # json detailing when they confirmed, etc. - - # json detailing when they became a superuser, otherwise empty - superuser = Column(Text) - # if not superuser - - logins = relationship("Login", - order_by="desc(Login.timestamp)", - lazy='dynamic', # Necessary for filter in login_count - foreign_keys="Login.user", - ) - - user_collections = relationship("UserCollection", - order_by="asc(UserCollection.name)", - lazy='dynamic', - ) - - def display_num_collections(self): - """ - Returns the number of collections or a blank string if there are zero. - - - Because this is so unimportant...we wrap the whole thing in a try/expect...last thing we - want is a webpage not to be displayed because of an error here - - Importand TODO: use redis to cache this, don't want to be constantly computing it - - """ - try: - num = len(list(self.user_collections)) - return display_collapsible(num) - except Exception as why: - print("Couldn't display_num_collections:", why) - return "" - - def get_collection_by_name(self, collection_name): - try: - collect = self.user_collections.filter_by( - name=collection_name).first() - except sqlalchemy.orm.exc.NoResultFound: - collect = None - return collect - - @property - def name_and_org(self): - """Nice shortcut for printing out who the user is""" - if self.organization: - return "{} from {}".format(self.full_name, self.organization) - else: - return self.full_name - - @property - def login_count(self): - return self.logins.filter_by(successful=True).count() - - @property - def confirmed_at(self): - if self.confirmed: - confirmed_info = json.loads(self.confirmed) - return confirmed_info['timestamp'] - else: - return None - - @property - def superuser_info(self): - if self.superuser: - return json.loads(self.superuser) - else: - return None - - @property - def crowner(self): - """If made superuser, returns object of person who did the crowning""" - if self.superuser: - superuser_info = json.loads(self.superuser) - crowner = User.query.get(superuser_info['crowned_by']) - return crowner - else: - return None - - @property - def most_recent_login(self): - try: - return self.logins[0] - except IndexError: - return None - - -class Login(Base): - __tablename__ = "login" - id = Column(Unicode(36), primary_key=True, - default=lambda: str(uuid.uuid4())) - user = Column(Unicode(36), ForeignKey('user.id')) - timestamp = Column(DateTime(), default=lambda: datetime.datetime.utcnow()) - ip_address = Column(Unicode(39)) - # False if wrong password was entered - successful = Column(Boolean(), nullable=False) - # Set only if successfully logged in, otherwise should be blank - session_id = Column(Text) - - # Set to user who assumes identity if this was a login for debugging purposes by a superuser - assumed_by = Column(Unicode(36), ForeignKey('user.id')) - - def __init__(self, user): - self.user = user.id - self.ip_address = request.remote_addr - -################################################################################################## - - -class UserCollection(Base): - __tablename__ = "user_collection" - id = Column(Unicode(36), primary_key=True, - default=lambda: str(uuid.uuid4())) - user = Column(Unicode(36), ForeignKey('user.id')) - - # I'd prefer this to not have a length, but for the index below it needs one - name = Column(Unicode(50)) - created_timestamp = Column( - DateTime(), default=lambda: datetime.datetime.utcnow()) - changed_timestamp = Column( - DateTime(), default=lambda: datetime.datetime.utcnow()) - members = Column(Text) # We're going to store them as a json list - - # This index ensures a user doesn't have more than one collection with the same name - __table_args__ = (Index('usercollection_index', "user", "name"), ) - - @property - def num_members(self): - try: - return len(json.loads(self.members)) - except: - return 0 - - def members_as_set(self): - return set(json.loads(self.members)) - - -def display_collapsible(number): - if number: - return number - else: - return "" - - -def user_uuid(): - """Unique cookie for a user""" - user_uuid = request.cookies.get('user_uuid') -- cgit v1.2.3 From d60d30ca54106ff052d6953917f29465ae6e764b Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 6 May 2021 19:45:37 +0000 Subject: Changed the way metadata is displayed on the mapping loading page --- wqflask/wqflask/templates/loading.html | 98 ++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 47 deletions(-) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 124f0608..95cecf76 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -7,56 +7,60 @@ {% endfor %}
-
+
- {% if start_vars.tool_used == "Mapping" %} -

Computing the Maps

-
- n = {{ start_vars.n_samples }} -
- Method = {% if start_vars.method == "gemma" %}GEMMA {% if start_vars.use_loco == "True" %}using LOCO {% endif %}{% else %}{{ start_vars.method }}{% endif %} -
- {% set genofile_desc = start_vars.genofile.split(":")[1] %} - Genotype File = {{ genofile_desc }} - {% if start_vars.num_perm | int > 0 %} -
- # Permutations = {{ start_vars.num_perm }} - {% endif %} - {% if start_vars.num_bootstrap | int > 0 %} -
- # Bootstrap = {{ start_vars.num_bootstrap }} - {% endif %} - {% if start_vars.transform != "" %} -
- transform = {{ start_vars.transform }} - {% endif %} - {% if start_vars.maf != "" and start_vars.method != "reaper" %} -
- MAF >= {{ start_vars.maf }} - {% endif %} - {% if start_vars.covariates != "" and start_vars.method != "reaper" %} -
- {% set covariate_list = start_vars.covariates.split(",") %} - Trait Covariates: {% for covariate in covariate_list %}{% set this_covariate = covariate.split(":")[0] %}{{ this_covariate }}{% if not loop.last %}, {% endif %}{% endfor %} - {% endif %} - {% if start_vars.control_marker != "" and start_vars.do_control == "true" and start_vars.method != "gemma" %} -
- Marker Covariate: {{ start_vars.control_marker }} - {% endif %} - {% else %} -

Loading {{ start_vars.tool_used }} Results...

- {% endif %} -

- -
- -
-- cgit v1.2.3 From 6dc0c82ed93978be67357e0e84b6e5b3531054e6 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 6 May 2021 19:45:57 +0000 Subject: Include group/species in form parameters on trait page so they can be displayed on mapping loading page --- wqflask/wqflask/show_trait/show_trait.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index fcebbc4d..b6fcbcb8 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -263,6 +263,9 @@ class ShowTrait: hddn['temp_trait'] = True hddn['group'] = self.temp_group hddn['species'] = self.temp_species + else: + hddn['group'] = self.dataset.group.name + hddn['species'] = self.dataset.group.species hddn['use_outliers'] = False hddn['method'] = "gemma" hddn['selected_chr'] = -1 -- cgit v1.2.3 From 70735035fe7b59bae13fe955e4f32595055b3940 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 6 May 2021 19:54:48 +0000 Subject: Include group and species in form parameters for mapping results page, so they can be included in the loading page if the map is reloaded --- wqflask/wqflask/templates/mapping_results.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index f054506c..73d7501b 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -17,8 +17,9 @@ {% if temp_trait is defined %} - {% endif %} + + -- cgit v1.2.3 From 3c430082b767a29c3e35cb03e68c1b22373ad353 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 6 May 2021 20:02:35 +0000 Subject: Fixed vertical position of loading text so that it should be positioned in the center vertically --- wqflask/wqflask/templates/loading.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 95cecf76..d384a087 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -8,7 +8,7 @@
-
+
{% if start_vars.tool_used == "Mapping" %}

Computing the Maps


-- cgit v1.2.3