From cddae6a6593a3f65cdb9565207a9a58c338e9024 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 17 Nov 2021 01:19:00 +0300 Subject: temp disable query caching --- wqflask/base/data_set.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 70c58136..768ad49b 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -754,8 +754,8 @@ class DataSet: # Postgres doesn't have that limit, so we can get rid of this after we transition chunk_size = 50 number_chunks = int(math.ceil(len(sample_ids) / chunk_size)) - cached_results = fetch_cached_results(self.name, self.type) - # cached_results = None + # cached_results = fetch_cached_results(self.name, self.type) + cached_results = None if cached_results is None: trait_sample_data = [] for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): @@ -800,8 +800,6 @@ class DataSet: results = g.db.execute(query).fetchall() trait_sample_data.append([list(result) for result in results]) - cache_dataset_results( - self.name, self.type, trait_sample_data) else: trait_sample_data = cached_results -- cgit v1.2.3 From f5dd4b9f6b02ebe20733df0e8bd5e1b5e89f9e93 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 17 Nov 2021 01:31:38 +0300 Subject: temp disable traits metadata cache --- wqflask/wqflask/correlation/show_corr_results.py | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 42010a1e..55915a74 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -86,30 +86,17 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe corr_results = correlation_data['correlation_results'] results_list = [] - file_name = f"{target_dataset['name']}_metadata.json" - - file_path = os.path.join(TMPDIR, file_name) - new_traits_metadata = {} - - try: - with open(file_path,"r+") as file_handler: - dataset_metadata = json.load(file_handler) - - except FileNotFoundError: - Path(file_path).touch(exist_ok=True) - dataset_metadata = {} for i, trait_dict in enumerate(corr_results): trait_name = list(trait_dict.keys())[0] trait = trait_dict[trait_name] - target_trait = dataset_metadata.get(trait_name) + target_trait = None if target_trait is None: target_trait_ob = create_trait(dataset=target_dataset_ob, name=trait_name, get_qtl_info=True) target_trait = jsonable(target_trait_ob, target_dataset_ob) - new_traits_metadata[trait_name] = target_trait if target_trait['view'] == False: continue results_dict = {} @@ -184,12 +171,6 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe results_list.append(results_dict) - - if bool(new_traits_metadata): - # that means new traits exists - dataset_metadata.update(new_traits_metadata) - with open(file_path,"w+") as file_handler: - json.dump(dataset_metadata, file_handler) return json.dumps(results_list) -- cgit v1.2.3 From 9577152dfed7ae3fab2fb677a89b78ed88b9e127 Mon Sep 17 00:00:00 2001 From: jgart Date: Wed, 17 Nov 2021 02:37:17 -0500 Subject: update jupyter notebook link name This does not point to the launcher app but to the curated notebooks. --- wqflask/wqflask/templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index 6e922f24..a3bd1b63 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -87,7 +87,7 @@
  • Systems Genetics PheWAS
  • Genome Browser
  • BXD Power Calculator
    • -
  • Jupyter Notebook Launcher
    • +
  • Jupyter Notebooks
  • Interplanetary File System
    • -- cgit v1.2.3 From ea2ce4c05f9b367510ba60a9adc5da9e8337f032 Mon Sep 17 00:00:00 2001 From: jgart Date: Wed, 17 Nov 2021 02:47:52 -0500 Subject: update live jupyter notebooks --- wqflask/wqflask/jupyter_notebooks.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/wqflask/wqflask/jupyter_notebooks.py b/wqflask/wqflask/jupyter_notebooks.py index dbea04dd..7d76828e 100644 --- a/wqflask/wqflask/jupyter_notebooks.py +++ b/wqflask/wqflask/jupyter_notebooks.py @@ -6,16 +6,12 @@ jupyter_notebooks = Blueprint('jupyter_notebooks', __name__) def launcher(): links = ( { - "main_url": "http://notebook.genenetwork.org/51091/tree?", - "notebook_name": "COVID-19 in mybinder.org federation", - "src_link_url": "https://github.com/jgarte/covid19_in_binder"}, + "main_url": "http://notebook.genenetwork.org/34301/notebooks/genenetwork-api-using-r.ipynb", + "notebook_name": "R notebook showing how to query the GeneNetwork API.", + "src_link_url": "https://github.com/jgarte/genenetwork-api-r-jupyter-notebook"}, { - "main_url": "http://notebook.genenetwork.org/35639/tree?", - "notebook_name": "Simple requirements.txt based example", - "src_link_url": "https://github.com/jgarte/requirements"}, - { - "main_url": "http://notebook.genenetwork.org/40733/tree?", - "notebook_name": "Guile Jupyter Notebook Querying GeneNetwork API", - "src_link_url": "https://github.com/jgarte/guile-notebook-genenetwork-api"}) + "main_url": "http://notebook.genenetwork.org/57675/notebooks/genenetwork.ipynb", + "notebook_name": "Querying the GeneNetwork API declaratively with python.", + "src_link_url": "https://github.com/jgarte/genenetwork-jupyter-notebook-example"}) return render_template("jupyter_notebooks.html", links=links) -- cgit v1.2.3 From 9c17856e0aa047408d91b0fe0bd6591d305cca42 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 17 Nov 2021 04:07:08 -0600 Subject: Moven GN API doc to gn-docs repo --- doc/API_readme.md | 168 +----------------------------------------------------- 1 file changed, 1 insertion(+), 167 deletions(-) diff --git a/doc/API_readme.md b/doc/API_readme.md index be6668dc..17d10e44 100644 --- a/doc/API_readme.md +++ b/doc/API_readme.md @@ -1,169 +1,3 @@ # API Query Documentation # ---- -# Fetching Dataset/Trait info/data # ---- -## Fetch Species List ## -To get a list of species with data available in GN (and their associated names and ids): -``` -curl http://genenetwork.org/api/v_pre1/species -[ { "FullName": "Mus musculus", "Id": 1, "Name": "mouse", "TaxonomyId": 10090 }, ... { "FullName": "Populus trichocarpa", "Id": 10, "Name": "poplar", "TaxonomyId": 3689 } ] -``` - -Or to get a single species info: -``` -curl http://genenetwork.org/api/v_pre1/species/mouse -``` -OR -``` -curl http://genenetwork.org/api/v_pre1/species/mouse.json -``` - -*For all queries where the last field is a user-specified name/ID, there will be the option to append a file format type. Currently there is only JSON (and it will default to JSON if none is provided), but other formats will be added later* - -## Fetch Groups/RISets ## - -This query can optionally filter by species: - -``` -curl http://genenetwork.org/api/v_pre1/groups (for all species) -``` -OR -``` -curl http://genenetwork.org/api/v_pre1/groups/mouse (for just mouse groups/RISets) -[ { "DisplayName": "BXD", "FullName": "BXD RI Family", "GeneticType": "riset", "Id": 1, "MappingMethodId": "1", "Name": "BXD", "SpeciesId": 1, "public": 2 }, ... { "DisplayName": "AIL LGSM F34 and F39-43 (GBS)", "FullName": "AIL LGSM F34 and F39-43 (GBS)", "GeneticType": "intercross", "Id": 72, "MappingMethodId": "2", "Name": "AIL-LGSM-F34-F39-43-GBS", "SpeciesId": 1, "public": 2 } ] -``` - -## Fetch Genotypes for Group/RISet ## -``` -curl http://genenetwork.org/api/v_pre1/genotypes/bimbam/BXD -curl http://genenetwork.org/api/v_pre1/genotypes/BXD.bimbam -``` -Returns a group's genotypes in one of several formats - bimbam, rqtl2, or geno (a format used by qtlreaper which is just a CSV file consisting of marker positions and genotypes) - -Rqtl2 genotype queries can also include the dataset name and will return a zip of the genotypes, phenotypes, and gene map (marker names/positions). For example: -``` -curl http://genenetwork.org/api/v_pre1/genotypes/rqtl2/BXD/HC_M2_0606_P.zip -``` - -## Fetch Datasets ## -``` -curl http://genenetwork.org/api/v_pre1/datasets/bxd -``` -OR -``` -curl http://genenetwork.org/api/v_pre1/datasets/mouse/bxd -[ { "AvgID": 1, "CreateTime": "Fri, 01 Aug 2003 00:00:00 GMT", "DataScale": "log2", "FullName": "UTHSC/ETHZ/EPFL BXD Liver Polar Metabolites Extraction A, CD Cohorts (Mar 2017) log2", "Id": 1, "Long_Abbreviation": "BXDMicroArray_ProbeSet_August03", "ProbeFreezeId": 3, "ShortName": "Brain U74Av2 08/03 MAS5", "Short_Abbreviation": "Br_U_0803_M", "confidentiality": 0, "public": 0 }, ... { "AvgID": 3, "CreateTime": "Tue, 14 Aug 2018 00:00:00 GMT", "DataScale": "log2", "FullName": "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Aug18) RMA", "Id": 859, "Long_Abbreviation": "EPFLMouseLiverCDRMAApr18", "ProbeFreezeId": 181, "ShortName": "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Aug18) RMA", "Short_Abbreviation": "EPFLMouseLiverCDRMA0818", "confidentiality": 0, "public": 1 } ] -``` -(I added the option to specify species just in case we end up with the same group name across multiple species at some point, though it's currently unnecessary) - -## Fetch Individual Dataset Info ## -### For mRNA Assay/"ProbeSet" ### - -``` -curl http://genenetwork.org/api/v_pre1/dataset/HC_M2_0606_P -``` -OR -``` -curl http://genenetwork.org/api/v_pre1/dataset/bxd/HC_M2_0606_P``` -{ "confidential": 0, "data_scale": "log2", "dataset_type": "mRNA expression", "full_name": "Hippocampus Consortium M430v2 (Jun06) PDNN", "id": 112, "name": "HC_M2_0606_P", "public": 2, "short_name": "Hippocampus M430v2 BXD 06/06 PDNN", "tissue": "Hippocampus mRNA", "tissue_id": 9 } -``` -(This also has the option to specify group/riset) - -### For "Phenotypes" (basically non-mRNA Expression; stuff like weight, sex, etc) ### -For these traits, the query fetches publication info and takes the group and phenotype 'ID' as input. For example: -``` -curl http://genenetwork.org/api/v_pre1/dataset/bxd/10001 -{ "dataset_type": "phenotype", "description": "Central nervous system, morphology: Cerebellum weight, whole, bilateral in adults of both sexes [mg]", "id": 10001, "name": "CBLWT2", "pubmed_id": 11438585, "title": "Genetic control of the mouse cerebellum: identification of quantitative trait loci modulating size and architecture", "year": "2001" } -``` - -## Fetch Sample Data for Dataset ## -``` -curl http://genenetwork.org/api/v_pre1/sample_data/HSNIH-PalmerPublish.csv -``` - -Returns a CSV file with sample/strain names as the columns and trait IDs as rows - -## Fetch Sample Data for Single Trait ## -``` -curl http://genenetwork.org/api/v_pre1/sample_data/HC_M2_0606_P/1436869_at -[ { "data_id": 23415463, "sample_name": "129S1/SvImJ", "sample_name_2": "129S1/SvImJ", "se": 0.123, "value": 8.201 }, { "data_id": 23415463, "sample_name": "A/J", "sample_name_2": "A/J", "se": 0.046, "value": 8.413 }, { "data_id": 23415463, "sample_name": "AKR/J", "sample_name_2": "AKR/J", "se": 0.134, "value": 8.856 }, ... ] -``` - -## Fetch Trait List for Dataset ## -``` -curl http://genenetwork.org/api/v_pre1/traits/HXBBXHPublish.json -[ { "Additive": 0.0499967532467532, "Id": 10001, "LRS": 16.2831307029479, "Locus": "rs106114574", "PhenotypeId": 1449, "PublicationId": 319, "Sequence": 1 }, ... ] -``` - -Both JSON and CSV formats can be specified, with JSON as default. There is also an optional "ids_only" and "names_only" parameter that will only return a list of trait IDs or names, respectively. - -## Fetch Trait Info (Name, Description, Location, etc) ## -### For mRNA Expression/"ProbeSet" ### -``` -curl http://genenetwork.org/api/v_pre1/trait/HC_M2_0606_P/1436869_at -{ "additive": -0.214087568058076, "alias": "HHG1; HLP3; HPE3; SMMCI; Dsh; Hhg1", "chr": "5", "description": "sonic hedgehog (hedgehog)", "id": 99602, "locus": "rs8253327", "lrs": 12.7711275309832, "mb": 28.457155, "mean": 9.27909090909091, "name": "1436869_at", "p_value": 0.306, "se": null, "symbol": "Shh" } -``` - -### For "Phenotypes" ### -For phenotypes this just gets the max LRS, its location, and additive effect (as calculated by qtlreaper) - -Since each group/riset only has one phenotype "dataset", this query takes either the group/riset name or the group/riset name + "Publish" (for example "BXDPublish", which is the dataset name in the DB) as input -``` -curl http://genenetwork.org/api/v_pre1/trait/BXD/10001 -{ "additive": 2.39444435069444, "id": 4, "locus": "rs48756159", "lrs": 13.4974911471087 } -``` - ---- - -# Analyses # ---- -## Mapping ## -Currently two mapping tools can be used - GEMMA and R/qtl. qtlreaper will be added later with Christian Fischer's RUST implementation - https://github.com/chfi/rust-qtlreaper - -Each method's query takes the following parameters respectively (more will be added): -### GEMMA ### -* trait_id (*required*) - ID for trait being mapped -* db (*required*) - DB name for trait above (Short_Abbreviation listed when you query for datasets) -* use_loco - Whether to use LOCO (leave one chromosome out) method (default = false) -* maf - minor allele frequency (default = 0.01) - -Example query: -``` -curl http://genenetwork.org/api/v_pre1/mapping?trait_id=10015&db=BXDPublish&method=gemma&use_loco=true -``` - -### R/qtl ### -(See the R/qtl guide for information on some of these options - http://www.rqtl.org/manual/qtl-manual.pdf) -* trait_id (*required*) - ID for trait being mapped -* db (*required*) - DB name for trait above (Short_Abbreviation listed when you query for datasets) -* rqtl_method - hk (default) | ehk | em | imp | mr | mr-imp | mr-argmax ; Corresponds to the "method" option for the R/qtl scanone function. -* rqtl_model - normal (default) | binary | 2-part | np ; corresponds to the "model" option for the R/qtl scanone function -* num_perm - number of permutations; 0 by default -* control_marker - Name of marker to use as control; this relies on the user knowing the name of the marker they want to use as a covariate -* interval_mapping - Whether to use interval mapping; "false" by default -* pair_scan - *NYI* - -Example query: -``` -curl http://genenetwork.org/api/v_pre1/mapping?trait_id=1418701_at&db=HC_M2_0606_P&method=rqtl&num_perm=100 -``` - -Some combinations of methods/models may not make sense. The R/qtl manual should be referred to for any questions on its use (specifically the scanone function in this case) - -## Calculate Correlation ## -Currently only Sample and Tissue correlations are implemented - -This query currently takes the following parameters (though more will be added): -* trait_id (*required*) - ID for trait used for correlation -* db (*required*) - DB name for the trait above (this is the Short_Abbreviation listed when you query for datasets) -* target_db (*required*) - Target DB name to be correlated against -* type - sample (default) | tissue -* method - pearson (default) | spearman -* return - Number of results to return (default = 500) - -Example query: -``` -curl http://genenetwork.org/api/v_pre1/correlation?trait_id=1427571_at&db=HC_M2_0606_P&target_db=BXDPublish&type=sample&return_count=100 -[ { "#_strains": 6, "p_value": 0.004804664723032055, "sample_r": -0.942857142857143, "trait": 20511 }, { "#_strains": 6, "p_value": 0.004804664723032055, "sample_r": -0.942857142857143, "trait": 20724 }, { "#_strains": 12, "p_value": 1.8288943424888848e-05, "sample_r": -0.9233615170820528, "trait": 13536 }, { "#_strains": 7, "p_value": 0.006807187408935392, "sample_r": 0.8928571428571429, "trait": 10157 }, { "#_strains": 7, "p_value": 0.006807187408935392, "sample_r": -0.8928571428571429, "trait": 20392 }, ... ] -``` +This document has moved to [gn-docs](https://github.com/genenetwork/gn-docs/blob/master/api/GN2-REST-API.md)! -- cgit v1.2.3 From 83719648e505747f16281d7e8e14f1003297be5c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 17 Nov 2021 13:42:26 +0300 Subject: README.org: Replace broken link with environments page --- doc/README.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/README.org b/doc/README.org index 8839aefc..e1c6b614 100644 --- a/doc/README.org +++ b/doc/README.org @@ -26,7 +26,7 @@ * Introduction -Large system deployments can get very [[http://biogems.info/contrib/genenetwork/gn2.svg ][complex]]. In this document we +Large system deployments can get very [[http://genenetwork.org/environments/][complex]]. In this document we explain the GeneNetwork version 2 (GN2) reproducible deployment system which is based on GNU Guix (see also [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix system can be used to install GN with all its files and dependencies. -- cgit v1.2.3 From 747544906c4d331a8d46ed692070e0f9d335191b Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 17 Nov 2021 18:31:59 +0000 Subject: Fixes gene global search error caused by some genes not having a symbol (not sure why this is in the DB; seems to have only happened recently) --- wqflask/wqflask/gsearch.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index 2516e4fb..bf44408e 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -82,12 +82,13 @@ class GSearch: this_trait['species'] = line[0] this_trait['group'] = line[1] this_trait['tissue'] = line[2] - this_trait['symbol'] = line[6] + this_trait['symbol'] = "N/A" + if line[6]: + this_trait['symbol'] = line[6] + this_trait['description'] = "N/A" if line[7]: this_trait['description'] = line[7].decode( 'utf-8', 'replace') - else: - this_trait['description'] = "N/A" this_trait['location_repr'] = 'N/A' if (line[8] != "NULL" and line[8] != "") and (line[9] != 0): this_trait['location_repr'] = 'Chr%s: %.6f' % ( -- cgit v1.2.3 From 3325440faef1161da90d51444543d8d847fb9cdd Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 17 Nov 2021 18:27:56 +0000 Subject: Fix issue where gene global search throws an error due to some traits not having a gene symbol (not sure why they don't in the DB, but the code defaults it to N/A now) --- wqflask/wqflask/gsearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index bf44408e..31f3305c 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -89,7 +89,7 @@ class GSearch: if line[7]: this_trait['description'] = line[7].decode( 'utf-8', 'replace') - this_trait['location_repr'] = 'N/A' + this_trait['location_repr'] = "N/A" if (line[8] != "NULL" and line[8] != "") and (line[9] != 0): this_trait['location_repr'] = 'Chr%s: %.6f' % ( line[8], float(line[9])) -- cgit v1.2.3 From dea1afa01025026a0754de60615b974a5cb3fd24 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 17 Nov 2021 21:53:20 +0000 Subject: Changed the URL for editing trait metadata to use the dataset group ID instead of the PublishFreezeId --- wqflask/wqflask/templates/show_trait_details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 6b125221..4e9ea0fb 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -236,7 +236,7 @@ {% if admin_status != None and admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} {% if this_trait.dataset.type == 'Publish' %} - + {% endif %} {% if this_trait.dataset.type == 'ProbeSet' %} -- cgit v1.2.3