diff options
36 files changed, 1056 insertions, 317 deletions
@@ -1,12 +1,13 @@ [](https://zenodo.org/badge/latestdoi/5591/genenetwork/genenetwork2) [](http://joss.theoj.org/papers/10.21105/joss.00025) [](https://github.com/genenetwork/genenetwork2/actions) + # GeneNetwork This repository contains the current source code for GeneNetwork (GN) (https://www.genenetwork.org/ (version 2). GN2 is a Web -2.0-style framework that includes data and computational tools for online genetics and genomic analysis of -many different populations and many types of molecular, cellular, and physiological data. +2.0-style framework that includes data and computational tools for online genetics and genomic analysis of +many different populations and many types of molecular, cellular, and physiological data. The system is used by scientists and clinians in the field of precision health care and systems genetics. GN and its predecessors have been in operation since Jan 1994, making it one of the longest-lived web services in biomedical research (https://en.wikipedia.org/wiki/GeneNetwork, and see a partial list of publications using GN and its predecessor, WebQTL (https://genenetwork.org/references/). @@ -26,7 +27,7 @@ interface genenetwork2 ``` -(default is http://localhost:5003/). A quick example is +A quick example is ```sh env GN2_PROFILE=~/opt/gn-latest SERVER_PORT=5300 GENENETWORK_FILES=~/data/gn2_data/ ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 5f4e0f9a..2b94b2a2 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -209,7 +209,7 @@ if [ "$1" = '-gunicorn-prod' ] ; then echo PYTHONPATH=$PYTHONPATH if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi PID=$TMPDIR/gunicorn.$USER.pid - cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID --workers 20 --keep-alive 6000 --max-requests 1000 --timeout 1200 wsgi" + cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID --workers 20 --keep-alive 6000 --max-requests 100 --max-requests-jitter 30 --timeout 1200 wsgi" echo RUNNING gunicorn $cmd gunicorn $cmd exit $? diff --git a/doc/README.org b/doc/README.org index 43c92e3c..1236016e 100644 --- a/doc/README.org +++ b/doc/README.org @@ -2,7 +2,9 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - - [[#install][Install]] + - [[#check-list][Check list]] + - [[#installing-guix-packages][Installing Guix packages]] + - [[#creating-a-gnu-guix-profile][Creating a GNU Guix profile]] - [[#running-gn2][Running GN2]] - [[#run-gn-proxy][Run gn-proxy]] - [[#run-redis][Run Redis]] @@ -37,32 +39,106 @@ tree. Current supported versions can be found as the SHA values of For a full view of runtime dependencies as defined by GNU Guix, see an example of the [[#gn2-dependency-graph][GN2 Dependency Graph]]. -* Install +* Check list + +To run GeneNetwork the following services need to function: + +1. [ ] GNU Guix with a guix profile for genenetwork2 +1. [ ] A path to the (static) genotype files +1. [ ] Gn-proxy for authentication +1. [ ] The genenetwork3 service +1. [ ] Redis +1. [ ] Mariadb + +* Installing Guix packages Make sure to install GNU Guix using the binary download instructions on the main website. Follow the instructions on [[GUIX-Reproducible-from-source.org]] to download pre-built binaries. Note -the download amounts to several GBs of data. +the download amounts to several GBs of data. Debian-derived distros +may support + +: apt-get install guix + +* Creating a GNU Guix profile + +We run a GNU Guix channel with packages at [[https://git.genenetwork.org/guix-bioinformatics/guix-bioinformatics][guix-bioinformatics]]. The +README has instructions for hosting a channel, but typically we use +the GUIX_PACKAGE_PATH instead. First upgrade to a recent guix with + +: mkdir ~/opt +: guix pull -p ~/opt/guix-pull + +It should upgrade (ignore the locales warnings). You can optionally +specify the specific git checkout of guix with + +: guix pull -p ~/opt/guix-pull --commit=f04883d + +which is useful when you ned to roll back to an earlier version +(sometimes our channel goes out of sync). Next, we install +GeneNetwork2 with + +: source ~/opt/guix-pull/etc/profile +: git clone https://git.genenetwork.org/guix-bioinformatics/guix-bioinformatics.git ~/guix-bioinformatics +: cd ~/guix-bioinformatics +: git pull +: env GUIX_PACKAGE_PATH=$HOME/guix-bioinformatics guix package -i genenetwork2 -p ~/opt/genenetwork2 + +you probably also need guix-past (the upstream channel for older packages): + +: git clone https://gitlab.inria.fr/guix-hpc/guix-past.git ~/guix-past +: cd ~/guix-past +: git pull +: env GUIX_PACKAGE_PATH=$HOME/guix-bioinformatics:$HOME/guix-past/modules ~/opt/guix-pull/bin/guix package -i genenetwork2 -p ~/opt/genenetwork2 + +ignore the warnings. Guix should install the software without trying +to build everything. If you system insists on building all packages, +try the `--dry-run` switch and fix the [[https://guix.gnu.org/manual/en/html_node/Substitute-Server-Authorization.html][substitutes]]. You may add the +`--substitute-urls="http://guix.genenetwork.org https://ci.guix.gnu.org https://mirror.hydra.gnu.org"` switch. + +The guix.genenetwork.org has most of our packages pre-built(!). To use +it on your own machine the public key is + +#+begin_src scheme +(public-key + (ecc + (curve Ed25519) + (q #E50F005E6DA2F85749B9AA62C8E86BB551CE2B541DC578C4DBE613B39EC9E750#))) +#+end_src + +Once we have a GNU Guix profile, a running database (see below) and the file storage, +we should be ready to fire up GeneNetwork: * Running GN2 -Default settings for GN2 are listed in a file called -[[../etc/default_settings.py][default_settings.py]]. You can copy this file and pass it as a new -parameter to the genenetwork2 command, e.g. +Check out the source with git: -: genenetwork2 mysettings.py +: git clone git@github.com:genenetwork/genenetwork2.git +: cd genenetwork2 -or you can set environment variables to override individual parameters, e.g. +Run GN2 with above Guix profile -: env SERVER_PORT=5004 SQL_URI=mysql://user:pwd@dbhostname/db_webqtl genenetwork2 +: export GN2_PROFILE=$HOME/opt/genenetwork2 +: env TMPDIR=$HOME/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5012 GENENETWORK_FILES=/export/data/genenetwork/genotype_files SQL_URI=mysql://webqtlout:webqtlout@localhost/db_webqtl ./bin/genenetwork2 etc/default_settings.py -gunicorn-dev the debug and logging switches can be particularly useful when -developing GN2. +developing GN2. Location and files are the current ones for Penguin2. + +It may be useful to tunnel the web server to your local browser with +an ssh tunnel: + +If you want to test a service running on the server on a certain +port (say 8202) use + + ssh -L 8202:127.0.0.1:8202 -f -N myname@penguin2.genenetwork.org + +And browse on your local machine to http://localhost:8202/ * Run gn-proxy GeneNetwork requires a separate gn-proxy server which handles -authorisation and access control. For instructions see the [[https://github.com/genenetwork/gn-proxy][README]]. +authorisation and access control. For instructions see the +[[https://github.com/genenetwork/gn-proxy][README]]. Note it may already be running on our servers! * Run Redis diff --git a/scripts/add_missing_columns.sh b/scripts/add_missing_columns.sh index 70d5fdeb..611e2dd6 100644 --- a/scripts/add_missing_columns.sh +++ b/scripts/add_missing_columns.sh @@ -13,6 +13,9 @@ ALTER TABLE PublishXRef ADD mean double AFTER DataId; + ALTER TABLE CaseAttribute + ADD Description varchar(255) AFTER Name; + -- This takes some time ALTER TABLE ProbeSet ADD UniProtID varchar(20) AFTER ProteinName; diff --git a/wqflask/.DS_Store b/wqflask/.DS_Store Binary files differdeleted file mode 100644 index d992942f..00000000 --- a/wqflask/.DS_Store +++ /dev/null diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 4cb82665..8906ab69 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -277,7 +277,6 @@ class Markers: filtered_markers = [] for marker in self.markers: if marker['name'] in p_values: - # logger.debug("marker {} IS in p_values".format(i)) marker['p_value'] = p_values[marker['name']] if math.isnan(marker['p_value']) or (marker['p_value'] <= 0): marker['lod_score'] = 0 @@ -298,7 +297,6 @@ class HumanMarkers(Markers): self.markers = [] for line in marker_data_fh: splat = line.strip().split() - # logger.debug("splat:", splat) if len(specified_markers) > 0: if splat[1] in specified_markers: marker = {} @@ -398,6 +396,15 @@ class DatasetGroup: if maternal and paternal: self.parlist = [maternal, paternal] + def get_study_samplelists(self): + study_sample_file = locate_ignore_error(self.name + ".json", 'study_sample_lists') + try: + f = open(study_sample_file) + except: + return [] + study_samples = json.load(f) + return study_samples + def get_genofiles(self): jsonfile = "%s/%s.json" % (webqtlConfig.GENODIR, self.name) try: @@ -737,7 +744,6 @@ class DataSet: and Strain.SpeciesId=Species.Id and Species.name = '{}' """.format(create_in_clause(self.samplelist), *mescape(self.group.species)) - logger.sql(query) results = dict(g.db.execute(query).fetchall()) sample_ids = [results[item] for item in self.samplelist] @@ -908,7 +914,6 @@ class PhenotypeDataSet(DataSet): Geno.Name = '%s' and Geno.SpeciesId = Species.Id """ % (species, this_trait.locus) - logger.sql(query) result = g.db.execute(query).fetchone() if result: @@ -938,7 +943,6 @@ class PhenotypeDataSet(DataSet): Order BY Strain.Name """ - logger.sql(query) results = g.db.execute(query, (trait, self.id)).fetchall() return results @@ -1005,7 +1009,6 @@ class GenotypeDataSet(DataSet): Order BY Strain.Name """ - logger.sql(query) results = g.db.execute(query, (webqtlDatabaseFunction.retrieve_species_id(self.group.name), trait, self.name)).fetchall() @@ -1126,8 +1129,6 @@ class MrnaAssayDataSet(DataSet): ProbeSet.Name = '%s' """ % (escape(str(this_trait.dataset.id)), escape(this_trait.name))) - - logger.sql(query) result = g.db.execute(query).fetchone() mean = result[0] if result else 0 @@ -1147,7 +1148,6 @@ class MrnaAssayDataSet(DataSet): Geno.Name = '{}' and Geno.SpeciesId = Species.Id """.format(species, this_trait.locus) - logger.sql(query) result = g.db.execute(query).fetchone() if result: @@ -1179,7 +1179,6 @@ class MrnaAssayDataSet(DataSet): Order BY Strain.Name """ % (escape(trait), escape(self.name)) - logger.sql(query) results = g.db.execute(query).fetchall() return results @@ -1190,7 +1189,6 @@ class MrnaAssayDataSet(DataSet): where ProbeSetXRef.ProbeSetFreezeId = %s and ProbeSetXRef.ProbeSetId=ProbeSet.Id; """ % (column_name, escape(str(self.id))) - logger.sql(query) results = g.db.execute(query).fetchall() return dict(results) @@ -1224,7 +1222,6 @@ def geno_mrna_confidentiality(ob): query = '''SELECT Id, Name, FullName, confidentiality, AuthorisedUsers FROM %s WHERE Name = "%s"''' % (dataset_table, ob.name) - logger.sql(query) result = g.db.execute(query) (dataset_id, diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 10851e00..96a09302 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -27,11 +27,13 @@ def create_trait(**kw): assert bool(kw.get('name')), "Needs trait name" - if kw.get('dataset_name'): + if bool(kw.get('dataset')): + dataset = kw.get('dataset') + else: if kw.get('dataset_name') != "Temp": dataset = create_dataset(kw.get('dataset_name')) - else: - dataset = kw.get('dataset') + else: + dataset = create_dataset("Temp", group_name=kw.get('group_name')) if dataset.type == 'Publish': permissions = check_resource_availability( diff --git a/wqflask/runserver.py b/wqflask/runserver.py index df957bd9..8198b921 100644 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -23,6 +23,9 @@ app_config() werkzeug_logger = logging.getLogger('werkzeug') if WEBSERVER_MODE == 'DEBUG': + from flask_debugtoolbar import DebugToolbarExtension + app.debug = True + toolbar = DebugToolbarExtension(app) app.run(host='0.0.0.0', port=SERVER_PORT, debug=True, diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index 9b2c6735..d4256a46 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -139,7 +139,7 @@ def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLab max_D = max(data) min_D = min(data) # add by NL 06-20-2011: fix the error: when max_D is infinite, log function in detScale will go wrong - if max_D == float('inf') or max_D > webqtlConfig.MAXLRS: + if (max_D == float('inf') or max_D > webqtlConfig.MAXLRS) and min_D < webqtlConfig.MAXLRS: max_D = webqtlConfig.MAXLRS # maximum LRS value xLow, xTop, stepX = detScale(min_D, max_D) @@ -156,7 +156,7 @@ def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLab j += step for i, item in enumerate(data): - if item == float('inf') or item > webqtlConfig.MAXLRS: + if (item == float('inf') or item > webqtlConfig.MAXLRS) and min_D < webqtlConfig.MAXLRS: item = webqtlConfig.MAXLRS # maximum LRS value j = int((item - xLow) / step) Count[j] += 1 diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py index 56d0af6f..778fb64d 100644 --- a/wqflask/utility/startup_config.py +++ b/wqflask/utility/startup_config.py @@ -20,8 +20,12 @@ def app_config(): import os app.config['SECRET_KEY'] = str(os.urandom(24)) mode = WEBSERVER_MODE - if mode == "DEV" or mode == "DEBUG": + if mode in ["DEV", "DEBUG"]: app.config['TEMPLATES_AUTO_RELOAD'] = True + if mode == "DEBUG": + from flask_debugtoolbar import DebugToolbarExtension + app.debug = True + toolbar = DebugToolbarExtension(app) print("==========================================") diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index aea91220..d0d4bcba 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -18,7 +18,10 @@ from gn3.db_utils import database_connector def create_target_this_trait(start_vars): """this function creates the required trait and target dataset for correlation""" - this_dataset = data_set.create_dataset(dataset_name=start_vars['dataset']) + if start_vars['dataset'] == "Temp": + this_dataset = data_set.create_dataset(dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) + else: + this_dataset = data_set.create_dataset(dataset_name=start_vars['dataset']) target_dataset = data_set.create_dataset( dataset_name=start_vars['corr_dataset']) this_trait = create_trait(dataset=this_dataset, @@ -145,10 +148,7 @@ def lit_for_trait_list(corr_results, this_dataset, this_trait): def fetch_sample_data(start_vars, this_trait, this_dataset, target_dataset): sample_data = process_samples( - start_vars, this_dataset.group.samplelist) - - sample_data = test_process_data(this_trait, this_dataset, start_vars) - + start_vars, this_dataset.group.all_samples_ordered()) if target_dataset.type == "ProbeSet": target_dataset.get_probeset_data(list(sample_data.keys())) @@ -202,17 +202,22 @@ def compute_correlation(start_vars, method="pearson", compute_all=False): if tissue_input is not None: (primary_tissue_data, target_tissue_data) = tissue_input - corr_input_data = { - "primary_tissue": primary_tissue_data, - "target_tissues_dict": target_tissue_data - } - correlation_results = compute_tissue_correlation( - primary_tissue_dict=corr_input_data["primary_tissue"], - target_tissues_data=corr_input_data[ - "target_tissues_dict"], - corr_method=method - - ) + corr_input_data = { + "primary_tissue": primary_tissue_data, + "target_tissues_dict": target_tissue_data + } + correlation_results = compute_tissue_correlation( + primary_tissue_dict=corr_input_data["primary_tissue"], + target_tissues_data=corr_input_data[ + "target_tissues_dict"], + corr_method=method + + ) + else: + return {"correlation_results": [], + "this_trait": this_trait.name, + "target_dataset": start_vars['corr_dataset'], + "return_results": corr_return_results} elif corr_type == "lit": (this_trait_geneid, geneid_dict, species) = do_lit_correlation( @@ -303,4 +308,3 @@ def get_tissue_correlation_input(this_trait, trait_symbol_dict): "symbol_tissue_vals_dict": corr_result_tissue_vals_dict } return (primary_tissue_data, target_tissue_data) - return None diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index e2daa991..d73965da 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -30,7 +30,10 @@ def set_template_vars(start_vars, correlation_data): corr_type = start_vars['corr_type'] corr_method = start_vars['corr_sample_method'] - this_dataset_ob = create_dataset(dataset_name=start_vars['dataset']) + if start_vars['dataset'] == "Temp": + this_dataset_ob = create_dataset(dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) + else: + this_dataset_ob = create_dataset(dataset_name=start_vars['dataset']) this_trait = create_trait(dataset=this_dataset_ob, name=start_vars['trait_id']) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index f941267e..3986c441 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -2113,7 +2113,7 @@ class DisplayMappingResults: thisChr.append( [_locus.name, _locus.cM - Locus0CM]) else: - for j in (0, nLoci / 4, nLoci / 2, nLoci * 3 / 4, -1): + for j in (0, round(nLoci / 4), round(nLoci / 2), round(nLoci * 3 / 4), -1): while _chr[j].name == ' - ': j += 1 if _chr[j].cM != preLpos: diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index f88c5ac8..623ab87f 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -11,6 +11,7 @@ from utility.tools import flat_files from utility.tools import GEMMA_WRAPPER_COMMAND from utility.tools import TEMPDIR from utility.tools import WEBSERVER_MODE +from gn3.computations.gemma import generate_hash_of_string import utility.logger logger = utility.logger.getLogger(__name__) @@ -34,10 +35,7 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, genofile_name = this_dataset.group.name if first_run: - trait_filename = (f"{str(this_dataset.group.name)}_" - f"{str(this_trait.name)}_" - f"{generate_random_n_string(6)}") - gen_pheno_txt_file(this_dataset, genofile_name, vals, trait_filename) + pheno_filename = gen_pheno_txt_file(this_dataset, genofile_name, vals) if not os.path.isfile(f"{webqtlConfig.GENERATED_IMAGE_DIR}" f"{genofile_name}_output.assoc.txt"): @@ -56,13 +54,13 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, chr_list_string = ",".join(this_chromosomes_name) if covariates != "": - gen_covariates_file(this_dataset, covariates, samples) + covar_filename = gen_covariates_file(this_dataset, covariates, samples) if use_loco == "True": generate_k_command = (f"{GEMMA_WRAPPER_COMMAND} --json --loco " f"{chr_list_string} -- {GEMMAOPTS} " f"-g {flat_files('genotype/bimbam')}/" f"{genofile_name}_geno.txt -p " - f"{TEMPDIR}/gn2/{trait_filename}.txt -a " + f"{TEMPDIR}/gn2/{pheno_filename}.txt -a " f"{flat_files('genotype/bimbam')}/" f"{genofile_name}_snps.txt -gk > " f"{TEMPDIR}/gn2/{k_output_filename}.json") @@ -73,10 +71,10 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, f"-- {GEMMAOPTS} " f"-g {flat_files('genotype/bimbam')}/" f"{genofile_name}_geno.txt " - f"-p {TEMPDIR}/gn2/{trait_filename}.txt ") + f"-p {TEMPDIR}/gn2/{pheno_filename}.txt ") if covariates != "": gemma_command += (f"-c {flat_files('mapping')}/" - f"{this_dataset.group.name}_covariates.txt " + f"{covar_filename}.txt " f"-a {flat_files('genotype/bimbam')}/" f"{genofile_name}_snps.txt " f"-lmm 9 -maf {maf} > {TEMPDIR}/gn2/" @@ -92,7 +90,7 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, f"{GEMMAOPTS} " f" -g {flat_files('genotype/bimbam')}/" f"{genofile_name}_geno.txt -p " - f"{TEMPDIR}/gn2/{trait_filename}.txt -a " + f"{TEMPDIR}/gn2/{pheno_filename}.txt -a " f"{flat_files('genotype/bimbam')}/" f"{genofile_name}_snps.txt -gk > " f"{TEMPDIR}/gn2/{k_output_filename}.json") @@ -106,12 +104,11 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, f"{genofile_name}_snps.txt " f"-lmm 9 -g {flat_files('genotype/bimbam')}/" f"{genofile_name}_geno.txt -p " - f"{TEMPDIR}/gn2/{trait_filename}.txt ") + f"{TEMPDIR}/gn2/{pheno_filename}.txt ") if covariates != "": gemma_command += (f" -c {flat_files('mapping')}/" - f"{this_dataset.group.name}" - f"_covariates.txt > " + f"{covar_filename}.txt > " f"{TEMPDIR}/gn2/{gwa_output_filename}.json") else: gemma_command += f" > {TEMPDIR}/gn2/{gwa_output_filename}.json" @@ -129,16 +126,20 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, return marker_obs, gwa_output_filename -def gen_pheno_txt_file(this_dataset, genofile_name, vals, trait_filename): +def gen_pheno_txt_file(this_dataset, genofile_name, vals): """Generates phenotype file for GEMMA""" - with open(f"{TEMPDIR}/gn2/{trait_filename}.txt", "w") as outfile: + filename = "PHENO_" + generate_hash_of_string(this_dataset.name + str(vals)).replace("/", "_") + + with open(f"{TEMPDIR}/gn2/{filename}.txt", "w") as outfile: for value in vals: if value == "x": outfile.write("NA\n") else: outfile.write(value + "\n") + return filename + def gen_covariates_file(this_dataset, covariates, samples): covariate_list = covariates.split(",") @@ -168,14 +169,18 @@ def gen_covariates_file(this_dataset, covariates, samples): this_covariate_data.append("-9") covariate_data_object.append(this_covariate_data) + filename = "COVAR_" + generate_hash_of_string(this_dataset.name + str(covariate_data_object)).replace("/", "_") + with open((f"{flat_files('mapping')}/" - f"{this_dataset.group.name}_covariates.txt"), + f"{filename}.txt"), "w") as outfile: for i in range(len(covariate_data_object[0])): for this_covariate in covariate_data_object: outfile.write(str(this_covariate[i]) + "\t") outfile.write("\n") + return filename + def parse_loco_output(this_dataset, gwa_output_filename, loco="True"): diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 09afb8d1..6e816b47 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -20,7 +20,7 @@ logger = utility.logger.getLogger(__name__) GN3_RQTL_URL = "http://localhost:8086/api/rqtl/compute" GN3_TMP_PATH = "/export/local/home/zas1024/genenetwork3/tmp" -def run_rqtl(trait_name, vals, samples, dataset, mapping_scale, model, method, num_perm, perm_strata_list, do_control, control_marker, manhattan_plot, cofactors): +def run_rqtl(trait_name, vals, samples, dataset, pair_scan, mapping_scale, model, method, num_perm, perm_strata_list, do_control, control_marker, manhattan_plot, cofactors): """Run R/qtl by making a request to the GN3 endpoint and reading in the output file(s)""" pheno_file = write_phenotype_file(trait_name, samples, vals, dataset, cofactors, perm_strata_list) @@ -38,6 +38,9 @@ def run_rqtl(trait_name, vals, samples, dataset, mapping_scale, model, method, n "scale": mapping_scale } + if pair_scan: + post_data["pairscan"] = True + if do_control == "true" and control_marker: post_data["control_marker"] = control_marker diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index fcb98db9..7a6636c5 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -109,6 +109,7 @@ class RunMapping: self.mapping_results_path = "{}{}.csv".format( webqtlConfig.GENERATED_IMAGE_DIR, mapping_results_filename) + self.pair_scan = False self.manhattan_plot = False if 'manhattan_plot' in start_vars: if start_vars['manhattan_plot'].lower() != "false": @@ -218,7 +219,7 @@ class RunMapping: elif self.mapping_method == "rqtl_plink": results = self.run_rqtl_plink() elif self.mapping_method == "rqtl_geno": - perm_strata = [] + self.perm_strata = [] if "perm_strata" in start_vars and "categorical_vars" in start_vars: self.categorical_vars = start_vars["categorical_vars"].split( ",") @@ -227,13 +228,13 @@ class RunMapping: sample_names=self.samples, this_trait=self.this_trait) - perm_strata = get_perm_strata( + self.perm_strata = get_perm_strata( self.this_trait, primary_samples, self.categorical_vars, self.samples) self.score_type = "LOD" self.control_marker = start_vars['control_marker'] self.do_control = start_vars['do_control'] - if 'mapmethod_rqtl_geno' in start_vars: - self.method = start_vars['mapmethod_rqtl_geno'] + if 'mapmethod_rqtl' in start_vars: + self.method = start_vars['mapmethod_rqtl'] else: self.method = "em" self.model = start_vars['mapmodel_rqtl_geno'] @@ -242,10 +243,10 @@ class RunMapping: self.pair_scan = True if self.permCheck and self.num_perm > 0: self.perm_output, self.suggestive, self.significant, results = rqtl_mapping.run_rqtl( - self.this_trait.name, self.vals, self.samples, self.dataset, self.mapping_scale, self.model, self.method, self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.covariates) + self.this_trait.name, self.vals, self.samples, self.dataset, self.pair_scan, self.mapping_scale, self.model, self.method, self.num_perm, self.perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.covariates) else: - results = rqtl_mapping.run_rqtl(self.this_trait.name, self.vals, self.samples, self.dataset, self.mapping_scale, self.model, self.method, - self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.covariates) + results = rqtl_mapping.run_rqtl(self.this_trait.name, self.vals, self.samples, self.dataset, self.pair_scan, self.mapping_scale, self.model, self.method, + self.num_perm, self.perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.covariates) elif self.mapping_method == "reaper": if "startMb" in start_vars: # ZS: Check if first time page loaded, so it can default to ON if "additiveCheck" in start_vars: @@ -326,33 +327,8 @@ class RunMapping: self.no_results = True else: if self.pair_scan == True: - self.qtl_results = [] - highest_chr = 1 # This is needed in order to convert the highest chr to X/Y - for marker in results: - if marker['chr1'] > 0 or marker['chr1'] == "X" or marker['chr1'] == "X/Y": - if marker['chr1'] > highest_chr or marker['chr1'] == "X" or marker['chr1'] == "X/Y": - highest_chr = marker['chr1'] - if 'lod_score' in list(marker.keys()): - self.qtl_results.append(marker) - - self.trimmed_markers = results - - for qtl in enumerate(self.qtl_results): - self.json_data['chr1'].append(str(qtl['chr1'])) - self.json_data['chr2'].append(str(qtl['chr2'])) - self.json_data['Mb'].append(qtl['Mb']) - self.json_data['markernames'].append(qtl['name']) - - self.js_data = dict( - json_data=self.json_data, - this_trait=self.this_trait.name, - data_set=self.dataset.name, - maf=self.maf, - manhattan_plot=self.manhattan_plot, - mapping_scale=self.mapping_scale, - qtl_results=self.qtl_results - ) - + self.figure_data = results[0] + self.table_data = results[1] else: self.qtl_results = [] self.results_for_browser = [] @@ -764,9 +740,9 @@ def get_perm_strata(this_trait, sample_list, categorical_vars, used_samples): if sample in list(sample_list.sample_attribute_values.keys()): combined_string = "" for var in categorical_vars: - if var.lower() in sample_list.sample_attribute_values[sample]: + if var in sample_list.sample_attribute_values[sample]: combined_string += str( - sample_list.sample_attribute_values[sample][var.lower()]) + sample_list.sample_attribute_values[sample][var]) else: combined_string += "NA" else: diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index 92cea550..ae30aa59 100644 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -32,7 +32,7 @@ class SampleList: for counter, sample_name in enumerate(sample_names, 1): sample_name = sample_name.replace("_2nd_", "") - # ZS: self.this_trait will be a list if it is a Temp trait + # self.this_trait will be a list if it is a Temp trait if isinstance(self.this_trait, list): sample = webqtlCaseData.webqtlCaseData(name=sample_name) if counter <= len(self.this_trait): @@ -47,7 +47,7 @@ class SampleList: name=sample_name, value=float(self.this_trait[counter - 1])) else: - # ZS - If there's no value for the sample/strain, + # If there's no value for the sample/strain, # create the sample object (so samples with no value # are still displayed in the table) try: @@ -63,29 +63,29 @@ class SampleList: sample.this_id = str(counter) - # ZS: For extra attribute columns; currently only used by + # For extra attribute columns; currently only used by # several datasets if self.sample_attribute_values: sample.extra_attributes = self.sample_attribute_values.get( sample_name, {}) - # ZS: Add a url so RRID case attributes can be displayed as links - if 'rrid' in sample.extra_attributes: + # Add a url so RRID case attributes can be displayed as links + if '36' in sample.extra_attributes: if self.dataset.group.species == "mouse": - if len(sample.extra_attributes['rrid'].split(":")) > 1: - the_rrid = sample.extra_attributes['rrid'].split(":")[ + if len(sample.extra_attributes['36'].split(":")) > 1: + the_rrid = sample.extra_attributes['36'].split(":")[ 1] - sample.extra_attributes['rrid'] = [ - sample.extra_attributes['rrid']] - sample.extra_attributes['rrid'].append( + sample.extra_attributes['36'] = [ + sample.extra_attributes['36']] + sample.extra_attributes['36'].append( webqtlConfig.RRID_MOUSE_URL % the_rrid) elif self.dataset.group.species == "rat": - if len(str(sample.extra_attributes['rrid'])): - the_rrid = sample.extra_attributes['rrid'].split("_")[ + if len(str(sample.extra_attributes['36'])): + the_rrid = sample.extra_attributes['36'].split("_")[ 1] - sample.extra_attributes['rrid'] = [ - sample.extra_attributes['rrid']] - sample.extra_attributes['rrid'].append( + sample.extra_attributes['36'] = [ + sample.extra_attributes['36']] + sample.extra_attributes['36'].append( webqtlConfig.RRID_RAT_URL % the_rrid) self.sample_list.append(sample) @@ -124,17 +124,19 @@ class SampleList: # Get attribute names and distinct values for each attribute results = g.db.execute(''' - SELECT DISTINCT CaseAttribute.Id, CaseAttribute.Name, CaseAttributeXRefNew.Value + SELECT DISTINCT CaseAttribute.Id, CaseAttribute.Name, CaseAttribute.Description, CaseAttributeXRefNew.Value FROM CaseAttribute, CaseAttributeXRefNew WHERE CaseAttributeXRefNew.CaseAttributeId = CaseAttribute.Id AND CaseAttributeXRefNew.InbredSetId = %s - ORDER BY lower(CaseAttribute.Name)''', (str(self.dataset.group.id),)) + ORDER BY CaseAttribute.Id''', (str(self.dataset.group.id),)) self.attributes = {} - for attr, values in itertools.groupby(results.fetchall(), lambda row: (row.Id, row.Name)): - key, name = attr + for attr, values in itertools.groupby(results.fetchall(), lambda row: (row.Id, row.Name, row.Description)): + key, name, description = attr self.attributes[key] = Bunch() + self.attributes[key].id = key self.attributes[key].name = name + self.attributes[key].description = description self.attributes[key].distinct_values = [ item.Value for item in values] self.attributes[key].distinct_values = natural_sort( @@ -168,10 +170,13 @@ class SampleList: for sample_name, items in itertools.groupby(results.fetchall(), lambda row: row.SampleName): attribute_values = {} + # Make a list of attr IDs without values (that have values for other samples) + valueless_attr_ids = [self.attributes[key].id for key in self.attributes.keys()] for item in items: + valueless_attr_ids.remove(item.Id) attribute_value = item.Value - # ZS: If it's an int, turn it into one for sorting + # If it's an int, turn it into one for sorting # (for example, 101 would be lower than 80 if # they're strings instead of ints) try: @@ -179,8 +184,10 @@ class SampleList: except ValueError: pass - attribute_values[self.attributes[item.Id].name.lower( - )] = attribute_value + attribute_values[str(item.Id)] = attribute_value + for attr_id in valueless_attr_ids: + attribute_values[str(attr_id)] = "" + self.sample_attribute_values[sample_name] = attribute_values def get_first_attr_col(self): diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index c07430dd..c947a3b4 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -1,3 +1,5 @@ +from typing import Dict + import string import datetime import uuid @@ -192,6 +194,8 @@ class ShowTrait: [self.dataset.species.chromosomes.chromosomes[this_chr].name, i]) self.genofiles = self.dataset.group.get_genofiles() + study_samplelist_json = self.dataset.group.get_study_samplelists() + self.study_samplelists = [study["title"] for study in study_samplelist_json] # ZS: No need to grab scales from .geno file unless it's using # a mapping method that reads .geno files @@ -277,9 +281,13 @@ class ShowTrait: hddn['species'] = self.dataset.group.species hddn['use_outliers'] = False hddn['method'] = "gemma" + hddn['mapmethod_rqtl'] = "hk" + hddn['mapmodel_rqtl'] = "normal" + hddn['pair_scan'] = "" hddn['selected_chr'] = -1 hddn['mapping_display_all'] = True hddn['suggestive'] = 0 + hddn['study_samplelists'] = json.dumps(study_samplelist_json) hddn['num_perm'] = 0 hddn['categorical_vars'] = "" if categorical_var_list: @@ -295,7 +303,7 @@ class ShowTrait: hddn['compare_traits'] = [] hddn['export_data'] = "" hddn['export_format'] = "excel" - if len(self.scales_in_geno) < 2: + if len(self.scales_in_geno) < 2 and bool(self.scales_in_geno): hddn['mapping_scale'] = self.scales_in_geno[list( self.scales_in_geno.keys())[0]][0][0] @@ -520,6 +528,9 @@ class ShowTrait: sample_group_type='primary', header="%s Only" % (self.dataset.group.name)) self.sample_groups = (primary_samples,) + print("\nttttttttttttttttttttttttttttttttttttttttttttt\n") + print(self.sample_groups) + print("\nttttttttttttttttttttttttttttttttttttttttttttt\n") self.primary_sample_names = primary_sample_names self.dataset.group.allsamples = all_samples_ordered @@ -693,7 +704,7 @@ def get_categorical_variables(this_trait, sample_list) -> list: if len(sample_list.attributes) > 0: for attribute in sample_list.attributes: if len(sample_list.attributes[attribute].distinct_values) < 10: - categorical_var_list.append(sample_list.attributes[attribute].name) + categorical_var_list.append(str(sample_list.attributes[attribute].id)) return categorical_var_list @@ -799,3 +810,41 @@ def get_scales_from_genofile(file_location): return [["physic", "Mb"], ["morgan", "cM"]] else: return [["physic", "Mb"]] + + + +def get_diff_of_vals(new_vals: Dict, trait_id: str) -> Dict: + """ Get the diff between current sample values and the values in the DB + + Given a dict of the changed values and the trait/dataset ID, return a Dict + with keys corresponding to each sample with a changed value and a value + that is a dict with keys for the old_value and new_value + + """ + + trait_name = trait_id.split(":")[0] + dataset_name = trait_id.split(":")[1] + trait_ob = create_trait(name=trait_name, dataset_name=dataset_name) + + old_vals = {sample : trait_ob.data[sample].value for sample in trait_ob.data} + + shared_samples = set.union(set(new_vals.keys()), set(old_vals.keys())) + + diff_dict = {} + for sample in shared_samples: + try: + new_val = round(float(new_vals[sample]), 3) + except: + new_val = "x" + try: + old_val = round(float(old_vals[sample]), 3) + except: + old_val = "x" + + if new_val != old_val: + diff_dict[sample] = { + "new_val": new_val, + "old_val": old_val + } + + return diff_dict diff --git a/wqflask/wqflask/static/new/css/bootstrap-custom.css b/wqflask/wqflask/static/new/css/bootstrap-custom.css index 7c8549e1..a0d3ff6a 100644 --- a/wqflask/wqflask/static/new/css/bootstrap-custom.css +++ b/wqflask/wqflask/static/new/css/bootstrap-custom.css @@ -327,7 +327,7 @@ th { font-family: 'Glyphicons Halflings'; src: url('../fonts/glyphicons-halflings-regular.eot'); - src: url('../fonts/glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'), url('../fonts/glyphicons-halflings-regular.woff') format('woff'), url('../fonts/glyphicons-halflings-regular.ttf') format('truetype'), url('../fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular') format('svg'); + src: url('../fonts/glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'), url('../fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular') format('svg'); } .glyphicon { @@ -7554,5 +7554,3 @@ button.close { display: none !important; } } - -/*# sourceMappingURL=bootstrap.css.map */
\ No newline at end of file diff --git a/wqflask/wqflask/static/new/css/show_trait.css b/wqflask/wqflask/static/new/css/show_trait.css index 782dabc2..b0514e01 100644 --- a/wqflask/wqflask/static/new/css/show_trait.css +++ b/wqflask/wqflask/static/new/css/show_trait.css @@ -260,3 +260,33 @@ input.trait-value-input { div.inline-div { display: inline; } + +/* div.colorbox_border { + border: 1px solid grey; +} */ +div#cboxContent { + /* box-shadow: + 0 2.8px 2.2px rgba(0, 0, 0, 0.034), + 0 6.7px 5.3px rgba(0, 0, 0, 0.048), + 0 12.5px 10px rgba(0, 0, 0, 0.06), + 0 22.3px 17.9px rgba(0, 0, 0, 0.072), + 0 41.8px 33.4px rgba(0, 0, 0, 0.086), + 0 100px 80px rgba(0, 0, 0, 0.12) */ + + padding: 10px 10px 5px 10px; + + -moz-box-shadow: 3px 3px 5px #535353; + -webkit-box-shadow: 3px 3px 5px #535353; + box-shadow: 3px 3px 5px #535353; + + -moz-border-radius: 6px 6px 6px 6px; + -webkit-border-radius: 6px; + border-radius: 6px 6px 6px 6px; + + /* border: 2px solid grey; */ +} + +#cboxClose { + margin-right: 5px; + margin-bottom: 2px; +} diff --git a/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js b/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js index 3e414034..00025a32 100644 --- a/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js +++ b/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js @@ -65,10 +65,8 @@ if ( ! $.fn.DataTable.isDataTable( '#collection_table' ) ) { collection_click = function() { var this_collection_url; - console.log("Clicking on:", $(this)); this_collection_url = $(this).find('.collection_name').prop("href"); this_collection_url += "&json"; - console.log("this_collection_url", this_collection_url); collection_list = $("#collections_holder").html(); return $.ajax({ dataType: "json", @@ -79,32 +77,57 @@ collection_click = function() { submit_click = function() { var covariates_string = ""; - var covariates_display_string = ""; + var covariates_as_set = new Set(); + $(".selected-covariates:first option").each(function() { + if ($(this).val() != ""){ + covariates_as_set.add($(this).val() + "," + $(this).text()); + } + }); $('#collections_holder').find('input[type=checkbox]:checked').each(function() { var this_dataset, this_trait; this_trait = $(this).parents('tr').find('.trait').text(); this_trait_display = $(this).parents('tr').find('.trait').data("display_name"); this_description = $(this).parents('tr').find('.description').text(); - console.log("this_trait is:", this_trait_display); this_dataset = $(this).parents('tr').find('.dataset').data("dataset"); - console.log("this_dataset is:", this_dataset); - covariates_string += this_trait + ":" + this_dataset + "," - //this_covariate_display_string = this_trait + ": " + this_description this_covariate_display_string = this_trait_display if (this_covariate_display_string.length > 50) { this_covariate_display_string = this_covariate_display_string.substring(0, 45) + "..." } - covariates_display_string += this_covariate_display_string + "\n" + covariates_as_set.add(this_trait + ":" + this_dataset + "," + this_covariate_display_string) + }); + + covariates_as_list = Array.from(covariates_as_set) + + // Removed the starting "No covariates selected" option before adding options for each covariate + if (covariates_as_list.length > 0){ + $(".selected-covariates option[value='']").each(function() { + $(this).remove(); + }); + } + + $(".selected-covariates option").each(function() { + $(this).remove(); }); - // Trim the last newline from display_string - covariates_display_string = covariates_display_string.replace(/\n$/, "") - // Trim the last comma - covariates_string = covariates_string.substring(0, covariates_string.length - 1) - //covariates_display_string = covariates_display_string.substring(0, covariates_display_string.length - 2) + covariate_list_for_form = [] + $.each(covariates_as_list, function (index, value) { + option_value = value.split(",")[0] + option_text = value.split(",")[1] + $(".selected-covariates").append($("<option/>", { + value: option_value, + text: option_text + })) + covariate_list_for_form.push(option_value) + }); - $("input[name=covariates]").val(covariates_string) - $(".selected-covariates").val(covariates_display_string) + $("input[name=covariates]").val(covariate_list_for_form.join(",")); + + cofactor_count = $(".selected-covariates:first option").length; + if (cofactor_count > 10){ + $(".selected-covariates").attr("size", 10); + } else { + $(".selected-covariates").attr("size", cofactor_count); + } return $.colorbox.close(); }; @@ -186,9 +209,8 @@ color_by_trait = function(trait_sample_data, textStatus, jqXHR) { process_traits = function(trait_data, textStatus, jqXHR) { var the_html, trait, _i, _len; console.log('in process_traits with trait_data:', trait_data); - the_html = "<button id='back_to_collections' class='btn btn-inverse btn-small'>"; - the_html += "<i class='icon-white icon-arrow-left'></i> Back </button>"; - the_html += " <button id='submit' class='btn btn-primary btn-small'> Submit </button>"; + the_html = "<button class='btn btn-success btn-small submit'> Submit </button>"; + the_html += "<button id='back_to_collections' class='btn btn-inverse btn-small' style='float: right;'>Back</button>"; the_html += "<table id='collection_table' style='padding-top: 10px;' class='table table-hover'>"; the_html += "<thead><tr><th></th><th>Record</th><th>Data Set</th><th>Description</th></tr></thead>"; the_html += "<tbody>"; @@ -221,6 +243,6 @@ back_to_collections = function() { }; $(".collection_line").on("click", collection_click); -$("#submit").on("click", submit_click); +$(".submit").on("click", submit_click); $(".trait").on("click", trait_click); -$("#back_to_collections").on("click", back_to_collections);
\ No newline at end of file +$("#back_to_collections").on("click", back_to_collections); diff --git a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js index 6ca92fb6..4de1b0ac 100644 --- a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js +++ b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js @@ -93,15 +93,15 @@ build_columns = function() { ); } - attr_keys = Object.keys(js_data.attributes).sort((a, b) => (js_data.attributes[a].name.toLowerCase() > js_data.attributes[b].name.toLowerCase()) ? 1 : -1) + attr_keys = Object.keys(js_data.attributes).sort((a, b) => (js_data.attributes[a].id > js_data.attributes[b].id) ? 1 : -1) for (i = 0; i < attr_keys.length; i++){ column_list.push( { - 'title': "<div style='text-align: " + js_data.attributes[attr_keys[i]].alignment + "'>" + js_data.attributes[attr_keys[i]].name + "</div>", + 'title': "<div title='" + js_data.attributes[attr_keys[i]].description + "' style='text-align: " + js_data.attributes[attr_keys[i]].alignment + "'>" + js_data.attributes[attr_keys[i]].name + "</div>", 'type': "natural", 'data': null, 'render': function(data, type, row, meta) { - attr_name = Object.keys(data.extra_attributes).sort()[meta.col - data.first_attr_col] + attr_name = Object.keys(data.extra_attributes).sort((a, b) => (parseInt(a) > parseInt(b)) ? 1 : -1)[meta.col - data.first_attr_col] if (attr_name != null && attr_name != undefined){ if (Array.isArray(data.extra_attributes[attr_name])){ diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index 77ef1720..f050d4ae 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -98,11 +98,54 @@ sample_group_types = js_data.sample_group_types; $(".select_covariates").click(function () { open_covariate_selection(); }); + $(".remove_covariates").click(function () { - $("input[name=covariates]").val("") - $(".selected-covariates").val("") + $(".selected-covariates option:selected").each(function() { + this_val = $(this).val(); + $(".selected-covariates option").each(function(){ + if ($(this).val() == this_val){ + $(this).remove(); + } + }) + cofactor_count = $(".selected-covariates:first option").length + if (cofactor_count > 2 && cofactor_count < 11){ + $(".selected-covariates").each(function() { + $(this).attr("size", $(".selected-covariates:first option").length) + }); + } else if (cofactor_count > 10) { + $(".selected-covariates").each(function() { + $(this).attr("size", 10) + }); + } else { + $(".selected-covariates").each(function() { + $(this).attr("size", 2) + }); + } + if (cofactor_count == 0){ + $(".selected-covariates").each(function() { + $(this).append($("<option/>", { + value: "", + text: "No covariates selected" + })) + }) + } + }); + + covariates_list = []; + $(".selected-covariates:first option").each(function() { + covariates_list.push($(this).val()); + }) + $("input[name=covariates]").val(covariates_list.join(",")) }); +$(".remove_all_covariates").click(function() { + $(".selected-covariates option").each(function() { + $(this).remove(); + }); + $(".selected-covariates").attr("size", 2) + $("input[name=covariates]").val(""); +}) + open_trait_selection = function() { return $('#collections_holder').load('/collections/list?color_by_trait #collections_list', (function(_this) { return function() { @@ -608,13 +651,14 @@ $(".corr_compute").on("click", (function(_this) { create_value_dropdown = function(value) { return "<option val=" + value + ">" + value + "</option>"; }; + populate_sample_attributes_values_dropdown = function() { var attribute_info, key, sample_attributes, selected_attribute, value, _i, _len, _ref, _ref1, _results; $('#attribute_values').empty(); sample_attributes = []; var attributes_as_list = Object.keys(js_data.attributes).map(function(key) { - return [key, js_data.attributes[key].name.toLowerCase()]; + return [key, js_data.attributes[key].id]; }); attributes_as_list.sort(function(first, second) { @@ -628,7 +672,7 @@ populate_sample_attributes_values_dropdown = function() { }); for (i=0; i < attributes_as_list.length; i++) { - attribute_info = js_data.attributes[attributes_as_list[i][0]] + attribute_info = js_data.attributes[attributes_as_list[i][1]] sample_attributes.push(attribute_info.distinct_values); } @@ -667,11 +711,13 @@ block_by_attribute_value = function() { let exclude_val_nodes = table_api.column(attribute_start_pos + parseInt(exclude_column)).nodes().to$(); for (i = 0; i < exclude_val_nodes.length; i++) { - let this_col_value = exclude_val_nodes[i].childNodes[0].data; - let this_val_node = val_nodes[i].childNodes[0]; + if (exclude_val_nodes[i].hasChildNodes()) { + let this_col_value = exclude_val_nodes[i].childNodes[0].data; + let this_val_node = val_nodes[i].childNodes[0]; - if (this_col_value == exclude_by_value){ - this_val_node.value = "x"; + if (this_col_value == exclude_by_value){ + this_val_node.value = "x"; + } } } @@ -713,10 +759,34 @@ block_by_index = function() { for (_k = 0, _len1 = index_list.length; _k < _len1; _k++) { index = index_list[_k]; val_nodes[index - 1].childNodes[0].value = "x"; - } }; +filter_by_study = function() { + let this_study = $('#filter_study').val(); + + let study_sample_data = JSON.parse($('input[name=study_samplelists]').val()) + let filter_samples = study_sample_data[parseInt(this_study)]['samples'] + + if ($('#filter_study_group').length){ + let block_group = $('#filter_study_group').val(); + if (block_group === "other") { + table_api = $('#samples_other').DataTable(); + } else { + table_api = $('#samples_primary').DataTable(); + } + } + + let sample_nodes = table_api.column(2).nodes().to$(); + let val_nodes = table_api.column(3).nodes().to$(); + for (i = 0; i < sample_nodes.length; i++) { + this_sample = sample_nodes[i].childNodes[0].innerText; + if (!filter_samples.includes(this_sample)){ + val_nodes[i].childNodes[0].value = "x"; + } + } +} + filter_by_value = function() { let filter_logic = $('#filter_logic').val(); let filter_column = $('#filter_column').val(); @@ -748,7 +818,7 @@ filter_by_value = function() { var this_col_value = filter_val_nodes[i].childNodes[0].value; } else { if (filter_val_nodes[i].childNodes[0] !== undefined){ - var this_col_value = filter_val_nodes[i].childNodes[0].data; + var this_col_value = filter_val_nodes[i].innerText; } else { continue } @@ -1690,6 +1760,11 @@ $('#block_by_index').click(function(){ edit_data_change(); }); +$('#filter_by_study').click(function(){ + filter_by_study(); + edit_data_change(); +}) + $('#filter_by_value').click(function(){ filter_by_value(); edit_data_change(); diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js index e457fa4a..4f994eae 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js +++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js @@ -145,7 +145,7 @@ var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form 'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars', 'num_bootstrap', 'bootCheck', 'bootstrap_results', 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker', 'do_control', 'genofile', 'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', 'haplotypeAnalystCheck', - 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'reaper_version', 'primary_samples'] + 'mapmethod_rqtl', 'mapmodel_rqtl', 'temp_trait', 'group', 'species', 'reaper_version', 'primary_samples'] $(".rqtl-geno-tab, #rqtl_geno_compute").on("click", (function(_this) { return function() { @@ -156,6 +156,8 @@ $(".rqtl-geno-tab, #rqtl_geno_compute").on("click", (function(_this) { $('input[name=selected_chr]').val($('#chr_rqtl_geno').val()); $('input[name=mapping_scale]').val($('#scale_rqtl_geno').val()); $('input[name=genofile]').val($('#genofile_rqtl_geno').val()); + $('input[name=mapmodel_rqtl]').val($('#mapmodel_rqtl_geno').val()); + $('input[name=mapmethod_rqtl]').val($('#mapmethod_rqtl_geno').val()); $('input[name=num_perm]').val($('input[name=num_perm_rqtl_geno]').val()); $('input[name=categorical_vars]').val(js_data.categorical_vars) $('input[name=manhattan_plot]').val($('input[name=manhattan_plot_rqtl]:checked').val()); @@ -177,12 +179,14 @@ $(".rqtl-pair-tab, #rqtl_pair_compute").on("click", (function(_this) { var form_data, url; url = "/loading"; $('input[name=method]').val("rqtl_geno"); - $('input[name=pair_scan]').val(""); - $('input[name=genofile]').val($('#genofile_rqtl_geno').val()); - $('input[name=num_perm]').val($('input[name=num_perm_rqtl_geno]').val()); + $('input[name=pair_scan]').val("true"); + $('input[name=genofile]').val($('#genofile_rqtl_pair').val()); + $('input[name=mapmodel_rqtl]').val($('#mapmodel_rqtl_pair').val()); + $('input[name=mapmethod_rqtl]').val($('#mapmethod_rqtl_pair').val()); + $('input[name=num_perm]').val($('input[name=num_perm_rqtl_pair]').val()); $('input[name=categorical_vars]').val(js_data.categorical_vars) - $('input[name=control_marker]').val($('input[name=control_rqtl_geno]').val()); - $('input[name=do_control]').val($('input[name=do_control_rqtl]:checked').val()); + $('input[name=control_marker]').val($('input[name=control_rqtl_pair]').val()); + $('input[name=do_control]').val($('input[name=do_control_rqtl_pair]:checked').val()); $('input[name=tool_used]').val("Mapping"); $('input[name=form_url]').val("/run_mapping"); $('input[name=wanted_inputs]').val(mapping_input_list.join(",")); @@ -259,25 +263,25 @@ $("#use_composite_choice").change(composite_mapping_fields); $("#mapping_method_choice").change(mapping_method_fields); -$("#mapmodel_rqtl_geno").change(function() { +$("#mapmodel_rqtl_geno,#mapmodel_rqtl_pair").change(function() { if ($(this).val() == "np"){ $("#mapmethod_rqtl_geno").attr('disabled', 'disabled'); $("#mapmethod_rqtl_geno").css('background-color', '#CCC'); - $("#missing_geno").attr('disabled', 'disabled'); - $("#missing_geno").css('background-color', '#CCC'); + $("#missing_geno,#missing_geno_pair").attr('disabled', 'disabled'); + $("#missing_geno,#missing_geno_pair").css('background-color', '#CCC'); } else { $("#mapmethod_rqtl_geno").removeAttr('disabled'); $("#mapmethod_rqtl_geno").css('background-color', '#FFF'); - $("#missing_geno").removeAttr('disabled'); - $("#missing_geno").css('background-color', '#FFF'); + $("#missing_geno,#missing_geno_pair").removeAttr('disabled'); + $("#missing_geno,#missing_geno_pair").css('background-color', '#FFF'); } }); -$("#mapmethod_rqtl_geno").change(function() { +$("#mapmethod_rqtl_geno,#mapmethod_rqtl_pair").change(function() { if ($(this).val() == "mr"){ - $("#missing_geno_div").css('display', 'block'); + $("#missing_geno_div,#missing_geno_pair_div").css('display', 'block'); } else { - $("#missing_geno_div").css('display', 'none'); + $("#missing_geno_div,#missing_geno_pair_div").css('display', 'none'); } }); diff --git a/wqflask/wqflask/templates/collections/add.html b/wqflask/wqflask/templates/collections/add.html index 0398c6e4..8640fdb8 100644 --- a/wqflask/wqflask/templates/collections/add.html +++ b/wqflask/wqflask/templates/collections/add.html @@ -5,7 +5,7 @@ or add to an existing collection.</p> </div> <div class="modal-body" style="margin-left: 20px;"> - <form action="/collections/new" target="_blank" data-validate="parsley" id="add_form"> + <form action="/collections/new" target="_blank" data-validate="parsley" id="add_form" class="form-inline"> {% if traits is defined %} <input type="hidden" name="traits" value="{{ traits }}" /> {% else %} @@ -14,10 +14,8 @@ {% if collections|length > 0 %} <fieldset> <legend>1. Add to an existing collection</legend> - <div style="margin-left: 20px;"> - <!--<label>Existing collection name:</label>--> - <select name="existing_collection" class="form-control"> - <!--<option selected disabled>Select Collection</option>--> + <div style="margin-left: 20px;"> + <select name="existing_collection" class="form-control" style="width: 80%;"> {% for col in collections %} {% if loop.index == 1 %} <option value="{{ col.id }}:{{ col.name }}" selected>{{ col.name }}</option> @@ -26,8 +24,9 @@ {% endif %} {% endfor %} </select> - <br /> - <button type="submit" name="add_to_existing" class="btn btn-primary">Add to existing collection</button> + <input type="button" style="display: inline;" id="make_default" value="Make Default"> + <br><br> + <button type="submit" name="add_to_existing" class="btn btn-primary">Add</button> </div> </fieldset> {% endif %} @@ -35,7 +34,6 @@ <fieldset> <legend>{% if collections|length > 0 %}2. {% else %}{% endif %}Create a new collection</legend> <div style="margin-left: 20px;"> - <!--<label>Collection name:</label>--> <input type="text" name="new_collection" placeholder=" Name of new collection..." data-trigger="change" data-minlength="5" data-maxlength="50" style="width: 100%"> <button type="submit" name="create_new" class="btn btn-primary" style="margin-top: 20px;">Create collection</button> @@ -54,6 +52,21 @@ parent.jQuery.colorbox.close(); }); + make_default = function() { + alert("The current collection is now your default collection.") + let uc_id = $('[name=existing_collection] option:selected').val().split(":")[0] + $.cookie('default_collection', uc_id, { + expires: 365, + path: '/' + }); + + let default_collection_id = $.cookie('default_collection'); + }; + + $("#make_default").on("click", function(){ + make_default(); + }); + apply_default = function() { let default_collection_id = $.cookie('default_collection'); if (default_collection_id) { diff --git a/wqflask/wqflask/templates/display_files_admin.html b/wqflask/wqflask/templates/display_files_admin.html new file mode 100644 index 00000000..4b4babc4 --- /dev/null +++ b/wqflask/wqflask/templates/display_files_admin.html @@ -0,0 +1,32 @@ +{% extends "base.html" %} +{% block title %}Trait Submission{% endblock %} +{% block content %} +<!-- Start of body --> +{% with messages = get_flashed_messages(with_categories=true) %} +{% if messages %} +{% for category, message in messages %} +<div class="container-fluid bg-{{ category }}"> + <p>{{ message }}</p> +</div> +{% endfor %} +{% endif %} +{% endwith %} +Show files for approval + +<div> + <ul> + {% for file in files %} + <li><a href="/display-file/{{ file }}" target="_blank">{{ file }}</a><br/> + <button><a href="/data-samples/approve/{{ file }}">Approve</a></button> + <button><a href="/data-samples/reject/{{ file }}">Reject</a></button></li> + {% endfor %} + </ul> +</div> +{%endblock%} + +{% block js %} +<script> + gn_server_url = "{{ gn_server_url }}"; + +</script> +{% endblock %} diff --git a/wqflask/wqflask/templates/display_files_user.html b/wqflask/wqflask/templates/display_files_user.html new file mode 100644 index 00000000..b6bab709 --- /dev/null +++ b/wqflask/wqflask/templates/display_files_user.html @@ -0,0 +1,31 @@ +{% extends "base.html" %} +{% block title %}Trait Submission{% endblock %} +{% block content %} +<!-- Start of body --> +{% with messages = get_flashed_messages(with_categories=true) %} +{% if messages %} +{% for category, message in messages %} +<div class="container-fluid bg-{{ category }}"> + <p>{{ message }}</p> +</div> +{% endfor %} +{% endif %} +{% endwith %} +Show files for approval + +<div> + <ul> + {% for file in files %} + <li><a href="/display-file/{{ file }}" target="_blank">{{ file }}</a><br/> + <button><a href="/data-samples/reject/{{ file }}">Reject</a></button></li> + {% endfor %} + </ul> +</div> +{%endblock%} + +{% block js %} +<script> + gn_server_url = "{{ gn_server_url }}"; + +</script> +{% endblock %} diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 7d4c65f8..7a841793 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -2,8 +2,18 @@ {% block title %}Trait Submission{% endblock %} {% block content %} <!-- Start of body --> -Edit Trait for Published Database -Submit Trait | Reset +{% with messages = get_flashed_messages(with_categories=true) %} +{% if messages %} +{% for category, message in messages %} +<div class="container-fluid bg-{{ category }}"> + <p>{{ message }}</p> +</div> +{% endfor %} +{% endif %} +{% endwith %} +<div class="page-header text-center"> + <h1>Edit Trait for Published Database</h1> +</div> {% if diff %} @@ -53,7 +63,7 @@ Submit Trait | Reset {% endif %} -<form id="edit-form" class="form-horizontal" method="post" action="/trait/update"> +<form id="edit-form" class="form-horizontal" method="post" action="/trait/update" enctype=multipart/form-data> <h2 class="text-center">Trait Information:</h2> <div class="form-group"> <label for="pubmed-id" class="col-sm-2 control-label">Pubmed ID:</label> @@ -207,10 +217,18 @@ Submit Trait | Reset <input name="old_sequence" class="changed" type="hidden" value="{{ publication.sequence |default('', true) }}"/> </div> </div> - <div class="controls" style="display:block; margin-left: 40%; margin-right: 20%;"> + <div style="margin-left: 13%;"> + <a href="/trait/{{ publish_xref.id_ }}/sampledata/{{ publish_xref.phenotype_id }}" class="btn btn-link btn-sm"> + Sample Data(CSV Download) + </a> + </div> + <div class="form-group"> + <input type = "file" class="col-sm-4 control-label" name = "file" /> + </div> + <div class="controls center-block" style="width: max-content;"> <input name="dataset-name" class="changed" type="hidden" value="{{ publish_xref.id_ }}"/> - <input name="phenotype-id" class="changed" type="hidden" value="{{ publish_xref.phenotype_id }}"/> <input name="inbred-set-id" class="changed" type="hidden" value="{{ publish_xref.inbred_set_id }}"/> + <input name="phenotype-id" class="changed" type="hidden" value="{{ publish_xref.phenotype_id }}"/> <input name="comments" class="changed" type="hidden" value="{{ publish_xref.comments }}"/> <input type="submit" style="width: 125px; margin-right: 25px;" class="btn btn-primary form-control col-xs-2 changed" value="Submit Change"> <input type="reset" style="width: 110px;" class="btn btn-primary form-control col-xs-2 changed" onClick="window.location.reload();" value="Reset"> diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 6d6136ac..1edde31e 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -25,6 +25,8 @@ <br> transformation = <b><i>{{ start_vars.transform }}</i></b> {% endif %} + <br> + hash of sample values = <b><i>{{ start_vars.vals_hash }}</i></b> <br><br> <b>Mapping Metadata</b> <br> @@ -68,6 +70,29 @@ <div style="text-align: center;"> <img align="center" src="/static/gif/89.gif"> </div> + {% if start_vars.vals_diff|length != 0 and start_vars.transform == "" %} + <br><br> + <button id="show_full_diff">Show Full Diff</button> + <br> + <div id="diff_table_container" style="display: none; height:200px; overflow:auto;"> + <table class="table table-hover"> + <thead> + <th>Sample</th> + <th>New Value</th> + <th>Old Value</th> + </thead> + <tbody> + {% for sample in start_vars.vals_diff %} + <tr> + <td>{{ sample }}</td> + <td>{{ start_vars.vals_diff[sample].new_val }}</td> + <td>{{ start_vars.vals_diff[sample].old_val }}</td> + </tr> + {% endfor %} + </tbody> + </table> + </div> + {% endif %} </div> </div> </div> @@ -76,7 +101,14 @@ <script src="{{ url_for('js', filename='jquery/jquery.min.js') }}" type="text/javascript"></script> <script src="{{ url_for('js', filename='bootstrap/js/bootstrap.min.js') }}" type="text/javascript"></script> <script type="text/javascript"> - $("#loading_form").attr("action", "{{ start_vars.form_url }}"); setTimeout(function(){ $("#loading_form").submit()}, 350); + +$('#show_full_diff').click(function() { + if ($('#diff_table_container').is(':visible')){ + $('#diff_table_container').hide(); + } else { + $('#diff_table_container').show(); + } +}) </script> diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index 35d8a157..81eb1ba1 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -44,7 +44,12 @@ {% endif %} <input type="hidden" name="num_perm" value="{{ nperm }}"> <input type="hidden" name="perm_info" value=""> - <input type="hidden" name="perm_strata" value="{{ perm_strata }}"> + {% if categorical_vars is defined %} + <input type="hidden" name="categorical_vars" value="{{ categorical_vars|join(',') }}"> + {% endif %} + {% if perm_strata is defined %} + <input type="hidden" name="perm_strata" value="True"> + {% endif %} <input type="hidden" name="num_bootstrap" value="{{ nboot }}"> <input type="hidden" name="do_control" value="{{ doControl }}"> <input type="hidden" name="control_marker" value="{{ controlLocus }}"> diff --git a/wqflask/wqflask/templates/pair_scan_results.html b/wqflask/wqflask/templates/pair_scan_results.html index fb825b90..43c753e2 100644 --- a/wqflask/wqflask/templates/pair_scan_results.html +++ b/wqflask/wqflask/templates/pair_scan_results.html @@ -1,70 +1,128 @@ {% extends "base.html" %} {% block title %}Pair Scan{% endblock %} {% block css %} - <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTables/css/jquery.dataTables.css') }}" /> - <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='d3-tip/d3-tip.css') }}" /> - <link rel="stylesheet" type="text/css" href="/static/new/css/panelutil.css" /> +<link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTables/css/jquery.dataTables.css') }}" /> +<link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='d3-tip/d3-tip.css') }}" /> +<link rel="stylesheet" type="text/css" href="/static/new/css/pair_scan.css" /> +<link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" /> +<link rel="stylesheet" type="text/css" href="/static/new/css/d3panels.css" /> {% endblock %} {% block content %} <!-- Start of body --> - {{ header("Mapping", - '{}: {}'.format(this_trait.name, this_trait.description_fmt)) }} - - <div class="container"> - <div> - <h2> - Pair Scan - </h2> - </div> - <div id="chart_container"> - <div class="pair_scan_figure" id="pair_scan_figure"> - <a href="/tmp/{{ pair_scan_filename }}"> - <img alt="Embedded Image" src="data:image/png;base64, - {% for elem in pair_scan_array -%} - {% print("%c"|format(elem)) %} - {%- endfor %} - " /></a> - </div> - </div> - <div> - <h2> - Results - </h2> - <table cellpadding="0" cellspacing="0" border="0" id="pair_scan_results" class="table table-hover table-striped table-bordered"> - <thead> - <tr> - <td>Index</td> - <td>Locus</td> - <td>Chr 1</td> - <td>Mb</td> - <td>Chr 2</td> - </tr> - </thead> - <tbody> - {% for marker in trimmed_markers %} - <tr> - <td>{{loop.index}}</td> - <td>{{marker.name}}</td> - <td>{{marker.chr1}}</td> - <td>{{marker.Mb}}</td> - <td>{{marker.chr2}}</td> - </tr> - {% endfor %} - </tbody> - </table> - </div> +{{ header("Mapping", + '{}: {}'.format(this_trait.name, this_trait.description_fmt)) }} + +<div id="main_div" class="container"> + <div> + <h2> + Pair Scan + </h2> + </div> + <div class="qtlcharts" id="chart_container"> + <div id="pairscan_chart"></div> </div> + <div style="width: 1100px;"> + <h2> + Results + </h2> + <table cellpadding="0" cellspacing="0" border="0" id="pair_scan_results" class="table table-hover table-striped table-bordered"> + <thead> + <tr> + <th colspan="3">Interval 1</th> + <th rowspan="3">LOD</th> + <th colspan="3">Interval 2</th> + </tr> + <tr> + <th rowspan="2">Position</th> + <th colspan="2">Flanking Markers</th> + <th rowspan="2">Position</th> + <th colspan="2">Flanking Markers</th> + </tr> + <tr> + <th>Proximal</th> + <th>Distal</th> + <th>Proximal</th> + <th>Distal</th> + </tr> + </thead> + <tbody> + {% for row in table_data %} + <tr> + <td>{{ row.pos1 }}</td> + <td>{{ row.proximal1 }}</td> + <td>{{ row.distal1 }}</td> + <td>{{ row.lod }}</td> + <td>{{ row.pos2 }}</td> + <td>{{ row.proximal2 }}</td> + <td>{{ row.distal2 }}</td> + </tr> + {% endfor %} + </tbody> + </table> + </div> +</div> {% endblock %} {% block js %} - <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='d3js/d3.min.js') }}"></script> - <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='d3-tip/d3-tip.js') }}"></script> - <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.js') }}"></script> - <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.dataTables.min.js') }}"></script> - <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/plugins/sorting/scientific.js') }}"></script> - <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='js_alt/underscore.min.js') }}"></script> +<script> + var figure_data = {{ figure_data | safe }} +</script> + +<script src="https://d3js.org/d3.v7.min.js"></script> +<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.js') }}"></script> +<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.dataTables.min.js') }}"></script> +<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/plugins/sorting/scientific.js') }}"></script> +<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/scroller/js/dataTables.scroller.min.js') }}"></script> +<script language="javascript" type="text/javascript" src="/static/new/javascript/d3panels.js"></script> + +<script type="text/javascript"> + +var data, mychart; + +// d3.json("data.json").then(function(data) { +// var mychart; +// mychart = d3panels.lod2dheatmap({ +// altrectcolor: "", +// chrlinecolor: "black", +// equalCells: true +// }); +// return mychart(d3.select("div#pairscan_chart"), data); +// }); -{% endblock %}
\ No newline at end of file +// d3.json("data.json").then(function(data) { +// var mychart; +// mychart = d3panels.lod2dheatmap({ +// oneAtTop: true, +// altrectcolor: "", +// chrlinecolor: "black", +// colors: ["crimson", "white", "slateblue"], +// equalCells: true +// }); +// data.poslabel = data.marker; +// return mychart(d3.select("div#chart2"), data); +// }); + +mychart = d3panels.lod2dheatmap({ + equalCells: true +}); + +mychart(d3.select('div#pairscan_chart'), figure_data); + +table_conf = { + "sDom": "itir", + "autoWidth": true, + "bSortClasses": false, + "order": [[3, "desc" ]], + "scrollY": "100vh", + "scroller": true, + "scrollCollapse": true + } + +trait_table = $('#pair_scan_results').DataTable(table_conf); + +</script> + +{% endblock %} diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index bb30c54c..53e16aa0 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -236,7 +236,7 @@ <button type="button" id="view_in_gn1" class="btn btn-primary" title="View Trait in GN1" onclick="window.open('http://gn1.genenetwork.org/webqtl/main.py?cmd=show&db={{ this_trait.dataset.name }}&probeset={{ this_trait.name }}', '_blank')">Go to GN1</button> {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} {% if this_trait.dataset.type == 'Publish' %} - <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('/trait/{{ this_trait.name }}/edit/{{ this_trait.dataset.id }}', '_blank')">Edit</button> + <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('/trait/{{ this_trait.name }}/edit/inbredset-id/{{ this_trait.dataset.id }}', '_blank')">Edit</button> {% endif %} {% if this_trait.dataset.type == 'ProbeSet' %} diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index a32c45fb..80bc6509 100755 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -77,17 +77,20 @@ No collections available. Please add traits to a collection to use them as covariates. {% else %} <div class="select-covar-div"> - <button type="button" class="btn btn-default select-covar-button select_covariates">Select</button> + <button type="button" class="btn btn-success select-covar-button select_covariates">Select</button> <button type="button" class="btn btn-default select-covar-button remove_covariates">Remove</button> + <button type="button" class="btn btn-danger select-covar-button remove_all_covariates">Clear</button> </div> - <textarea rows="3" cols="50" readonly placeholder="No covariates selected" class="selected-covariates"></textarea> + <select size="2" name="selected_covariates_gemma" class="form-control selected-covariates" multiple> + <option value="">No covariates selected</option> + </select> {% endif %} </div> </div> <div class="mapping_method_fields form-group"> <label class="col-xs-3 control-label"></label> <div class="col-xs-6"> - <button id="gemma_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression" value="Compute">Compute</button> + <button id="gemma_compute" type="button" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression" value="Compute">Compute</button> </div> </div> </div> @@ -190,7 +193,7 @@ <div class="mapping_method_fields form-group"> <label class="col-xs-3 control-label"></label> <div class="col-xs-6"> - <button id="interval_mapping_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Interval Mapping" value="Compute">Compute</button> + <button id="interval_mapping_compute" type="button" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Interval Mapping" value="Compute">Compute</button> </div> </div> </div> @@ -229,6 +232,17 @@ </select> </div> </div> + {% else %} + <div class="mapping_method_fields form-group"> + <label for="scale_select" class="col-xs-3 control-label">Map Scale</label> + <div class="col-xs-2 controls"> + <select id="scale_rqtl_geno" class="form-control scale-select"> + {% for item in scales_in_geno[dataset.group.name + ".geno"] %} + <option value="{{ item[0] }}">{{ item[1] }}</option> + {% endfor %} + </select> + </div> + </div> {% endif %} <div class="mapping_method_fields form-group"> <label for="mapping_permutations" class="col-xs-3 control-label">Permutations</label> @@ -320,6 +334,113 @@ No collections available. Please add traits to a collection to use them as covariates. {% else %} <div class="select-covar-div"> + <button type="button" class="btn btn-success select-covar-button select_covariates">Select</button> + <button type="button" class="btn btn-default select-covar-button remove_covariates">Remove</button> + <button type="button" class="btn btn-danger select-covar-button remove_all_covariates">Clear</button> + </div> + <select size="2" name="selected_covariates_rqtl" class="form-control selected-covariates" multiple> + <option value="">No covariates selected</option> + </select> + {% endif %} + </div> + </div> + <div class="mapping_method_fields form-group"> + <label class="col-xs-3 control-label"></label> + <div class="col-xs-6 controls"> + <button id="rqtl_geno_compute" type="button" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression" value="Compute">Compute</button> + </div> + </div> + </div> + </div> + <div class="tab-pane {% if dataset.group.mapping_id == '3' %}active{% endif %}" id="rqtl_pair"> + <div class="form-horizontal section-form-div"> + {% if genofiles and genofiles|length > 0 %} + <div class="mapping_method_fields form-group"> + <label for="genofiles" class="col-xs-3 control-label">Genotypes</label> + <div class="col-xs-6 controls"> + <select id="genofile_rqtl_pair" class="form-control"> + {% for item in genofiles %} + <option value="{{item['location']}}:{{item['title']}}">{{item['title']}}</option> + {% endfor %} + </select> + </div> + </div> + {% endif %} + <div class="mapping_method_fields form-group"> + <label for="mapping_permutations" class="col-xs-3 control-label">Permutations</label> + <div class="col-xs-4 controls"> + <input name="num_perm_rqtl_pair" value="200" type="text" class="form-control"> + </div> + </div> + {% if sample_groups[0].attributes|length > 0 %} + <div class="mapping_method_fields form-group"> + <label class="col-xs-3 control-label">Stratified</label> + <div class="col-xs-6 controls"> + <label class="radio-inline"> + <input type="radio" name="perm_strata" value="True" checked=""> + Yes + </label> + <label class="radio-inline"> + <input type="radio" name="perm_strata" value="False" > + No + </label> + </div> + </div> + {% endif %} + <div class="mapping_method_fields form-group"> + <label for="control_for" class="col-xs-3 control-label">Control for</label> + <div class="col-xs-6 controls"> + <input name="control_rqtl_pair" value="{% if dataset.type == 'ProbeSet' and this_trait.locus_chr != '' %}{{ nearest_marker }}{% endif %}" type="text" class="form-control cofactor-input" /> + <label class="radio-inline"> + <input type="radio" name="do_control_rqtl" value="true"> + Yes + </label> + <label class="radio-inline"> + <input type="radio" name="do_control_rqtl" value="false" checked=""> + No + </label> + </div> + </div> + <div class="mapping_method_fields form-group"> + <label for="mapmodel_rqtl_pair" class="col-xs-3 control-label">Model</label> + <div class="col-xs-4 controls"> + <select id="mapmodel_rqtl_pair" name="mapmodel_rqtl_pair" class="form-control"> + <option value="normal">Normal</option> + {% if binary == "true" %}<option value="binary">Binary</option>{% endif %} + <!--<option value="2part">2-part</option>--> + <option value="np">Non-parametric</option> + </select> + </div> + </div> + <div class="mapping_method_fields form-group"> + <label for="mapmethod_rqtl_pair" class="col-xs-3 control-label">Method</label> + <div class="col-xs-6 controls"> + <select id="mapmethod_rqtl_pair" name="mapmethod_rqtl_pair" class="form-control"> + <option value="hk" selected>Haley-Knott</option> + <option value="ehk">Extended Haley-Knott</option> + <option value="mr">Marker Regression</option> + <option value="em">Expectation-Maximization</option> + <option value="imp">Imputation</option> + </select> + </div> + </div> + <div id="missing_geno_pair_div" class="mapping_method_fields form-group" style="display: none;"> + <label for="missing_genotypes_pair" class="col-xs-3 control-label"></label> + <div class="col-xs-6 controls"> + <select id="missing_genotype_pair" name="missing_genotypes" class="form-control"> + <option value="mr">Remove Samples w/o Genotypes</option> + <option value="mr-imp">Single Imputation</option> + <option value="mr-argmax">Imputation w/ Viterbi Algorithm</option> + </select> + </div> + </div> + <div class="mapping_method_fields form-group"> + <label class="col-xs-3 control-label">Covariates<br><span class="covar-text">Select covariate(s) from a collection</span></label> + <div class="col-xs-8 covar-options"> + {% if g.user_session.num_collections < 1 %} + No collections available. Please add traits to a collection to use them as covariates. + {% else %} + <div class="select-covar-div"> <button type="button" class="btn btn-default select-covar-button select_covariates">Select</button> <button type="button" class="btn btn-default select-covar-button remove_covariates">Remove</button> </div> @@ -330,7 +451,7 @@ <div class="mapping_method_fields form-group"> <label class="col-xs-3 control-label"></label> <div class="col-xs-6 controls"> - <button id="rqtl_geno_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression" value="Compute">Compute</button> + <button id="rqtl_pair_compute" type="button" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Pair Scan" value="Compute">Compute</button> </div> </div> </div> diff --git a/wqflask/wqflask/templates/show_trait_transform_and_filter.html b/wqflask/wqflask/templates/show_trait_transform_and_filter.html index 20f78b48..5e6ed2cf 100644 --- a/wqflask/wqflask/templates/show_trait_transform_and_filter.html +++ b/wqflask/wqflask/templates/show_trait_transform_and_filter.html @@ -25,7 +25,7 @@ <label for="exclude_column">Block samples by group:</label> <select id="exclude_column" size=1> {% for attribute in sample_groups[0].attributes %} - {% if sample_groups[0].attributes[attribute].distinct_values|length <= 10 %} + {% if sample_groups[0].attributes[attribute].distinct_values|length <= 10 and sample_groups[0].attributes[attribute].distinct_values|length > 1 %} <option value="{{ loop.index }}"> {{ sample_groups[0].attributes[attribute].name }} </option> @@ -45,6 +45,27 @@ <input type="button" id="exclude_by_attr" class="btn btn-danger" value="Block"> </div> {% endif %} + {% if study_samplelists|length > 0 %} + <div id="filterMenuSpan" class="input-append block-div-2"> + <label for="filter_study_select">Filter samples by study: </label> + <select id="filter_study"> + {% for study in study_samplelists %} + <option value="{{ loop.index - 1 }}">{{ study }}</option> + {% endfor %} + </select> + {% if sample_groups|length != 1 %} + <select id="filter_study_group" size="1"> + <option value="primary"> + {{ sample_group_types['samples_primary'] }} + </option> + <option value="other"> + {{ sample_group_types['samples_other'] }} + </option> + </select> + {% endif %} + <input type="button" id="filter_by_study" class="btn btn-danger" value="Filter"> + </div> + {% endif %} <div id="filterMenuSpan" class="input-append block-div-2"> <label for="filter_samples_field">Filter samples by {% if (numerical_var_list|length == 0) and (not js_data.se_exists) %}value{% endif %} </label> {% if (numerical_var_list|length > 0) or js_data.se_exists %} @@ -53,10 +74,12 @@ {% if js_data.se_exists %} <option value="stderr">SE</option> {% endif %} - {% for attribute in numerical_var_list %} + {% for attribute in sample_groups[0].attributes %} + {% if sample_groups[0].attributes[attribute].name in numerical_var_list %} <option value="{{ loop.index }}"> - {{ attribute }} + {{ sample_groups[0].attributes[attribute].name }} </option> + {% endif %} {% endfor %} </select> {% endif %} diff --git a/wqflask/wqflask/templates/test_correlation_page.html b/wqflask/wqflask/templates/test_correlation_page.html index 0809b65e..991773a2 100644 --- a/wqflask/wqflask/templates/test_correlation_page.html +++ b/wqflask/wqflask/templates/test_correlation_page.html @@ -113,7 +113,7 @@ console.log(correlationResults) {"data":corr_type=="sample"?null:"fd","width":"25px"}, { "data": "index","width":"120px","title":"Index" }, { "data": "trait_name","title":"TraitName"}, - { "data": "corr_coeffient","defaultContent": "--"}, + { "data": "corr_coefficient","defaultContent": "--"}, { "data": "p_value","defaultContent":"--"}, { "data": "num_overlap","defaultContent":"--"}, {"data":"tissue_corr","defaultContent":"--","title":"Tissue r"}, diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 731ca291..000d71d9 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -27,6 +27,8 @@ from zipfile import ZIP_DEFLATED from wqflask import app +from gn3.commands import run_cmd +from gn3.computations.gemma import generate_hash_of_string from gn3.db import diff_from_dict from gn3.db import fetchall from gn3.db import fetchone @@ -38,10 +40,13 @@ from gn3.db.phenotypes import Probeset from gn3.db.phenotypes import Publication from gn3.db.phenotypes import PublishXRef from gn3.db.phenotypes import probeset_mapping +from gn3.db.traits import get_trait_csv_sample_data +from gn3.db.traits import update_sample_data from flask import current_app from flask import g +from flask import flash from flask import Response from flask import request from flask import make_response @@ -59,6 +64,7 @@ from wqflask import server_side from base.data_set import create_dataset # Used by YAML in marker_regression from wqflask.show_trait import show_trait from wqflask.show_trait import export_trait_data +from wqflask.show_trait.show_trait import get_diff_of_vals from wqflask.heatmap import heatmap from wqflask.external_tools import send_to_bnw from wqflask.external_tools import send_to_webgestalt @@ -302,6 +308,7 @@ def gsearchact(): elif type == "phenotype": return render_template("gsearch_pheno.html", **result) + @app.route("/gsearch_table", methods=('GET',)) def gsearchtable(): logger.info(request.url) @@ -316,6 +323,7 @@ def gsearchtable(): return flask.jsonify(current_page) + @app.route("/gsearch_updating", methods=('POST',)) def gsearch_updating(): logger.info("REQUEST ARGS:", request.values) @@ -359,20 +367,6 @@ def wcgna_setup(): return render_template("wgcna_setup.html", **request.form) -# @app.route("/wgcna_results", methods=('POST',)) -# def wcgna_results(): -# logger.info("In wgcna, request.form is:", request.form) -# logger.info(request.url) -# # Start R, load the package and pointers and create the analysis -# wgcna = wgcna_analysis.WGCNA() -# # Start the analysis, a wgcnaA object should be a separate long running thread -# wgcnaA = wgcna.run_analysis(request.form) -# # After the analysis is finished store the result -# result = wgcna.process_results(wgcnaA) -# # Display them using the template -# return render_template("wgcna_results.html", **result) - - @app.route("/ctl_setup", methods=('POST',)) def ctl_setup(): # We are going to get additional user input for the analysis @@ -382,20 +376,6 @@ def ctl_setup(): return render_template("ctl_setup.html", **request.form) -# @app.route("/ctl_results", methods=('POST',)) -# def ctl_results(): -# logger.info("In ctl, request.form is:", request.form) -# logger.info(request.url) -# # Start R, load the package and pointers and create the analysis -# ctl = ctl_analysis.CTL() -# # Start the analysis, a ctlA object should be a separate long running thread -# ctlA = ctl.run_analysis(request.form) -# # After the analysis is finished store the result -# result = ctl.process_results(ctlA) -# # Display them using the template -# return render_template("ctl_results.html", **result) - - @app.route("/intro") def intro(): doc = Docs("intro", request.args) @@ -430,9 +410,9 @@ def submit_trait_form(): version=GN_VERSION) -@app.route("/trait/<name>/edit/inbredset-id/<inbred_set_id>") +@app.route("/trait/<name>/edit/inbredset-id/<inbredset_id>") @admin_login_required -def edit_phenotype(name, inbred_set_id): +def edit_phenotype(name, inbredset_id): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), passwd=current_app.config.get("DB_PASS"), @@ -441,7 +421,7 @@ def edit_phenotype(name, inbred_set_id): conn=conn, table="PublishXRef", where=PublishXRef(id_=name, - inbred_set_id=inbred_set_id)) + inbred_set_id=inbredset_id)) phenotype_ = fetchone( conn=conn, table="Phenotype", @@ -488,7 +468,7 @@ def edit_phenotype(name, inbred_set_id): @app.route("/trait/edit/probeset-name/<dataset_name>") -# @admin_login_required +@admin_login_required def edit_probeset(dataset_name): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), @@ -538,6 +518,68 @@ def update_phenotype(): passwd=current_app.config.get("DB_PASS"), host=current_app.config.get("DB_HOST")) data_ = request.form.to_dict() + TMPDIR = current_app.config.get("TMPDIR") + author = g.user_session.record.get(b'user_name') + if 'file' not in request.files: + flash("No sample-data has been uploaded", "warning") + else: + file_ = request.files['file'] + trait_name = str(data_.get('dataset-name')) + phenotype_id = str(data_.get('phenotype-id', 35)) + SAMPLE_DATADIR = os.path.join(TMPDIR, "sample-data") + if not os.path.exists(SAMPLE_DATADIR): + os.makedirs(SAMPLE_DATADIR) + if not os.path.exists(os.path.join(SAMPLE_DATADIR, + "diffs")): + os.makedirs(os.path.join(SAMPLE_DATADIR, + "diffs")) + if not os.path.exists(os.path.join(SAMPLE_DATADIR, + "updated")): + os.makedirs(os.path.join(SAMPLE_DATADIR, + "updated")) + current_time = str(datetime.datetime.now().isoformat()) + new_file_name = (os.path.join(TMPDIR, + "sample-data/updated/", + (f"{author.decode('utf-8')}." + f"{trait_name}.{phenotype_id}." + f"{current_time}.csv"))) + uploaded_file_name = (os.path.join( + TMPDIR, + "sample-data/updated/", + (f"updated.{author.decode('utf-8')}." + f"{trait_name}.{phenotype_id}." + f"{current_time}.csv"))) + file_.save(new_file_name) + publishdata_id = "" + lines = [] + with open(new_file_name, "r") as f: + lines = f.read() + first_line = lines.split('\n', 1)[0] + publishdata_id = first_line.split("Id:")[-1].strip() + with open(new_file_name, "w") as f: + f.write(lines.split("\n\n")[-1]) + csv_ = get_trait_csv_sample_data(conn=conn, + trait_name=str(trait_name), + phenotype_id=str(phenotype_id)) + with open(uploaded_file_name, "w") as f_: + f_.write(csv_.split("\n\n")[-1]) + r = run_cmd(cmd=("csvdiff " + f"'{uploaded_file_name}' '{new_file_name}' " + "--format json")) + diff_output = (f"{TMPDIR}/sample-data/diffs/" + f"{trait_name}.{author.decode('utf-8')}." + f"{phenotype_id}.{current_time}.json") + with open(diff_output, "w") as f: + dict_ = json.loads(r.get("output")) + dict_.update({ + "author": author.decode('utf-8'), + "publishdata_id": publishdata_id, + "dataset_id": data_.get("dataset-name"), + "timestamp": datetime.datetime.now().strftime( + "%Y-%m-%d %H:%M:%S") + }) + f.write(json.dumps(dict_)) + flash("Sample-data has been successfully uploaded", "success") # Run updates: phenotype_ = { "pre_pub_description": data_.get("pre-pub-desc"), @@ -579,7 +621,6 @@ def update_phenotype(): diff_data.update({"Publication": diff_from_dict(old={ k: data_.get(f"old_{k}") for k, v in publication_.items() if v is not None}, new=publication_)}) - author = g.user_session.record.get(b'user_name') if diff_data: diff_data.update({"dataset_id": data_.get("dataset-name")}) diff_data.update({"author": author.decode('utf-8')}) @@ -590,8 +631,9 @@ def update_phenotype(): data=MetadataAudit(dataset_id=data_.get("dataset-name"), editor=author.decode("utf-8"), json_data=json.dumps(diff_data))) + flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success") return redirect(f"/trait/{data_.get('dataset-name')}" - f"/edit/inbredset-id/{data_.get('inbred-set-id')}") + f"/edit/phenotype-id/{data_.get('phenotype-id')}") @app.route("/probeset/update", methods=["POST"]) @@ -956,16 +998,16 @@ def loading_page(): if key in wanted: start_vars[key] = value + sample_vals_dict = json.loads(start_vars['sample_vals']) if 'n_samples' in start_vars: n_samples = int(start_vars['n_samples']) else: - sample_vals_dict = json.loads(start_vars['sample_vals']) if 'group' in start_vars: dataset = create_dataset( start_vars['dataset'], group_name=start_vars['group']) else: dataset = create_dataset(start_vars['dataset']) - samples = start_vars['primary_samples'].split(",") + samples = dataset.group.samplelist if 'genofile' in start_vars: if start_vars['genofile'] != "": genofile_string = start_vars['genofile'] @@ -981,6 +1023,10 @@ def loading_page(): n_samples += 1 start_vars['n_samples'] = n_samples + start_vars['vals_hash'] = generate_hash_of_string(str(sample_vals_dict)) + if start_vars['dataset'] != "Temp": # Currently can't get diff for temp traits + start_vars['vals_diff'] = get_diff_of_vals(sample_vals_dict, str(start_vars['trait_id'] + ":" + str(start_vars['dataset']))) + start_vars['wanted_inputs'] = initial_start_vars['wanted_inputs'] start_vars_container['start_vars'] = start_vars @@ -1021,7 +1067,6 @@ def mapping_results_page(): 'num_perm', 'permCheck', 'perm_strata', - 'strat_var', 'categorical_vars', 'perm_output', 'num_bootstrap', @@ -1048,8 +1093,8 @@ def mapping_results_page(): 'showGenes', 'viewLegend', 'haplotypeAnalystCheck', - 'mapmethod_rqtl_geno', - 'mapmodel_rqtl_geno', + 'mapmethod_rqtl', + 'mapmodel_rqtl', 'temp_trait', 'reaper_version', 'n_samples', @@ -1087,32 +1132,23 @@ def mapping_results_page(): rendered_template = render_template("mapping_error.html") return rendered_template - template_vars.js_data = json.dumps(template_vars.js_data, - default=json_default_handler, - indent=" ") + if not template_vars.pair_scan: + template_vars.js_data = json.dumps(template_vars.js_data, + default=json_default_handler, + indent=" ") result = template_vars.__dict__ if result['pair_scan']: with Bench("Rendering template"): - img_path = result['pair_scan_filename'] - logger.info("img_path:", img_path) - initial_start_vars = request.form - logger.info("initial_start_vars:", initial_start_vars) - imgfile = open(TEMPDIR + img_path, 'rb') - imgdata = imgfile.read() - imgB64 = base64.b64encode(imgdata) - bytesarray = array.array('B', imgB64) - result['pair_scan_array'] = bytesarray rendered_template = render_template( "pair_scan_results.html", **result) else: gn1_template_vars = display_mapping_results.DisplayMappingResults( result).__dict__ - with Bench("Rendering template"): - rendered_template = render_template( - "mapping_results.html", **gn1_template_vars) + rendered_template = render_template( + "mapping_results.html", **gn1_template_vars) return rendered_template @@ -1193,9 +1229,10 @@ def corr_compute_page(): @app.route("/test_corr_compute", methods=["POST"]) def test_corr_compute_page(): - correlation_data = compute_correlation(request.form) + correlation_data = compute_correlation(request.form, compute_all=True) return render_template("test_correlation_page.html", **correlation_data) - + + @app.route("/corr_matrix", methods=('POST',)) def corr_matrix_page(): logger.info("In corr_matrix, request.form is:", pf(request.form)) @@ -1293,8 +1330,6 @@ def browser_inputs(): return flask.jsonify(file_contents) -########################################################################## - def json_default_handler(obj): """Based on http://stackoverflow.com/a/2680060/1175849""" @@ -1310,3 +1345,112 @@ def json_default_handler(obj): else: raise TypeError('Object of type %s with value of %s is not JSON serializable' % ( type(obj), repr(obj))) + + +@app.route("/trait/<trait_name>/sampledata/<phenotype_id>") +def get_sample_data_as_csv(trait_name: int, phenotype_id: int): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + csv_ = get_trait_csv_sample_data(conn, str(trait_name), + str(phenotype_id)) + return Response( + csv_, + mimetype="text/csv", + headers={"Content-disposition": + "attachment; filename=myplot.csv"} + ) + + +@app.route("/admin/data-sample/diffs/") +@admin_login_required +def display_diffs_admin(): + TMPDIR = current_app.config.get("TMPDIR") + DIFF_DIR = f"{TMPDIR}/sample-data/diffs" + files = [] + if os.path.exists(DIFF_DIR): + files = os.listdir(DIFF_DIR) + files = filter(lambda x: not(x.endswith((".approved", ".rejected"))), + files) + return render_template("display_files_admin.html", + files=files) + + +@app.route("/user/data-sample/diffs/") +def display_diffs_users(): + TMPDIR = current_app.config.get("TMPDIR") + DIFF_DIR = f"{TMPDIR}/sample-data/diffs" + files = [] + author = g.user_session.record.get(b'user_name').decode("utf-8") + if os.path.exists(DIFF_DIR): + files = os.listdir(DIFF_DIR) + files = filter(lambda x: not(x.endswith((".approved", ".rejected"))) \ + and author in x, + files) + return render_template("display_files_user.html", + files=files) + + +@app.route("/data-samples/approve/<name>") +def approve_data(name): + sample_data = {} + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + TMPDIR = current_app.config.get("TMPDIR") + with open(os.path.join(f"{TMPDIR}/sample-data/diffs", + name), 'r') as myfile: + sample_data = json.load(myfile) + PUBLISH_ID = sample_data.get("publishdata_id") + modifications = [d for d in sample_data.get("Modifications")] + row_counts = len(modifications) + for modification in modifications: + if modification.get("Current"): + (strain_id, + strain_name, + value, se, count) = modification.get("Current").split(",") + update_sample_data( + conn=conn, + strain_name=strain_name, + strain_id=int(strain_id), + publish_data_id=int(PUBLISH_ID), + value=value, + error=se, + count=count + ) + insert(conn, + table="metadata_audit", + data=MetadataAudit( + dataset_id=name.split(".")[0], # use the dataset name + editor=sample_data.get("author"), + json_data=json.dumps(sample_data))) + if modifications: + # Once data is approved, rename it! + os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", name), + os.path.join(f"{TMPDIR}/sample-data/diffs", + f"{name}.approved")) + flash((f"Just updated data from: {name}; {row_counts} " + "row(s) modified!"), + "success") + return redirect("/admin/data-sample/diffs/") + + +@app.route("/data-samples/reject/<name>") +def reject_data(name): + TMPDIR = current_app.config.get("TMPDIR") + os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", name), + os.path.join(f"{TMPDIR}/sample-data/diffs", + f"{name}.rejected")) + flash(f"{name} has been rejected!", "success") + return redirect("/admin/data-sample/diffs/") + + +@app.route("/display-file/<name>") +def display_file(name): + TMPDIR = current_app.config.get("TMPDIR") + with open(os.path.join(f"{TMPDIR}/sample-data/diffs", + name), 'r') as myfile: + content = myfile.read() + return Response(content, mimetype='text/json') |