diff options
| author | zsloan | 2021-10-05 16:41:13 -0500 |
|---|---|---|
| committer | GitHub | 2021-10-05 16:41:13 -0500 |
| commit | 70023c835bdeeffc48efafe96626ac5b01b5a6d2 (patch) | |
| tree | 5e23e1139d2e01e2ad42969f58d148132db963a7 | |
| parent | de79d0d5087bf97f59a16327132fd287d57e1c3f (diff) | |
| parent | 4c6a7e46dd7afe311c0bed38c4a69ddadf3cb416 (diff) | |
| download | genenetwork2-70023c835bdeeffc48efafe96626ac5b01b5a6d2.tar.gz | |
Merge branch 'testing' into feature/add_resizeable_columns
63 files changed, 1952 insertions, 1143 deletions
@@ -1,12 +1,13 @@ [](https://zenodo.org/badge/latestdoi/5591/genenetwork/genenetwork2) [](http://joss.theoj.org/papers/10.21105/joss.00025) [](https://github.com/genenetwork/genenetwork2/actions) + # GeneNetwork This repository contains the current source code for GeneNetwork (GN) (https://www.genenetwork.org/ (version 2). GN2 is a Web -2.0-style framework that includes data and computational tools for online genetics and genomic analysis of -many different populations and many types of molecular, cellular, and physiological data. +2.0-style framework that includes data and computational tools for online genetics and genomic analysis of +many different populations and many types of molecular, cellular, and physiological data. The system is used by scientists and clinians in the field of precision health care and systems genetics. GN and its predecessors have been in operation since Jan 1994, making it one of the longest-lived web services in biomedical research (https://en.wikipedia.org/wiki/GeneNetwork, and see a partial list of publications using GN and its predecessor, WebQTL (https://genenetwork.org/references/). @@ -26,7 +27,7 @@ interface genenetwork2 ``` -(default is http://localhost:5003/). A quick example is +A quick example is ```sh env GN2_PROFILE=~/opt/gn-latest SERVER_PORT=5300 GENENETWORK_FILES=~/data/gn2_data/ ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 5f4e0f9a..2b94b2a2 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -209,7 +209,7 @@ if [ "$1" = '-gunicorn-prod' ] ; then echo PYTHONPATH=$PYTHONPATH if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi PID=$TMPDIR/gunicorn.$USER.pid - cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID --workers 20 --keep-alive 6000 --max-requests 1000 --timeout 1200 wsgi" + cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID --workers 20 --keep-alive 6000 --max-requests 100 --max-requests-jitter 30 --timeout 1200 wsgi" echo RUNNING gunicorn $cmd gunicorn $cmd exit $? diff --git a/doc/README.org b/doc/README.org index 43c92e3c..1236016e 100644 --- a/doc/README.org +++ b/doc/README.org @@ -2,7 +2,9 @@ * Table of Contents :TOC: - [[#introduction][Introduction]] - - [[#install][Install]] + - [[#check-list][Check list]] + - [[#installing-guix-packages][Installing Guix packages]] + - [[#creating-a-gnu-guix-profile][Creating a GNU Guix profile]] - [[#running-gn2][Running GN2]] - [[#run-gn-proxy][Run gn-proxy]] - [[#run-redis][Run Redis]] @@ -37,32 +39,106 @@ tree. Current supported versions can be found as the SHA values of For a full view of runtime dependencies as defined by GNU Guix, see an example of the [[#gn2-dependency-graph][GN2 Dependency Graph]]. -* Install +* Check list + +To run GeneNetwork the following services need to function: + +1. [ ] GNU Guix with a guix profile for genenetwork2 +1. [ ] A path to the (static) genotype files +1. [ ] Gn-proxy for authentication +1. [ ] The genenetwork3 service +1. [ ] Redis +1. [ ] Mariadb + +* Installing Guix packages Make sure to install GNU Guix using the binary download instructions on the main website. Follow the instructions on [[GUIX-Reproducible-from-source.org]] to download pre-built binaries. Note -the download amounts to several GBs of data. +the download amounts to several GBs of data. Debian-derived distros +may support + +: apt-get install guix + +* Creating a GNU Guix profile + +We run a GNU Guix channel with packages at [[https://git.genenetwork.org/guix-bioinformatics/guix-bioinformatics][guix-bioinformatics]]. The +README has instructions for hosting a channel, but typically we use +the GUIX_PACKAGE_PATH instead. First upgrade to a recent guix with + +: mkdir ~/opt +: guix pull -p ~/opt/guix-pull + +It should upgrade (ignore the locales warnings). You can optionally +specify the specific git checkout of guix with + +: guix pull -p ~/opt/guix-pull --commit=f04883d + +which is useful when you ned to roll back to an earlier version +(sometimes our channel goes out of sync). Next, we install +GeneNetwork2 with + +: source ~/opt/guix-pull/etc/profile +: git clone https://git.genenetwork.org/guix-bioinformatics/guix-bioinformatics.git ~/guix-bioinformatics +: cd ~/guix-bioinformatics +: git pull +: env GUIX_PACKAGE_PATH=$HOME/guix-bioinformatics guix package -i genenetwork2 -p ~/opt/genenetwork2 + +you probably also need guix-past (the upstream channel for older packages): + +: git clone https://gitlab.inria.fr/guix-hpc/guix-past.git ~/guix-past +: cd ~/guix-past +: git pull +: env GUIX_PACKAGE_PATH=$HOME/guix-bioinformatics:$HOME/guix-past/modules ~/opt/guix-pull/bin/guix package -i genenetwork2 -p ~/opt/genenetwork2 + +ignore the warnings. Guix should install the software without trying +to build everything. If you system insists on building all packages, +try the `--dry-run` switch and fix the [[https://guix.gnu.org/manual/en/html_node/Substitute-Server-Authorization.html][substitutes]]. You may add the +`--substitute-urls="http://guix.genenetwork.org https://ci.guix.gnu.org https://mirror.hydra.gnu.org"` switch. + +The guix.genenetwork.org has most of our packages pre-built(!). To use +it on your own machine the public key is + +#+begin_src scheme +(public-key + (ecc + (curve Ed25519) + (q #E50F005E6DA2F85749B9AA62C8E86BB551CE2B541DC578C4DBE613B39EC9E750#))) +#+end_src + +Once we have a GNU Guix profile, a running database (see below) and the file storage, +we should be ready to fire up GeneNetwork: * Running GN2 -Default settings for GN2 are listed in a file called -[[../etc/default_settings.py][default_settings.py]]. You can copy this file and pass it as a new -parameter to the genenetwork2 command, e.g. +Check out the source with git: -: genenetwork2 mysettings.py +: git clone git@github.com:genenetwork/genenetwork2.git +: cd genenetwork2 -or you can set environment variables to override individual parameters, e.g. +Run GN2 with above Guix profile -: env SERVER_PORT=5004 SQL_URI=mysql://user:pwd@dbhostname/db_webqtl genenetwork2 +: export GN2_PROFILE=$HOME/opt/genenetwork2 +: env TMPDIR=$HOME/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5012 GENENETWORK_FILES=/export/data/genenetwork/genotype_files SQL_URI=mysql://webqtlout:webqtlout@localhost/db_webqtl ./bin/genenetwork2 etc/default_settings.py -gunicorn-dev the debug and logging switches can be particularly useful when -developing GN2. +developing GN2. Location and files are the current ones for Penguin2. + +It may be useful to tunnel the web server to your local browser with +an ssh tunnel: + +If you want to test a service running on the server on a certain +port (say 8202) use + + ssh -L 8202:127.0.0.1:8202 -f -N myname@penguin2.genenetwork.org + +And browse on your local machine to http://localhost:8202/ * Run gn-proxy GeneNetwork requires a separate gn-proxy server which handles -authorisation and access control. For instructions see the [[https://github.com/genenetwork/gn-proxy][README]]. +authorisation and access control. For instructions see the +[[https://github.com/genenetwork/gn-proxy][README]]. Note it may already be running on our servers! * Run Redis diff --git a/doc/guix_profile_setup.org b/doc/guix_profile_setup.org new file mode 100644 index 00000000..c397377c --- /dev/null +++ b/doc/guix_profile_setup.org @@ -0,0 +1,39 @@ +* Setting up GUIX profile for GN
+
+First create a guix profile with the latest packages:
+
+: ~/opt/guix/bin/guix pull
+
+This will create a profile with the latest packages under`~/.config/guix/current`
+
+Now you have the latest guix. Check: `$HOME/.config/guix/current/bin/guix --version`
+
+At this point, it's worth mentioning that installing
+python3-genenetwork using `$HOME/.config/guix/current/bin/guix` should
+work; but let's use the dev version(since that may come handy in
+time), and it's a nice thing to know.
+
+Next, we ensure that the appropriate GUILE<sub>PATHS</sub> are set:
+
+: export GUILE_LOAD_PATH=$HOME/.config/guix/current/share/guile/site/3.0/
+: export GUILE_LOAD_COMPILED_PATH=$HOME/.config/guix/current/lib/guile/3.0/site-ccache/
+
+Get into the container:
+
+: $HOME/.config/guix/current/bin/guix environment -C guix --ad-hoc bash gcc-toolchain
+: ./bootstrap
+: ./configure --localstatedir=/var --sysconfdir=/etc
+
+Check that everything works:
+
+: make check
+
+Clean up and build:
+
+: make clean-go
+: make -j 4
+: exit
+
+Install Python3 (substitute paths when necessary):
+
+: env GUIX_PACKAGE_PATH='/home/zas1024/guix-bioinformatics:/home/zas1024/guix-past/modules' $HOME/.config/guix/current/bin/guix install python3-genenetwork2 -p ~/opt/python3-genenetwork2 --substitute-urls="http://guix.genenetwork.org https://berlin.guixsd.org https://ci.guix.gnu.org https://mirror.hydra.gnu.org"
diff --git a/scripts/add_missing_columns.sh b/scripts/add_missing_columns.sh index 70d5fdeb..611e2dd6 100644 --- a/scripts/add_missing_columns.sh +++ b/scripts/add_missing_columns.sh @@ -13,6 +13,9 @@ ALTER TABLE PublishXRef ADD mean double AFTER DataId; + ALTER TABLE CaseAttribute + ADD Description varchar(255) AFTER Name; + -- This takes some time ALTER TABLE ProbeSet ADD UniProtID varchar(20) AFTER ProteinName; diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py new file mode 100644 index 00000000..76c7fb4f --- /dev/null +++ b/scripts/authentication/group.py @@ -0,0 +1,142 @@ +"""A script for adding users to a specific group. + +Example: + +Assuming there are no groups and 'test@bonfacemunyoki.com' does not +exist in Redis: + +.. code-block:: bash + python group.py -g "editors" -m "test@bonfacemunyoki.com" + +results in:: + + Successfully created the group: 'editors' + Data: '{"admins": [], "members": []}' + +If 'me@bonfacemunyoki.com' exists in 'users' in Redis and we run: + +.. code-block:: bash + python group.py -g "editors" -m "me@bonfacemunyoki.com" + +now results in:: + + No new group was created. + Updated Data: {'admins': [], 'members': ['me@bonfacemunyoki.com']} + +""" + +import argparse +import datetime +import redis +import json + +from typing import Dict, List, Optional, Set + +def create_group_data(users: Dict, target_group: str, + members: Optional[str] = None, + admins: Optional[str] = None) -> Dict: + """Return a dictionary that contains the following keys: "key", + "field", and "value" that can be used in a redis hash as follows: + HSET key field value + + Parameters: + + - `users`: a list of users for example: + + {'8ad942fe-490d-453e-bd37-56f252e41603': + '{"email_address": "me@test.com", + "full_name": "John Doe", + "organization": "Genenetwork", + "password": {"algorithm": "pbkdf2", + "hashfunc": "sha256", + "salt": "gJrd1HnPSSCmzB5veMPaVk2ozzDlS1Z7Ggcyl1+pciA=", + "iterations": 100000, "keylength": 32, + "created_timestamp": "2021-09-22T11:32:44.971912", + "password": "edcdaa60e84526c6"}, + "user_id": "8ad942fe", "confirmed": 1, + "registration_info": { + "timestamp": "2021-09-22T11:32:45.028833", + "ip_address": "127.0.0.1", + "user_agent": "Mozilla/5.0"}}'} + + - `target_group`: the group name that will be stored inside the + "groups" hash in Redis. + + - `members`: a comma-separated list of values that contain members + of the `target_group` e.g. "me@test1.com, me@test2.com, + me@test3.com" + + - `admins`: a comma-separated list of values that contain + administrators of the `target_group` e.g. "me@test1.com, + me@test2.com, me@test3.com" + + """ + + _members: List = "".join(members.split()).split(",") if members else [] + _admins: List = "".join(admins.split()).split(",") if admins else [] + + user_ids: Dict = dict() + for user_id, user_details in users.items(): + _details = json.loads(user_details) + if _details.get("email_address"): + user_ids[_details.get("email_address")] = user_id + print(user_ids) + return {"key": "groups", + "field": target_group, + "value": json.dumps({ + "id": target_group, + "name": target_group, + "admins": [user_ids[admin] for admin in _admins + if admin in user_ids], + "members": [user_ids[member] for member in _members + if member in user_ids], + "changed_timestamp": datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + })} + + +if __name__ == "__main__": + # Initialising the parser CLI arguments + parser = argparse.ArgumentParser() + parser.add_argument("-g", "--group-name", + help="This is the name of the GROUP mask") + parser.add_argument("-m", "--members", + help="Members of the GROUP mask") + parser.add_argument("-a", "--admins", + help="Admins of the GROUP mask") + args = parser.parse_args() + + if not args.group_name: + exit("\nExiting. Please specify a group name to use!\n") + + members = args.members if args.members else None + admins = args.admins if args.admins else None + + REDIS_CONN = redis.Redis(decode_responses=True) + USERS = REDIS_CONN.hgetall("users") + + if not any([members, admins]): + exit("\nExiting. Please provide a value for " + "MEMBERS(-m) or ADMINS(-a)!\n") + + data = create_group_data( + users=USERS, + target_group=args.group_name, + members=members, + admins=admins) + + if not REDIS_CONN.hget("groups", data.get("field", "")): + updated_data = json.loads(data["value"]) + updated_data["created_timestamp"] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + data["value"] = json.dumps(updated_data) + + created_p = REDIS_CONN.hset(data.get("key", ""), + data.get("field", ""), + data.get("value", "")) + + groups = json.loads(REDIS_CONN.hget("groups", + args.group_name)) # type: ignore + if created_p: + exit(f"\nSuccessfully created the group: '{args.group_name}'\n" + f"`HGETALL groups {args.group_name}`: {groups}\n") + exit("\nNo new group was created.\n" + f"`HGETALL groups {args.group_name}`: {groups}\n") diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py new file mode 100644 index 00000000..8fcf09d7 --- /dev/null +++ b/scripts/authentication/resource.py @@ -0,0 +1,100 @@ +"""A script that: + +- Optionally restores data from a json file. + +- By default, without any args provided, adds the group: 'editors' to +every resource. 'editors' should have the right to edit both metadata +and data. + +- Optionally creates a back-up every time you edit a resource. + + +To restore a back-up: + +.. code-block:: python + python resource.py --restore <PATH/TO/RESOURCE/BACK-UP/FILE> + +To add editors to every resource without creating a back-up: + +.. code-block:: python + python resource.py + +To add editors to every resource while creating a back-up before any +destructive edits: + +.. code-block:: python + python resource.py --enable-backup + +""" +import argparse +import json +import redis +import os + +from datetime import datetime + + +def recover_hash(name: str, file_path: str, set_function) -> bool: + """Recover back-ups using the `set_function` + + Parameters: + + - `name`: Redis hash where `file_path` will be restored + + - `file_path`: File path where redis hash is sourced from + + - `set_function`: Function used to do the Redis backup for + example: HSET + + """ + try: + with open(file_path, "r") as f: + resources = json.load(f) + for resource_id, resource in resources.items(): + set_function(name=name, + key=resource_id, + value=resource) + return True + except Exception as e: + print(e) + return False + + +if __name__ == "__main__": + # Initialising the parser CLI arguments + parser = argparse.ArgumentParser() + parser.add_argument("--restore", + help="Restore from a given backup") + parser.add_argument("--enable-backup", action="store_true", + help="Create a back up before edits") + args = parser.parse_args() + + if args.restore: + if recover_hash(name="resources", + file_path=args.back_up, + set_function=redis.Redis(decode_responses=True).hset): + exit(f"\n Done restoring {args.back_up}!\n") + else: + exit(f"\n There was an error restoring {args.back_up}!\n") + + REDIS_CONN = redis.Redis(decode_responses=True) + RESOURCES = REDIS_CONN.hgetall("resources") + BACKUP_DIR = os.path.join(os.getenv("HOME"), "redis") + if args.enable_backup: + FILENAME = ("resources-" + f"{datetime.now().strftime('%Y-%m-%d-%I:%M:%S-%p')}" + ".json") + if not os.path.exists(BACKUP_DIR): + os.mkdir(BACKUP_DIR) + with open(os.path.join(BACKUP_DIR, FILENAME), "w") as f: + json.dump(RESOURCES, f, indent=4) + print(f"\nDone backing upto {FILENAME}") + + for resource_id, resource in RESOURCES.items(): + _resource = json.loads(resource) # str -> dict conversion + _resource["group_masks"] = {"editors": {"metadata": "edit", + "data": "edit"}} + REDIS_CONN.hset("resources", + resource_id, + json.dumps(_resource)) + exit("Done updating `resources`\n") diff --git a/wqflask/.DS_Store b/wqflask/.DS_Store Binary files differdeleted file mode 100644 index d992942f..00000000 --- a/wqflask/.DS_Store +++ /dev/null diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 6dc44829..8906ab69 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -277,7 +277,6 @@ class Markers: filtered_markers = [] for marker in self.markers: if marker['name'] in p_values: - # logger.debug("marker {} IS in p_values".format(i)) marker['p_value'] = p_values[marker['name']] if math.isnan(marker['p_value']) or (marker['p_value'] <= 0): marker['lod_score'] = 0 @@ -298,7 +297,6 @@ class HumanMarkers(Markers): self.markers = [] for line in marker_data_fh: splat = line.strip().split() - # logger.debug("splat:", splat) if len(specified_markers) > 0: if splat[1] in specified_markers: marker = {} @@ -398,6 +396,15 @@ class DatasetGroup: if maternal and paternal: self.parlist = [maternal, paternal] + def get_study_samplelists(self): + study_sample_file = locate_ignore_error(self.name + ".json", 'study_sample_lists') + try: + f = open(study_sample_file) + except: + return [] + study_samples = json.load(f) + return study_samples + def get_genofiles(self): jsonfile = "%s/%s.json" % (webqtlConfig.GENODIR, self.name) try: @@ -557,6 +564,7 @@ class DataSet: self.fullname = None self.type = None self.data_scale = None # ZS: For example log2 + self.accession_id = None self.setup() @@ -573,6 +581,17 @@ class DataSet: self.group.get_samplelist() self.species = species.TheSpecies(self) + def as_dict(self): + return { + 'name': self.name, + 'shortname': self.shortname, + 'fullname': self.fullname, + 'type': self.type, + 'data_scale': self.data_scale, + 'group': self.group.name, + 'accession_id': self.accession_id + } + def get_accession_id(self): if self.type == "Publish": results = g.db.execute("""select InfoFiles.GN_AccesionId from InfoFiles, PublishFreeze, InbredSet where @@ -725,7 +744,6 @@ class DataSet: and Strain.SpeciesId=Species.Id and Species.name = '{}' """.format(create_in_clause(self.samplelist), *mescape(self.group.species)) - logger.sql(query) results = dict(g.db.execute(query).fetchall()) sample_ids = [results[item] for item in self.samplelist] @@ -896,7 +914,6 @@ class PhenotypeDataSet(DataSet): Geno.Name = '%s' and Geno.SpeciesId = Species.Id """ % (species, this_trait.locus) - logger.sql(query) result = g.db.execute(query).fetchone() if result: @@ -926,7 +943,6 @@ class PhenotypeDataSet(DataSet): Order BY Strain.Name """ - logger.sql(query) results = g.db.execute(query, (trait, self.id)).fetchall() return results @@ -993,7 +1009,6 @@ class GenotypeDataSet(DataSet): Order BY Strain.Name """ - logger.sql(query) results = g.db.execute(query, (webqtlDatabaseFunction.retrieve_species_id(self.group.name), trait, self.name)).fetchall() @@ -1114,8 +1129,6 @@ class MrnaAssayDataSet(DataSet): ProbeSet.Name = '%s' """ % (escape(str(this_trait.dataset.id)), escape(this_trait.name))) - - logger.sql(query) result = g.db.execute(query).fetchone() mean = result[0] if result else 0 @@ -1135,7 +1148,6 @@ class MrnaAssayDataSet(DataSet): Geno.Name = '{}' and Geno.SpeciesId = Species.Id """.format(species, this_trait.locus) - logger.sql(query) result = g.db.execute(query).fetchone() if result: @@ -1167,7 +1179,6 @@ class MrnaAssayDataSet(DataSet): Order BY Strain.Name """ % (escape(trait), escape(self.name)) - logger.sql(query) results = g.db.execute(query).fetchall() return results @@ -1178,7 +1189,6 @@ class MrnaAssayDataSet(DataSet): where ProbeSetXRef.ProbeSetFreezeId = %s and ProbeSetXRef.ProbeSetId=ProbeSet.Id; """ % (column_name, escape(str(self.id))) - logger.sql(query) results = g.db.execute(query).fetchall() return dict(results) @@ -1212,7 +1222,6 @@ def geno_mrna_confidentiality(ob): query = '''SELECT Id, Name, FullName, confidentiality, AuthorisedUsers FROM %s WHERE Name = "%s"''' % (dataset_table, ob.name) - logger.sql(query) result = g.db.execute(query) (dataset_id, diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index d09cfd40..96a09302 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -27,11 +27,13 @@ def create_trait(**kw): assert bool(kw.get('name')), "Needs trait name" - if kw.get('dataset_name'): + if bool(kw.get('dataset')): + dataset = kw.get('dataset') + else: if kw.get('dataset_name') != "Temp": dataset = create_dataset(kw.get('dataset_name')) - else: - dataset = kw.get('dataset') + else: + dataset = create_dataset("Temp", group_name=kw.get('group_name')) if dataset.type == 'Publish': permissions = check_resource_availability( @@ -284,17 +286,19 @@ def get_sample_data(): return None -def jsonable(trait): +def jsonable(trait, dataset=None): """Return a dict suitable for using as json Actual turning into json doesn't happen here though""" - dataset = create_dataset(dataset_name=trait.dataset.name, - dataset_type=trait.dataset.type, - group_name=trait.dataset.group.name) + if not dataset: + dataset = create_dataset(dataset_name=trait.dataset.name, + dataset_type=trait.dataset.type, + group_name=trait.dataset.group.name) if dataset.type == "ProbeSet": return dict(name=trait.name, + view=trait.view, symbol=trait.symbol, dataset=dataset.name, dataset_name=dataset.shortname, @@ -308,37 +312,44 @@ def jsonable(trait): elif dataset.type == "Publish": if trait.pubmed_id: return dict(name=trait.name, + view=trait.view, dataset=dataset.name, dataset_name=dataset.shortname, description=trait.description_display, abbreviation=trait.abbreviation, authors=trait.authors, + pubmed_id=trait.pubmed_id, pubmed_text=trait.pubmed_text, pubmed_link=trait.pubmed_link, + mean=trait.mean, lrs_score=trait.LRS_score_repr, lrs_location=trait.LRS_location_repr, additive=trait.additive ) else: return dict(name=trait.name, + view=trait.view, dataset=dataset.name, dataset_name=dataset.shortname, description=trait.description_display, abbreviation=trait.abbreviation, authors=trait.authors, pubmed_text=trait.pubmed_text, + mean=trait.mean, lrs_score=trait.LRS_score_repr, lrs_location=trait.LRS_location_repr, additive=trait.additive ) elif dataset.type == "Geno": return dict(name=trait.name, + view=trait.view, dataset=dataset.name, dataset_name=dataset.shortname, location=trait.location_repr ) elif dataset.name == "Temp": return dict(name=trait.name, + view=trait.view, dataset="Temp", dataset_name="Temp") else: diff --git a/wqflask/runserver.py b/wqflask/runserver.py index df957bd9..8198b921 100644 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -23,6 +23,9 @@ app_config() werkzeug_logger = logging.getLogger('werkzeug') if WEBSERVER_MODE == 'DEBUG': + from flask_debugtoolbar import DebugToolbarExtension + app.debug = True + toolbar = DebugToolbarExtension(app) app.run(host='0.0.0.0', port=SERVER_PORT, debug=True, diff --git a/wqflask/tests/integration/test_markdown_routes.py b/wqflask/tests/integration/test_markdown_routes.py deleted file mode 100644 index 5e3e5045..00000000 --- a/wqflask/tests/integration/test_markdown_routes.py +++ /dev/null @@ -1,21 +0,0 @@ -"Integration tests for markdown routes" -import unittest - -from bs4 import BeautifulSoup - -from wqflask import app - - -class TestGenMenu(unittest.TestCase): - """Tests for glossary""" - - def setUp(self): - self.app = app.test_client() - - def tearDown(self): - pass - - def test_glossary_page(self): - """Test that the glossary page is rendered properly""" - response = self.app.get('/glossary', follow_redirects=True) - pass diff --git a/wqflask/tests/unit/wqflask/correlation/test_correlation_functions.py b/wqflask/tests/unit/wqflask/correlation/test_correlation_functions.py index 2bbeab1f..a8cf6006 100644 --- a/wqflask/tests/unit/wqflask/correlation/test_correlation_functions.py +++ b/wqflask/tests/unit/wqflask/correlation/test_correlation_functions.py @@ -1,10 +1,30 @@ +"""module contains tests for correlation functions""" + import unittest from unittest import mock + from wqflask.correlation.correlation_functions import get_trait_symbol_and_tissue_values from wqflask.correlation.correlation_functions import cal_zero_order_corr_for_tiss class TestCorrelationFunctions(unittest.TestCase): + """test for correlation helper functions""" + + @mock.patch("wqflask.correlation.correlation_functions.compute_corr_coeff_p_value") + def test_tissue_corr_computation(self, mock_tiss_corr_computation): + """test for cal_zero_order_corr_for_tiss""" + + primary_values = [9.288, 9.313, 8.988, 9.660, 8.21] + target_values = [9.586, 8.498, 9.362, 8.820, 8.786] + + mock_tiss_corr_computation.return_value = (0.51, 0.7) + + results = cal_zero_order_corr_for_tiss(primary_values, target_values) + mock_tiss_corr_computation.assert_called_once_with( + primary_values=primary_values, target_values=target_values, + corr_method="pearson") + + self.assertEqual(len(results), 3) @mock.patch("wqflask.correlation.correlation_functions.MrnaAssayTissueData") def test_get_trait_symbol_and_tissue_values(self, mock_class): diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index 9b2c6735..d4256a46 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -139,7 +139,7 @@ def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLab max_D = max(data) min_D = min(data) # add by NL 06-20-2011: fix the error: when max_D is infinite, log function in detScale will go wrong - if max_D == float('inf') or max_D > webqtlConfig.MAXLRS: + if (max_D == float('inf') or max_D > webqtlConfig.MAXLRS) and min_D < webqtlConfig.MAXLRS: max_D = webqtlConfig.MAXLRS # maximum LRS value xLow, xTop, stepX = detScale(min_D, max_D) @@ -156,7 +156,7 @@ def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLab j += step for i, item in enumerate(data): - if item == float('inf') or item > webqtlConfig.MAXLRS: + if (item == float('inf') or item > webqtlConfig.MAXLRS) and min_D < webqtlConfig.MAXLRS: item = webqtlConfig.MAXLRS # maximum LRS value j = int((item - xLow) / step) Count[j] += 1 diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py index 57dbf8ba..6802d689 100644 --- a/wqflask/utility/authentication_tools.py +++ b/wqflask/utility/authentication_tools.py @@ -11,7 +11,6 @@ from utility.redis_tools import (get_redis_conn, add_resource) Redis = get_redis_conn() - def check_resource_availability(dataset, trait_id=None): # At least for now assume temporary entered traits are accessible if type(dataset) == str or dataset.type == "Temp": @@ -133,12 +132,17 @@ def check_owner_or_admin(dataset=None, trait_id=None, resource_id=None): else: resource_id = get_resource_id(dataset, trait_id) - if g.user_session.user_id in Redis.smembers("super_users"): + try: + user_id = g.user_session.user_id.encode('utf-8') + except: + user_id = g.user_session.user_id + + if user_id in Redis.smembers("super_users"): return "owner" resource_info = get_resource_info(resource_id) if resource_info: - if g.user_session.user_id == resource_info['owner_id']: + if user_id == resource_info['owner_id']: return "owner" else: return check_admin(resource_id) diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py index ff125bd2..de9dde46 100644 --- a/wqflask/utility/redis_tools.py +++ b/wqflask/utility/redis_tools.py @@ -127,22 +127,20 @@ def check_verification_code(code): def get_user_groups(user_id): - # ZS: Get the groups where a user is an admin or a member and + # Get the groups where a user is an admin or a member and # return lists corresponding to those two sets of groups - admin_group_ids = [] # ZS: Group IDs where user is an admin - user_group_ids = [] # ZS: Group IDs where user is a regular user + admin_group_ids = [] # Group IDs where user is an admin + user_group_ids = [] # Group IDs where user is a regular user groups_list = Redis.hgetall("groups") - for key in groups_list: + for group_id, group_details in groups_list.items(): try: - group_ob = json.loads(groups_list[key]) - group_admins = set([this_admin.encode( - 'utf-8') if this_admin else None for this_admin in group_ob['admins']]) - group_members = set([this_member.encode( - 'utf-8') if this_member else None for this_member in group_ob['members']]) + _details = json.loads(group_details) + group_admins = set([this_admin if this_admin else None for this_admin in _details['admins']]) + group_members = set([this_member if this_member else None for this_member in _details['members']]) if user_id in group_admins: - admin_group_ids.append(group_ob['id']) + admin_group_ids.append(group_id) elif user_id in group_members: - user_group_ids.append(group_ob['id']) + user_group_ids.append(group_id) else: continue except: diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py index 56d0af6f..778fb64d 100644 --- a/wqflask/utility/startup_config.py +++ b/wqflask/utility/startup_config.py @@ -20,8 +20,12 @@ def app_config(): import os app.config['SECRET_KEY'] = str(os.urandom(24)) mode = WEBSERVER_MODE - if mode == "DEV" or mode == "DEBUG": + if mode in ["DEV", "DEBUG"]: app.config['TEMPLATES_AUTO_RELOAD'] = True + if mode == "DEBUG": + from flask_debugtoolbar import DebugToolbarExtension + app.debug = True + toolbar = DebugToolbarExtension(app) print("==========================================") diff --git a/wqflask/wqflask/correlation/correlation_functions.py b/wqflask/wqflask/correlation/correlation_functions.py index c8b9da0e..85b25d60 100644 --- a/wqflask/wqflask/correlation/correlation_functions.py +++ b/wqflask/wqflask/correlation/correlation_functions.py @@ -21,15 +21,10 @@ # This module is used by GeneNetwork project (www.genenetwork.org) # # Created by GeneNetwork Core Team 2010/08/10 -# -# Last updated by NL 2011/03/23 -import math -import string from base.mrna_assay_tissue_data import MrnaAssayTissueData - -from flask import Flask, g +from gn3.computations.correlations import compute_corr_coeff_p_value ##################################################################################### @@ -45,31 +40,14 @@ from flask import Flask, g # the same tissue order ##################################################################################### -def cal_zero_order_corr_for_tiss(primaryValue=[], targetValue=[], method='pearson'): - - N = len(primaryValue) - # R_primary = rpy2.robjects.FloatVector(list(range(len(primaryValue)))) - # for i in range(len(primaryValue)): - # R_primary[i] = primaryValue[i] - # R_target = rpy2.robjects.FloatVector(list(range(len(targetValue)))) - # for i in range(len(targetValue)): - # R_target[i] = targetValue[i] +def cal_zero_order_corr_for_tiss(primary_values, target_values, method="pearson"): + """function use calls gn3 to compute corr,p_val""" - # R_corr_test = rpy2.robjects.r['cor.test'] - # if method == 'spearman': - # R_result = R_corr_test(R_primary, R_target, method='spearman') - # else: - # R_result = R_corr_test(R_primary, R_target) - - # corr_result = [] - # corr_result.append(R_result[3][0]) - # corr_result.append(N) - # corr_result.append(R_result[2][0]) - - return [None, N, None] - # return corr_result + (corr_coeff, p_val) = compute_corr_coeff_p_value( + primary_values=primary_values, target_values=target_values, corr_method=method) + return (corr_coeff, len(primary_values), p_val) ######################################################################################################## # input: cursor, symbolList (list), dataIdDict(Dict): key is symbol @@ -80,8 +58,9 @@ def cal_zero_order_corr_for_tiss(primaryValue=[], targetValue=[], method='pearso # then call getSymbolValuePairDict function and merge the results. ######################################################################################################## + def get_trait_symbol_and_tissue_values(symbol_list=None): tissue_data = MrnaAssayTissueData(gene_symbols=symbol_list) - if len(tissue_data.gene_symbols) >0: + if len(tissue_data.gene_symbols) > 0: results = tissue_data.get_symbol_values_pairs() - return results + return results diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 30c05f03..a18bceaf 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -18,7 +18,10 @@ from gn3.db_utils import database_connector def create_target_this_trait(start_vars): """this function creates the required trait and target dataset for correlation""" - this_dataset = data_set.create_dataset(dataset_name=start_vars['dataset']) + if start_vars['dataset'] == "Temp": + this_dataset = data_set.create_dataset(dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) + else: + this_dataset = data_set.create_dataset(dataset_name=start_vars['dataset']) target_dataset = data_set.create_dataset( dataset_name=start_vars['corr_dataset']) this_trait = create_trait(dataset=this_dataset, @@ -145,15 +148,9 @@ def lit_for_trait_list(corr_results, this_dataset, this_trait): def fetch_sample_data(start_vars, this_trait, this_dataset, target_dataset): sample_data = process_samples( - start_vars, this_dataset.group.samplelist) - - sample_data = test_process_data(this_trait, this_dataset, start_vars) - + start_vars, this_dataset.group.all_samples_ordered()) - if target_dataset.type == "ProbeSet": - target_dataset.get_probeset_data(list(sample_data.keys())) - else: - target_dataset.get_trait_data(list(sample_data.keys())) + target_dataset.get_trait_data(list(sample_data.keys())) this_trait = retrieve_sample_data(this_trait, this_dataset) this_trait_data = { "trait_sample_data": sample_data, @@ -165,8 +162,14 @@ def fetch_sample_data(start_vars, this_trait, this_dataset, target_dataset): return (this_trait_data, results) -def compute_correlation(start_vars, method="pearson"): - """compute correlation for to call gn3 api""" +def compute_correlation(start_vars, method="pearson", compute_all=False): + """Compute correlations using GN3 API + + Keyword arguments: + start_vars -- All input from form; includes things like the trait/dataset names + method -- Correlation method to be used (pearson, spearman, or bicor) + compute_all -- Include sample, tissue, and literature correlations (when applicable) + """ # pylint: disable-msg=too-many-locals corr_type = start_vars['corr_type'] @@ -196,17 +199,22 @@ def compute_correlation(start_vars, method="pearson"): if tissue_input is not None: (primary_tissue_data, target_tissue_data) = tissue_input - corr_input_data = { - "primary_tissue": primary_tissue_data, - "target_tissues_dict": target_tissue_data - } - correlation_results = compute_tissue_correlation( - primary_tissue_dict=corr_input_data["primary_tissue"], - target_tissues_data=corr_input_data[ - "target_tissues_dict"], - corr_method=method - - ) + corr_input_data = { + "primary_tissue": primary_tissue_data, + "target_tissues_dict": target_tissue_data + } + correlation_results = compute_tissue_correlation( + primary_tissue_dict=corr_input_data["primary_tissue"], + target_tissues_data=corr_input_data[ + "target_tissues_dict"], + corr_method=method + + ) + else: + return {"correlation_results": [], + "this_trait": this_trait.name, + "target_dataset": start_vars['corr_dataset'], + "return_results": corr_return_results} elif corr_type == "lit": (this_trait_geneid, geneid_dict, species) = do_lit_correlation( @@ -220,11 +228,9 @@ def compute_correlation(start_vars, method="pearson"): correlation_results = correlation_results[0:corr_return_results] - compute_all = True # later to be passed as argument - if (compute_all): - - correlation_results = compute_corr_for_top_results(correlation_results, + correlation_results = compute_corr_for_top_results(start_vars, + correlation_results, this_trait, this_dataset, target_dataset, @@ -238,7 +244,8 @@ def compute_correlation(start_vars, method="pearson"): return correlation_data -def compute_corr_for_top_results(correlation_results, +def compute_corr_for_top_results(start_vars, + correlation_results, this_trait, this_dataset, target_dataset, @@ -261,8 +268,12 @@ def compute_corr_for_top_results(correlation_results, correlation_results = merge_correlation_results( correlation_results, lit_result) - if corr_type != "sample": - pass + if corr_type != "sample" and this_dataset.type == "ProbeSet" and target_dataset.type == "ProbeSet": + sample_result = sample_for_trait_lists( + correlation_results, target_dataset, this_trait, this_dataset, start_vars) + if sample_result: + correlation_results = merge_correlation_results( + correlation_results, sample_result) return correlation_results @@ -294,4 +305,3 @@ def get_tissue_correlation_input(this_trait, trait_symbol_dict): "symbol_tissue_vals_dict": corr_result_tissue_vals_dict } return (primary_tissue_data, target_tissue_data) - return None diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 2f3df67a..d73965da 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -18,491 +18,152 @@ # # This module is used by GeneNetwork project (www.genenetwork.org) -import collections import json -import scipy -import numpy -# import rpy2.robjects as ro # R Objects -import utility.logger -import utility.webqtlUtil -from base.trait import create_trait +from base.trait import create_trait, jsonable +from base.data_set import create_dataset -from base import data_set -from utility import helper_functions -from utility import corr_result_helpers from utility import hmac -from wqflask.correlation import correlation_functions -from utility.benchmark import Bench - -from utility.type_checking import is_str -from utility.type_checking import get_float -from utility.type_checking import get_int -from utility.type_checking import get_string -from utility.db_tools import escape - -from flask import g - -logger = utility.logger.getLogger(__name__) - -METHOD_LIT = "3" -METHOD_TISSUE_PEARSON = "4" -METHOD_TISSUE_RANK = "5" - -TISSUE_METHODS = [METHOD_TISSUE_PEARSON, METHOD_TISSUE_RANK] - -TISSUE_MOUSE_DB = 1 - - -class CorrelationResults: - def __init__(self, start_vars): - # get trait list from db (database name) - # calculate correlation with Base vector and targets - - # Check parameters - assert('corr_type' in start_vars) - assert(is_str(start_vars['corr_type'])) - assert('dataset' in start_vars) - # assert('group' in start_vars) permitted to be empty? - assert('corr_sample_method' in start_vars) - assert('corr_samples_group' in start_vars) - assert('corr_dataset' in start_vars) - assert('corr_return_results' in start_vars) - if 'loc_chr' in start_vars: - assert('min_loc_mb' in start_vars) - assert('max_loc_mb' in start_vars) - - with Bench("Doing correlations"): - if start_vars['dataset'] == "Temp": - self.dataset = data_set.create_dataset( - dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) - self.trait_id = start_vars['trait_id'] - self.this_trait = create_trait(dataset=self.dataset, - name=self.trait_id, - cellid=None) - else: - helper_functions.get_species_dataset_trait(self, start_vars) - - corr_samples_group = start_vars['corr_samples_group'] - - self.sample_data = {} - self.corr_type = start_vars['corr_type'] - self.corr_method = start_vars['corr_sample_method'] - self.min_expr = get_float(start_vars, 'min_expr') - self.p_range_lower = get_float(start_vars, 'p_range_lower', -1.0) - self.p_range_upper = get_float(start_vars, 'p_range_upper', 1.0) - - - if ('loc_chr' in start_vars - and 'min_loc_mb' in start_vars - and 'max_loc_mb' in start_vars): - self.location_type = get_string(start_vars, 'location_type') - self.location_chr = get_string(start_vars, 'loc_chr') - self.min_location_mb = get_int(start_vars, 'min_loc_mb') - self.max_location_mb = get_int(start_vars, 'max_loc_mb') - else: - self.location_type = self.location_chr = self.min_location_mb = self.max_location_mb = None - - self.get_formatted_corr_type() - self.return_number = int(start_vars['corr_return_results']) - - # The two if statements below append samples to the sample list based upon whether the user - # rselected Primary Samples Only, Other Samples Only, or All Samples - - primary_samples = self.dataset.group.samplelist - if self.dataset.group.parlist != None: - primary_samples += self.dataset.group.parlist - if self.dataset.group.f1list != None: - primary_samples += self.dataset.group.f1list - - # If either BXD/whatever Only or All Samples, append all of that group's samplelist - if corr_samples_group != 'samples_other': - self.process_samples(start_vars, primary_samples) - - # If either Non-BXD/whatever or All Samples, get all samples from this_trait.data and - # exclude the primary samples (because they would have been added in the previous - # if statement if the user selected All Samples) - if corr_samples_group != 'samples_primary': - if corr_samples_group == 'samples_other': - primary_samples = [x for x in primary_samples if x not in ( - self.dataset.group.parlist + self.dataset.group.f1list)] - self.process_samples(start_vars, list( - self.this_trait.data.keys()), primary_samples) - - self.target_dataset = data_set.create_dataset( - start_vars['corr_dataset']) - self.target_dataset.get_trait_data(list(self.sample_data.keys())) - - self.header_fields = get_header_fields( - self.target_dataset.type, self.corr_method) - - if self.target_dataset.type == "ProbeSet": - self.filter_cols = [7, 6] - elif self.target_dataset.type == "Publish": - self.filter_cols = [6, 0] - else: - self.filter_cols = [4, 0] +def set_template_vars(start_vars, correlation_data): + corr_type = start_vars['corr_type'] + corr_method = start_vars['corr_sample_method'] - self.correlation_results = [] + if start_vars['dataset'] == "Temp": + this_dataset_ob = create_dataset(dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) + else: + this_dataset_ob = create_dataset(dataset_name=start_vars['dataset']) + this_trait = create_trait(dataset=this_dataset_ob, + name=start_vars['trait_id']) - self.correlation_data = {} + correlation_data['this_trait'] = jsonable(this_trait, this_dataset_ob) + correlation_data['this_dataset'] = this_dataset_ob.as_dict() - if self.corr_type == "tissue": - self.trait_symbol_dict = self.dataset.retrieve_genes("Symbol") + target_dataset_ob = create_dataset(correlation_data['target_dataset']) + correlation_data['target_dataset'] = target_dataset_ob.as_dict() - tissue_corr_data = self.do_tissue_correlation_for_all_traits() - if tissue_corr_data != None: - for trait in list(tissue_corr_data.keys())[:self.return_number]: - self.get_sample_r_and_p_values( - trait, self.target_dataset.trait_data[trait]) - else: - for trait, values in list(self.target_dataset.trait_data.items()): - self.get_sample_r_and_p_values(trait, values) - - elif self.corr_type == "lit": - self.trait_geneid_dict = self.dataset.retrieve_genes("GeneId") - lit_corr_data = self.do_lit_correlation_for_all_traits() - - for trait in list(lit_corr_data.keys())[:self.return_number]: - self.get_sample_r_and_p_values( - trait, self.target_dataset.trait_data[trait]) - - elif self.corr_type == "sample": - for trait, values in list(self.target_dataset.trait_data.items()): - self.get_sample_r_and_p_values(trait, values) - - self.correlation_data = collections.OrderedDict(sorted(list(self.correlation_data.items()), - key=lambda t: -abs(t[1][0]))) - - # ZS: Convert min/max chromosome to an int for the location range option - range_chr_as_int = None - for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): - if 'loc_chr' in start_vars: - if chr_info.name == self.location_chr: - range_chr_as_int = order_id - - for _trait_counter, trait in enumerate(list(self.correlation_data.keys())[:self.return_number]): - trait_object = create_trait( - dataset=self.target_dataset, name=trait, get_qtl_info=True, get_sample_info=False) - if not trait_object: - continue - - chr_as_int = 0 - for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): - if self.location_type == "highest_lod": - if chr_info.name == trait_object.locus_chr: - chr_as_int = order_id - else: - if chr_info.name == trait_object.chr: - chr_as_int = order_id - - - if (float(self.correlation_data[trait][0]) >= self.p_range_lower - and float(self.correlation_data[trait][0]) <= self.p_range_upper): - - if (self.target_dataset.type == "ProbeSet" or self.target_dataset.type == "Publish") and bool(trait_object.mean): - if (self.min_expr != None) and (float(trait_object.mean) < self.min_expr): - continue - - if range_chr_as_int != None and (chr_as_int != range_chr_as_int): - continue - if self.location_type == "highest_lod": - if (self.min_location_mb != None) and (float(trait_object.locus_mb) < float(self.min_location_mb)): - continue - if (self.max_location_mb != None) and (float(trait_object.locus_mb) > float(self.max_location_mb)): - continue - else: - if (self.min_location_mb != None) and (float(trait_object.mb) < float(self.min_location_mb)): - continue - if (self.max_location_mb != None) and (float(trait_object.mb) > float(self.max_location_mb)): - continue - - (trait_object.sample_r, - trait_object.sample_p, - trait_object.num_overlap) = self.correlation_data[trait] - - # Set some sane defaults - trait_object.tissue_corr = 0 - trait_object.tissue_pvalue = 0 - trait_object.lit_corr = 0 - if self.corr_type == "tissue" and tissue_corr_data != None: - trait_object.tissue_corr = tissue_corr_data[trait][1] - trait_object.tissue_pvalue = tissue_corr_data[trait][2] - elif self.corr_type == "lit": - trait_object.lit_corr = lit_corr_data[trait][1] - - self.correlation_results.append(trait_object) - - if self.corr_type != "lit" and self.dataset.type == "ProbeSet" and self.target_dataset.type == "ProbeSet": - self.do_lit_correlation_for_trait_list() - - if self.corr_type != "tissue" and self.dataset.type == "ProbeSet" and self.target_dataset.type == "ProbeSet": - self.do_tissue_correlation_for_trait_list() - - self.json_results = generate_corr_json( - self.correlation_results, self.this_trait, self.dataset, self.target_dataset) - -############################################################################################################################################ - - def get_formatted_corr_type(self): - self.formatted_corr_type = "" - if self.corr_type == "lit": - self.formatted_corr_type += "Literature Correlation " - elif self.corr_type == "tissue": - self.formatted_corr_type += "Tissue Correlation " - elif self.corr_type == "sample": - self.formatted_corr_type += "Genetic Correlation " - - if self.corr_method == "pearson": - self.formatted_corr_type += "(Pearson's r)" - elif self.corr_method == "spearman": - self.formatted_corr_type += "(Spearman's rho)" - elif self.corr_method == "bicor": - self.formatted_corr_type += "(Biweight r)" - - def do_tissue_correlation_for_trait_list(self, tissue_dataset_id=1): - """Given a list of correlation results (self.correlation_results), gets the tissue correlation value for each""" - - # Gets tissue expression values for the primary trait - primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list=[self.this_trait.symbol]) - - if self.this_trait.symbol.lower() in primary_trait_tissue_vals_dict: - primary_trait_tissue_values = primary_trait_tissue_vals_dict[self.this_trait.symbol.lower( - )] - gene_symbol_list = [ - trait.symbol for trait in self.correlation_results if trait.symbol] - - corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list=gene_symbol_list) - - for trait in self.correlation_results: - if trait.symbol and trait.symbol.lower() in corr_result_tissue_vals_dict: - this_trait_tissue_values = corr_result_tissue_vals_dict[trait.symbol.lower( - )] - - result = correlation_functions.cal_zero_order_corr_for_tiss(primary_trait_tissue_values, - this_trait_tissue_values, - self.corr_method) - - trait.tissue_corr = result[0] - trait.tissue_pvalue = result[2] - - def do_tissue_correlation_for_all_traits(self, tissue_dataset_id=1): - # Gets tissue expression values for the primary trait - primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list=[self.this_trait.symbol]) - - if self.this_trait.symbol.lower() in primary_trait_tissue_vals_dict: - primary_trait_tissue_values = primary_trait_tissue_vals_dict[self.this_trait.symbol.lower( - )] - - #print("trait_gene_symbols: ", pf(trait_gene_symbols.values())) - corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list=list(self.trait_symbol_dict.values())) - - #print("corr_result_tissue_vals: ", pf(corr_result_tissue_vals_dict)) - - #print("trait_gene_symbols: ", pf(trait_gene_symbols)) - - tissue_corr_data = {} - for trait, symbol in list(self.trait_symbol_dict.items()): - if symbol and symbol.lower() in corr_result_tissue_vals_dict: - this_trait_tissue_values = corr_result_tissue_vals_dict[symbol.lower( - )] + table_json = correlation_json_for_table(correlation_data, + correlation_data['this_trait'], + correlation_data['this_dataset'], + target_dataset_ob) - result = correlation_functions.cal_zero_order_corr_for_tiss(primary_trait_tissue_values, - this_trait_tissue_values, - self.corr_method) + correlation_data['table_json'] = table_json - tissue_corr_data[trait] = [symbol, result[0], result[2]] + if target_dataset_ob.type == "ProbeSet": + filter_cols = [7, 6] + elif target_dataset_ob.type == "Publish": + filter_cols = [6, 0] + else: + filter_cols = [4, 0] - tissue_corr_data = collections.OrderedDict(sorted(list(tissue_corr_data.items()), - key=lambda t: -abs(t[1][1]))) + correlation_data['corr_method'] = corr_method + correlation_data['filter_cols'] = filter_cols + correlation_data['header_fields'] = get_header_fields( + target_dataset_ob.type, correlation_data['corr_method']) + correlation_data['formatted_corr_type'] = get_formatted_corr_type( + corr_type, corr_method) - return tissue_corr_data + return correlation_data - def do_lit_correlation_for_trait_list(self): - input_trait_mouse_gene_id = self.convert_to_mouse_gene_id( - self.dataset.group.species.lower(), self.this_trait.geneid) +def correlation_json_for_table(correlation_data, this_trait, this_dataset, target_dataset_ob): + """Return JSON data for use with the DataTable in the correlation result page - for trait in self.correlation_results: + Keyword arguments: + correlation_data -- Correlation results + this_trait -- Trait being correlated against a dataset, as a dict + this_dataset -- Dataset of this_trait, as a dict + target_dataset_ob - Target dataset, as a Dataset ob + """ + this_trait = correlation_data['this_trait'] + this_dataset = correlation_data['this_dataset'] + target_dataset = target_dataset_ob.as_dict() - if trait.geneid: - trait.mouse_gene_id = self.convert_to_mouse_gene_id( - self.dataset.group.species.lower(), trait.geneid) - else: - trait.mouse_gene_id = None - - if trait.mouse_gene_id and str(trait.mouse_gene_id).find(";") == -1: - result = g.db.execute( - """SELECT value - FROM LCorrRamin3 - WHERE GeneId1='%s' and - GeneId2='%s' - """ % (escape(str(trait.mouse_gene_id)), escape(str(input_trait_mouse_gene_id))) - ).fetchone() - if not result: - result = g.db.execute("""SELECT value - FROM LCorrRamin3 - WHERE GeneId2='%s' and - GeneId1='%s' - """ % (escape(str(trait.mouse_gene_id)), escape(str(input_trait_mouse_gene_id))) - ).fetchone() - - if result: - lit_corr = result.value - trait.lit_corr = lit_corr - else: - trait.lit_corr = 0 - else: - trait.lit_corr = 0 - - def do_lit_correlation_for_all_traits(self): - input_trait_mouse_gene_id = self.convert_to_mouse_gene_id( - self.dataset.group.species.lower(), self.this_trait.geneid) - - lit_corr_data = {} - for trait, gene_id in list(self.trait_geneid_dict.items()): - mouse_gene_id = self.convert_to_mouse_gene_id( - self.dataset.group.species.lower(), gene_id) - - if mouse_gene_id and str(mouse_gene_id).find(";") == -1: - #print("gene_symbols:", input_trait_mouse_gene_id + " / " + mouse_gene_id) - result = g.db.execute( - """SELECT value - FROM LCorrRamin3 - WHERE GeneId1='%s' and - GeneId2='%s' - """ % (escape(mouse_gene_id), escape(input_trait_mouse_gene_id)) - ).fetchone() - if not result: - result = g.db.execute("""SELECT value - FROM LCorrRamin3 - WHERE GeneId2='%s' and - GeneId1='%s' - """ % (escape(mouse_gene_id), escape(input_trait_mouse_gene_id)) - ).fetchone() - if result: - #print("result:", result) - lit_corr = result.value - lit_corr_data[trait] = [gene_id, lit_corr] + corr_results = correlation_data['correlation_results'] + results_list = [] + for i, trait_dict in enumerate(corr_results): + trait_name = list(trait_dict.keys())[0] + trait = trait_dict[trait_name] + target_trait_ob = create_trait(dataset=target_dataset_ob, + name=trait_name, + get_qtl_info=True) + target_trait = jsonable(target_trait_ob, target_dataset_ob) + if target_trait['view'] == False: + continue + results_dict = {} + results_dict['index'] = i + 1 + results_dict['trait_id'] = target_trait['name'] + results_dict['dataset'] = target_dataset['name'] + results_dict['hmac'] = hmac.data_hmac( + '{}:{}'.format(target_trait['name'], target_dataset['name'])) + results_dict['sample_r'] = f"{float(trait['corr_coefficient']):.3f}" + results_dict['num_overlap'] = trait['num_overlap'] + results_dict['sample_p'] = f"{float(trait['p_value']):.3e}" + if target_dataset['type'] == "ProbeSet": + results_dict['symbol'] = target_trait['symbol'] + results_dict['description'] = "N/A" + results_dict['location'] = target_trait['location'] + results_dict['mean'] = "N/A" + results_dict['additive'] = "N/A" + if bool(target_trait['description']): + results_dict['description'] = target_trait['description'] + if bool(target_trait['mean']): + results_dict['mean'] = f"{float(target_trait['mean']):.3f}" + try: + results_dict['lod_score'] = f"{float(target_trait['lrs_score']) / 4.61:.1f}" + except: + results_dict['lod_score'] = "N/A" + results_dict['lrs_location'] = target_trait['lrs_location'] + if bool(target_trait['additive']): + results_dict['additive'] = f"{float(target_trait['additive']):.3f}" + results_dict['lit_corr'] = "--" + results_dict['tissue_corr'] = "--" + results_dict['tissue_pvalue'] = "--" + if this_dataset['type'] == "ProbeSet": + if 'lit_corr' in trait: + results_dict['lit_corr'] = f"{float(trait['lit_corr']):.3f}" + if 'tissue_corr' in trait: + results_dict['tissue_corr'] = f"{float(trait['tissue_corr']):.3f}" + results_dict['tissue_pvalue'] = f"{float(trait['tissue_p_val']):.3e}" + elif target_dataset['type'] == "Publish": + results_dict['abbreviation_display'] = "N/A" + results_dict['description'] = "N/A" + results_dict['mean'] = "N/A" + results_dict['authors_display'] = "N/A" + results_dict['additive'] = "N/A" + results_dict['pubmed_link'] = "N/A" + results_dict['pubmed_text'] = "N/A" + + if bool(target_trait['abbreviation']): + results_dict['abbreviation_display'] = target_trait['abbreviation'] + if bool(target_trait['description']): + results_dict['description'] = target_trait['description'] + if bool(target_trait['mean']): + results_dict['mean'] = f"{float(target_trait['mean']):.3f}" + if bool(target_trait['authors']): + authors_list = target_trait['authors'].split(',') + if len(authors_list) > 6: + results_dict['authors_display'] = ", ".join( + authors_list[:6]) + ", et al." else: - lit_corr_data[trait] = [gene_id, 0] - else: - lit_corr_data[trait] = [gene_id, 0] - - lit_corr_data = collections.OrderedDict(sorted(list(lit_corr_data.items()), - key=lambda t: -abs(t[1][1]))) - - return lit_corr_data - - def convert_to_mouse_gene_id(self, species=None, gene_id=None): - """If the species is rat or human, translate the gene_id to the mouse geneid - - If there is no input gene_id or there's no corresponding mouse gene_id, return None - - """ - if not gene_id: - return None - - mouse_gene_id = None - - if species == 'mouse': - mouse_gene_id = gene_id - - elif species == 'rat': - - query = """SELECT mouse - FROM GeneIDXRef - WHERE rat='%s'""" % escape(gene_id) - - result = g.db.execute(query).fetchone() - if result != None: - mouse_gene_id = result.mouse - - elif species == 'human': - - query = """SELECT mouse - FROM GeneIDXRef - WHERE human='%s'""" % escape(gene_id) - - result = g.db.execute(query).fetchone() - if result != None: - mouse_gene_id = result.mouse - - return mouse_gene_id - - def get_sample_r_and_p_values(self, trait, target_samples): - """Calculates the sample r (or rho) and p-value - - Given a primary trait and a target trait's sample values, - calculates either the pearson r or spearman rho and the p-value - using the corresponding scipy functions. - - """ - - self.this_trait_vals = [] - target_vals = [] - for index, sample in enumerate(self.target_dataset.samplelist): - if sample in self.sample_data: - sample_value = self.sample_data[sample] - target_sample_value = target_samples[index] - self.this_trait_vals.append(sample_value) - target_vals.append(target_sample_value) - - self.this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values( - self.this_trait_vals, target_vals) - - if num_overlap > 5: - # ZS: 2015 could add biweight correlation, see http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3465711/ - # if self.corr_method == 'bicor': - # sample_r, sample_p = do_bicor( - # self.this_trait_vals, target_vals) - if self.corr_method == 'pearson': - sample_r, sample_p = scipy.stats.pearsonr( - self.this_trait_vals, target_vals) - else: - sample_r, sample_p = scipy.stats.spearmanr( - self.this_trait_vals, target_vals) - - if numpy.isnan(sample_r): - pass - else: - self.correlation_data[trait] = [ - sample_r, sample_p, num_overlap] - - def process_samples(self, start_vars, sample_names, excluded_samples=None): - if not excluded_samples: - excluded_samples = () - - sample_val_dict = json.loads(start_vars['sample_vals']) - for sample in sample_names: - if sample not in excluded_samples: - value = sample_val_dict[sample] - if not value.strip().lower() == 'x': - self.sample_data[str(sample)] = float(value) - - -# def do_bicor(this_trait_vals, target_trait_vals): -# r_library = ro.r["library"] # Map the library function -# r_options = ro.r["options"] # Map the options function - -# r_library("WGCNA") -# r_bicor = ro.r["bicorAndPvalue"] # Map the bicorAndPvalue function - -# r_options(stringsAsFactors=False) - -# this_vals = ro.Vector(this_trait_vals) -# target_vals = ro.Vector(target_trait_vals) + results_dict['authors_display'] = target_trait['authors'] + if 'pubmed_id' in target_trait: + results_dict['pubmed_link'] = target_trait['pubmed_link'] + results_dict['pubmed_text'] = target_trait['pubmed_text'] + try: + results_dict['lod_score'] = f"{float(target_trait['lrs_score']) / 4.61:.1f}" + except: + results_dict['lod_score'] = "N/A" + results_dict['lrs_location'] = target_trait['lrs_location'] + if bool(target_trait['additive']): + results_dict['additive'] = f"{float(target_trait['additive']):.3f}" + else: + results_dict['location'] = target_trait['location'] -# the_r, the_p, _fisher_transform, _the_t, _n_obs = [ -# numpy.asarray(x) for x in r_bicor(x=this_vals, y=target_vals)] + results_list.append(results_dict) -# return the_r, the_p + return json.dumps(results_list) def generate_corr_json(corr_results, this_trait, dataset, target_dataset, for_api=False): @@ -598,6 +259,25 @@ def generate_corr_json(corr_results, this_trait, dataset, target_dataset, for_ap return json.dumps(results_list) +def get_formatted_corr_type(corr_type, corr_method): + formatted_corr_type = "" + if corr_type == "lit": + formatted_corr_type += "Literature Correlation " + elif corr_type == "tissue": + formatted_corr_type += "Tissue Correlation " + elif corr_type == "sample": + formatted_corr_type += "Genetic Correlation " + + if corr_method == "pearson": + formatted_corr_type += "(Pearson's r)" + elif corr_method == "spearman": + formatted_corr_type += "(Spearman's rho)" + elif corr_method == "bicor": + formatted_corr_type += "(Biweight r)" + + return formatted_corr_type + + def get_header_fields(data_type, corr_method): if data_type == "ProbeSet": if corr_method == "spearman": diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 9ac02ac5..e7b16e77 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -23,6 +23,9 @@ import math import random import string +import rpy2.robjects as ro +from rpy2.robjects.packages import importr + import numpy as np import scipy @@ -160,23 +163,22 @@ class CorrelationMatrix: for sample in self.all_sample_list: groups.append(1) - # Not doing PCA until rpy2 is excised self.pca_works = "False" - # try: - # corr_result_eigen = np.linalg.eig(np.array(self.pca_corr_results)) - # corr_eigen_value, corr_eigen_vectors = sortEigenVectors( - # corr_result_eigen) - - # if self.do_PCA == True: - # self.pca_works = "True" - # self.pca_trait_ids = [] - # pca = self.calculate_pca( - # list(range(len(self.traits))), corr_eigen_value, corr_eigen_vectors) - # self.loadings_array = self.process_loadings() - # else: - # self.pca_works = "False" - # except: - # self.pca_works = "False" + try: + corr_result_eigen = np.linalg.eig(np.array(self.pca_corr_results)) + corr_eigen_value, corr_eigen_vectors = sortEigenVectors( + corr_result_eigen) + + if self.do_PCA == True: + self.pca_works = "True" + self.pca_trait_ids = [] + pca = self.calculate_pca( + list(range(len(self.traits))), corr_eigen_value, corr_eigen_vectors) + self.loadings_array = self.process_loadings() + else: + self.pca_works = "False" + except: + self.pca_works = "False" self.js_data = dict(traits=[trait.name for trait in self.traits], groups=groups, @@ -185,51 +187,51 @@ class CorrelationMatrix: samples=self.all_sample_list, sample_data=self.sample_data,) - # def calculate_pca(self, cols, corr_eigen_value, corr_eigen_vectors): - # base = importr('base') - # stats = importr('stats') - - # corr_results_to_list = robjects.FloatVector( - # [item for sublist in self.pca_corr_results for item in sublist]) - - # m = robjects.r.matrix(corr_results_to_list, nrow=len(cols)) - # eigen = base.eigen(m) - # pca = stats.princomp(m, cor="TRUE") - # self.loadings = pca.rx('loadings') - # self.scores = pca.rx('scores') - # self.scale = pca.rx('scale') - - # trait_array = zScore(self.trait_data_array) - # trait_array_vectors = np.dot(corr_eigen_vectors, trait_array) - - # pca_traits = [] - # for i, vector in enumerate(trait_array_vectors): - # # ZS: Check if below check is necessary - # # if corr_eigen_value[i-1] > 100.0/len(self.trait_list): - # pca_traits.append((vector * -1.0).tolist()) - - # this_group_name = self.trait_list[0][1].group.name - # temp_dataset = data_set.create_dataset( - # dataset_name="Temp", dataset_type="Temp", group_name=this_group_name) - # temp_dataset.group.get_samplelist() - # for i, pca_trait in enumerate(pca_traits): - # trait_id = "PCA" + str(i + 1) + "_" + temp_dataset.group.species + "_" + \ - # this_group_name + "_" + datetime.datetime.now().strftime("%m%d%H%M%S") - # this_vals_string = "" - # position = 0 - # for sample in temp_dataset.group.all_samples_ordered(): - # if sample in self.shared_samples_list: - # this_vals_string += str(pca_trait[position]) - # this_vals_string += " " - # position += 1 - # else: - # this_vals_string += "x " - # this_vals_string = this_vals_string[:-1] - - # Redis.set(trait_id, this_vals_string, ex=THIRTY_DAYS) - # self.pca_trait_ids.append(trait_id) - - # return pca + def calculate_pca(self, cols, corr_eigen_value, corr_eigen_vectors): + base = importr('base') + stats = importr('stats') + + corr_results_to_list = ro.FloatVector( + [item for sublist in self.pca_corr_results for item in sublist]) + + m = ro.r.matrix(corr_results_to_list, nrow=len(cols)) + eigen = base.eigen(m) + pca = stats.princomp(m, cor="TRUE") + self.loadings = pca.rx('loadings') + self.scores = pca.rx('scores') + self.scale = pca.rx('scale') + + trait_array = zScore(self.trait_data_array) + trait_array_vectors = np.dot(corr_eigen_vectors, trait_array) + + pca_traits = [] + for i, vector in enumerate(trait_array_vectors): + # ZS: Check if below check is necessary + # if corr_eigen_value[i-1] > 100.0/len(self.trait_list): + pca_traits.append((vector * -1.0).tolist()) + + this_group_name = self.trait_list[0][1].group.name + temp_dataset = data_set.create_dataset( + dataset_name="Temp", dataset_type="Temp", group_name=this_group_name) + temp_dataset.group.get_samplelist() + for i, pca_trait in enumerate(pca_traits): + trait_id = "PCA" + str(i + 1) + "_" + temp_dataset.group.species + "_" + \ + this_group_name + "_" + datetime.datetime.now().strftime("%m%d%H%M%S") + this_vals_string = "" + position = 0 + for sample in temp_dataset.group.all_samples_ordered(): + if sample in self.shared_samples_list: + this_vals_string += str(pca_trait[position]) + this_vals_string += " " + position += 1 + else: + this_vals_string += "x " + this_vals_string = this_vals_string[:-1] + + Redis.set(trait_id, this_vals_string, ex=THIRTY_DAYS) + self.pca_trait_ids.append(trait_id) + + return pca def process_loadings(self): loadings_array = [] diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index f0978fd3..54aa6795 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -1,14 +1,36 @@ """This module contains gn2 decorators""" from flask import g +from typing import Dict from functools import wraps +from utility.hmac import hmac_creation +import json +import requests -def admin_login_required(f): + +def edit_access_required(f): """Use this for endpoints where admins are required""" @wraps(f) def wrap(*args, **kwargs): - if g.user_session.record.get(b"user_email_address") not in [ - b"labwilliams@gmail.com"]: + resource_id: str = "" + if kwargs.get("inbredset_id"): # data type: dataset-publish + resource_id = hmac_creation("dataset-publish:" + f"{kwargs.get('inbredset_id')}:" + f"{kwargs.get('name')}") + if kwargs.get("dataset_name"): # data type: dataset-probe + resource_id = hmac_creation("dataset-probeset:" + f"{kwargs.get('dataset_name')}") + response: Dict = {} + try: + _user_id = g.user_session.record.get(b"user_id", + "").decode("utf-8") + response = json.loads( + requests.get("http://localhost:8080/" + "available?resource=" + f"{resource_id}&user={_user_id}").content) + except: + response = {} + if "edit" not in response.get("data", []): return "You need to be admin", 401 return f(*args, **kwargs) return wrap diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index ec17d3b0..6254b9b9 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -24,6 +24,7 @@ # # Last updated by Zach 12/14/2010 +import datetime import string from math import * from PIL import Image @@ -271,6 +272,7 @@ class DisplayMappingResults: # Needing for form submission when doing single chr # mapping or remapping after changing options self.sample_vals = start_vars['sample_vals'] + self.vals_hash= start_vars['vals_hash'] self.sample_vals_dict = json.loads(self.sample_vals) self.transform = start_vars['transform'] @@ -355,8 +357,7 @@ class DisplayMappingResults: if 'use_loco' in list(start_vars.keys()) and self.mapping_method == "gemma": self.use_loco = start_vars['use_loco'] - if 'reaper_version' in list(start_vars.keys()) and self.mapping_method == "reaper": - self.reaper_version = start_vars['reaper_version'] + if self.mapping_method == "reaper": if 'output_files' in start_vars: self.output_files = ",".join( [(the_file if the_file is not None else "") for the_file in start_vars['output_files']]) @@ -651,7 +652,7 @@ class DisplayMappingResults: btminfo.append( 'Mapping using genotype data as a trait will result in infinity LRS at one locus. In order to display the result properly, all LRSs higher than 100 are capped at 100.') - def plotIntMapping(self, canvas, offset=(80, 120, 90, 100), zoom=1, startMb=None, endMb=None, showLocusForm=""): + def plotIntMapping(self, canvas, offset=(80, 120, 110, 100), zoom=1, startMb=None, endMb=None, showLocusForm=""): im_drawer = ImageDraw.Draw(canvas) # calculating margins xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset @@ -661,7 +662,7 @@ class DisplayMappingResults: if self.legendChecked: yTopOffset += 10 if self.covariates != "" and self.controlLocus and self.doControl != "false": - yTopOffset += 20 + yTopOffset += 25 if len(self.transform) > 0: yTopOffset += 5 else: @@ -861,6 +862,9 @@ class DisplayMappingResults: (item[1], yZero - item[2] * bootHeightThresh / maxBootCount)), fill=self.BOOTSTRAP_BOX_COLOR, outline=BLACK) + if maxBootCount == 0: + return + # draw boot scale highestPercent = (maxBootCount * 100.0) / nboot bootScale = Plot.detScale(0, highestPercent) @@ -1192,43 +1196,47 @@ class DisplayMappingResults: dataset_label = "%s - %s" % (self.dataset.group.name, self.dataset.fullname) - string1 = 'Dataset: %s' % (dataset_label) + + self.current_datetime = datetime.datetime.now().strftime("%b %d %Y %H:%M:%S") + string1 = 'UTC Timestamp: %s' % (self.current_datetime) + string2 = 'Dataset: %s' % (dataset_label) + string3 = 'Trait Hash: %s' % (self.vals_hash) if self.genofile_string == "": - string2 = 'Genotype File: %s.geno' % self.dataset.group.name + string4 = 'Genotype File: %s.geno' % self.dataset.group.name else: - string2 = 'Genotype File: %s' % self.genofile_string + string4 = 'Genotype File: %s' % self.genofile_string.split(":")[1] - string4 = '' + string6 = '' if self.mapping_method == "gemma" or self.mapping_method == "gemma_bimbam": if self.use_loco == "True": - string3 = 'Using GEMMA mapping method with LOCO and ' + string5 = 'Using GEMMA mapping method with LOCO and ' else: - string3 = 'Using GEMMA mapping method with ' + string5 = 'Using GEMMA mapping method with ' if self.covariates != "": - string3 += 'the cofactors below:' + string5 += 'the cofactors below:' cofactor_names = ", ".join( [covar.split(":")[0] for covar in self.covariates.split(",")]) - string4 = cofactor_names + string6 = cofactor_names else: - string3 += 'no cofactors' + string5 += 'no cofactors' elif self.mapping_method == "rqtl_plink" or self.mapping_method == "rqtl_geno": - string3 = 'Using R/qtl mapping method with ' + string5 = 'Using R/qtl mapping method with ' if self.covariates != "": - string3 += 'the cofactors below:' + string5 += 'the cofactors below:' cofactor_names = ", ".join( [covar.split(":")[0] for covar in self.covariates.split(",")]) - string4 = cofactor_names + string6 = cofactor_names elif self.controlLocus and self.doControl != "false": - string3 += '%s as control' % self.controlLocus + string5 += '%s as control' % self.controlLocus else: - string3 += 'no cofactors' + string5 += 'no cofactors' else: - string3 = 'Using Haldane mapping function with ' + string5 = 'Using Haldane mapping function with ' if self.controlLocus and self.doControl != "false": - string3 += '%s as control' % self.controlLocus + string5 += '%s as control' % self.controlLocus else: - string3 += 'no control for other QTLs' + string5 += 'no control for other QTLs' y_constant = 10 if self.this_trait.name: @@ -1240,24 +1248,26 @@ class DisplayMappingResults: if self.this_trait.symbol: identification += "Trait: %s - %s" % ( - self.this_trait.name, self.this_trait.symbol) + self.this_trait.display_name, self.this_trait.symbol) elif self.dataset.type == "Publish": if self.this_trait.post_publication_abbreviation: identification += "Trait: %s - %s" % ( - self.this_trait.name, self.this_trait.post_publication_abbreviation) + self.this_trait.display_name, self.this_trait.post_publication_abbreviation) elif self.this_trait.pre_publication_abbreviation: identification += "Trait: %s - %s" % ( - self.this_trait.name, self.this_trait.pre_publication_abbreviation) + self.this_trait.display_name, self.this_trait.pre_publication_abbreviation) else: - identification += "Trait: %s" % (self.this_trait.name) + identification += "Trait: %s" % (self.this_trait.display_name) else: - identification += "Trait: %s" % (self.this_trait.name) + identification += "Trait: %s" % (self.this_trait.display_name) identification += " with %s samples" % (self.n_samples) d = 4 + max( im_drawer.textsize(identification, font=labelFont)[0], im_drawer.textsize(string1, font=labelFont)[0], - im_drawer.textsize(string2, font=labelFont)[0]) + im_drawer.textsize(string2, font=labelFont)[0], + im_drawer.textsize(string3, font=labelFont)[0], + im_drawer.textsize(string4, font=labelFont)[0]) im_drawer.text( text=identification, xy=(xLeftOffset, y_constant * fontZoom), font=labelFont, @@ -1266,7 +1276,9 @@ class DisplayMappingResults: else: d = 4 + max( im_drawer.textsize(string1, font=labelFont)[0], - im_drawer.textsize(string2, font=labelFont)[0]) + im_drawer.textsize(string2, font=labelFont)[0], + im_drawer.textsize(string3, font=labelFont)[0], + im_drawer.textsize(string4, font=labelFont)[0]) if len(self.transform) > 0: transform_text = "Transform - " @@ -1293,14 +1305,22 @@ class DisplayMappingResults: text=string2, xy=(xLeftOffset, y_constant * fontZoom), font=labelFont, fill=labelColor) y_constant += 15 - if string3 != '': + im_drawer.text( + text=string3, xy=(xLeftOffset, y_constant * fontZoom), + font=labelFont, fill=labelColor) + y_constant += 15 + im_drawer.text( + text=string4, xy=(xLeftOffset, y_constant * fontZoom), + font=labelFont, fill=labelColor) + y_constant += 15 + if string4 != '': im_drawer.text( - text=string3, xy=(xLeftOffset, y_constant * fontZoom), + text=string5, xy=(xLeftOffset, y_constant * fontZoom), font=labelFont, fill=labelColor) y_constant += 15 - if string4 != '': + if string5 != '': im_drawer.text( - text=string4, xy=(xLeftOffset, y_constant * fontZoom), + text=string6, xy=(xLeftOffset, y_constant * fontZoom), font=labelFont, fill=labelColor) def drawGeneBand(self, canvas, gifmap, plotXScale, offset=(40, 120, 80, 10), zoom=1, startMb=None, endMb=None): @@ -2110,7 +2130,7 @@ class DisplayMappingResults: thisChr.append( [_locus.name, _locus.cM - Locus0CM]) else: - for j in (0, nLoci / 4, nLoci / 2, nLoci * 3 / 4, -1): + for j in (0, round(nLoci / 4), round(nLoci / 2), round(nLoci * 3 / 4), -1): while _chr[j].name == ' - ': j += 1 if _chr[j].cM != preLpos: @@ -2286,20 +2306,9 @@ class DisplayMappingResults: font=VERDANA_FILE, size=int(18 * zoom * 1.5)) yZero = yTopOffset + plotHeight - # LRSHeightThresh = drawAreaHeight - # AdditiveHeightThresh = drawAreaHeight/2 - # DominanceHeightThresh = drawAreaHeight/2 - if self.selectedChr == 1: - LRSHeightThresh = drawAreaHeight - yTopOffset + 30 * (zoom - 1) - AdditiveHeightThresh = LRSHeightThresh / 2 - DominanceHeightThresh = LRSHeightThresh / 2 - else: - LRSHeightThresh = drawAreaHeight - AdditiveHeightThresh = drawAreaHeight / 2 - DominanceHeightThresh = drawAreaHeight / 2 - # LRSHeightThresh = (yZero - yTopOffset + 30*(zoom - 1)) - # AdditiveHeightThresh = LRSHeightThresh/2 - # DominanceHeightThresh = LRSHeightThresh/2 + LRSHeightThresh = drawAreaHeight + AdditiveHeightThresh = drawAreaHeight / 2 + DominanceHeightThresh = drawAreaHeight / 2 if LRS_LOD_Max > 100: LRSScale = 20.0 @@ -2380,8 +2389,7 @@ class DisplayMappingResults: # ZS: I don't know if what I did here with this inner function is clever or overly complicated, but it's the only way I could think of to avoid duplicating the code inside this function def add_suggestive_significant_lines_and_legend(start_pos_x, chr_length_dist): - rightEdge = int(start_pos_x + chr_length_dist * \ - plotXScale - self.SUGGESTIVE_WIDTH / 1.5) + rightEdge = xLeftOffset + plotWidth im_drawer.line( xy=((start_pos_x + self.SUGGESTIVE_WIDTH / 1.5, suggestiveY), (rightEdge, suggestiveY)), @@ -2569,7 +2577,10 @@ class DisplayMappingResults: Xc = startPosX + ((qtlresult['Mb'] - start_cm - startMb) * plotXScale) * ( ((qtlresult['Mb'] - start_cm - startMb) * plotXScale) / ((qtlresult['Mb'] - start_cm - startMb + self.GraphInterval) * plotXScale)) else: - Xc = startPosX + (qtlresult['Mb'] - startMb) * plotXScale + if self.selectedChr != -1 and qtlresult['Mb'] > endMb: + Xc = startPosX + endMb * plotXScale + else: + Xc = startPosX + (qtlresult['Mb'] - startMb) * plotXScale # updated by NL 06-18-2011: # fix the over limit LRS graph issue since genotype trait may give infinite LRS; @@ -2580,36 +2591,29 @@ class DisplayMappingResults: if 'lrs_value' in qtlresult: if self.LRS_LOD == "LOD" or self.LRS_LOD == "-logP": if qtlresult['lrs_value'] > 460 or qtlresult['lrs_value'] == 'inf': - #Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/(LRSAxisList[-1]*self.LODFACTOR) Yc = yZero - webqtlConfig.MAXLRS * \ LRSHeightThresh / \ (LRS_LOD_Max * self.LODFACTOR) else: - #Yc = yZero - qtlresult['lrs_value']*LRSHeightThresh/(LRSAxisList[-1]*self.LODFACTOR) Yc = yZero - \ qtlresult['lrs_value'] * LRSHeightThresh / \ (LRS_LOD_Max * self.LODFACTOR) else: if qtlresult['lrs_value'] > 460 or qtlresult['lrs_value'] == 'inf': - #Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/LRSAxisList[-1] Yc = yZero - webqtlConfig.MAXLRS * LRSHeightThresh / LRS_LOD_Max else: - #Yc = yZero - qtlresult['lrs_value']*LRSHeightThresh/LRSAxisList[-1] Yc = yZero - \ qtlresult['lrs_value'] * \ LRSHeightThresh / LRS_LOD_Max else: if qtlresult['lod_score'] > 100 or qtlresult['lod_score'] == 'inf': - #Yc = yZero - webqtlConfig.MAXLRS*LRSHeightThresh/LRSAxisList[-1] Yc = yZero - webqtlConfig.MAXLRS * LRSHeightThresh / LRS_LOD_Max else: if self.LRS_LOD == "LRS": - #Yc = yZero - qtlresult['lod_score']*self.LODFACTOR*LRSHeightThresh/LRSAxisList[-1] Yc = yZero - \ qtlresult['lod_score'] * self.LODFACTOR * \ LRSHeightThresh / LRS_LOD_Max else: - #Yc = yZero - qtlresult['lod_score']*LRSHeightThresh/LRSAxisList[-1] Yc = yZero - \ qtlresult['lod_score'] * \ LRSHeightThresh / LRS_LOD_Max @@ -2642,14 +2646,12 @@ class DisplayMappingResults: AdditiveHeightThresh / additiveMax AdditiveCoordXY.append((Xc, Yc)) + if self.selectedChr != -1 and qtlresult['Mb'] > endMb: + break + m += 1 if self.manhattan_plot != True: - # im_drawer.polygon( - # xy=LRSCoordXY, - # outline=thisLRSColor - # #, closed=0, edgeWidth=lrsEdgeWidth, clipX=(xLeftOffset, xLeftOffset + plotWidth) - # ) draw_open_polygon(canvas, xy=LRSCoordXY, outline=thisLRSColor, width=lrsEdgeWidth) diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index f88c5ac8..623ab87f 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -11,6 +11,7 @@ from utility.tools import flat_files from utility.tools import GEMMA_WRAPPER_COMMAND from utility.tools import TEMPDIR from utility.tools import WEBSERVER_MODE +from gn3.computations.gemma import generate_hash_of_string import utility.logger logger = utility.logger.getLogger(__name__) @@ -34,10 +35,7 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, genofile_name = this_dataset.group.name if first_run: - trait_filename = (f"{str(this_dataset.group.name)}_" - f"{str(this_trait.name)}_" - f"{generate_random_n_string(6)}") - gen_pheno_txt_file(this_dataset, genofile_name, vals, trait_filename) + pheno_filename = gen_pheno_txt_file(this_dataset, genofile_name, vals) if not os.path.isfile(f"{webqtlConfig.GENERATED_IMAGE_DIR}" f"{genofile_name}_output.assoc.txt"): @@ -56,13 +54,13 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, chr_list_string = ",".join(this_chromosomes_name) if covariates != "": - gen_covariates_file(this_dataset, covariates, samples) + covar_filename = gen_covariates_file(this_dataset, covariates, samples) if use_loco == "True": generate_k_command = (f"{GEMMA_WRAPPER_COMMAND} --json --loco " f"{chr_list_string} -- {GEMMAOPTS} " f"-g {flat_files('genotype/bimbam')}/" f"{genofile_name}_geno.txt -p " - f"{TEMPDIR}/gn2/{trait_filename}.txt -a " + f"{TEMPDIR}/gn2/{pheno_filename}.txt -a " f"{flat_files('genotype/bimbam')}/" f"{genofile_name}_snps.txt -gk > " f"{TEMPDIR}/gn2/{k_output_filename}.json") @@ -73,10 +71,10 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, f"-- {GEMMAOPTS} " f"-g {flat_files('genotype/bimbam')}/" f"{genofile_name}_geno.txt " - f"-p {TEMPDIR}/gn2/{trait_filename}.txt ") + f"-p {TEMPDIR}/gn2/{pheno_filename}.txt ") if covariates != "": gemma_command += (f"-c {flat_files('mapping')}/" - f"{this_dataset.group.name}_covariates.txt " + f"{covar_filename}.txt " f"-a {flat_files('genotype/bimbam')}/" f"{genofile_name}_snps.txt " f"-lmm 9 -maf {maf} > {TEMPDIR}/gn2/" @@ -92,7 +90,7 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, f"{GEMMAOPTS} " f" -g {flat_files('genotype/bimbam')}/" f"{genofile_name}_geno.txt -p " - f"{TEMPDIR}/gn2/{trait_filename}.txt -a " + f"{TEMPDIR}/gn2/{pheno_filename}.txt -a " f"{flat_files('genotype/bimbam')}/" f"{genofile_name}_snps.txt -gk > " f"{TEMPDIR}/gn2/{k_output_filename}.json") @@ -106,12 +104,11 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, f"{genofile_name}_snps.txt " f"-lmm 9 -g {flat_files('genotype/bimbam')}/" f"{genofile_name}_geno.txt -p " - f"{TEMPDIR}/gn2/{trait_filename}.txt ") + f"{TEMPDIR}/gn2/{pheno_filename}.txt ") if covariates != "": gemma_command += (f" -c {flat_files('mapping')}/" - f"{this_dataset.group.name}" - f"_covariates.txt > " + f"{covar_filename}.txt > " f"{TEMPDIR}/gn2/{gwa_output_filename}.json") else: gemma_command += f" > {TEMPDIR}/gn2/{gwa_output_filename}.json" @@ -129,16 +126,20 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, return marker_obs, gwa_output_filename -def gen_pheno_txt_file(this_dataset, genofile_name, vals, trait_filename): +def gen_pheno_txt_file(this_dataset, genofile_name, vals): """Generates phenotype file for GEMMA""" - with open(f"{TEMPDIR}/gn2/{trait_filename}.txt", "w") as outfile: + filename = "PHENO_" + generate_hash_of_string(this_dataset.name + str(vals)).replace("/", "_") + + with open(f"{TEMPDIR}/gn2/{filename}.txt", "w") as outfile: for value in vals: if value == "x": outfile.write("NA\n") else: outfile.write(value + "\n") + return filename + def gen_covariates_file(this_dataset, covariates, samples): covariate_list = covariates.split(",") @@ -168,14 +169,18 @@ def gen_covariates_file(this_dataset, covariates, samples): this_covariate_data.append("-9") covariate_data_object.append(this_covariate_data) + filename = "COVAR_" + generate_hash_of_string(this_dataset.name + str(covariate_data_object)).replace("/", "_") + with open((f"{flat_files('mapping')}/" - f"{this_dataset.group.name}_covariates.txt"), + f"{filename}.txt"), "w") as outfile: for i in range(len(covariate_data_object[0])): for this_covariate in covariate_data_object: outfile.write(str(this_covariate[i]) + "\t") outfile.write("\n") + return filename + def parse_loco_output(this_dataset, gwa_output_filename, loco="True"): diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py index 4d6715ba..801674e1 100644 --- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py +++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py @@ -178,101 +178,6 @@ def parse_reaper_output(gwa_filename, permu_filename, bootstrap_filename): return marker_obs, permu_vals, bootstrap_vals -def run_original_reaper(this_trait, dataset, samples_before, trait_vals, json_data, num_perm, bootCheck, num_bootstrap, do_control, control_marker, manhattan_plot): - genotype = dataset.group.read_genotype_file(use_reaper=True) - - if manhattan_plot != True: - genotype = genotype.addinterval() - - trimmed_samples = [] - trimmed_values = [] - for i in range(0, len(samples_before)): - try: - trimmed_values.append(float(trait_vals[i])) - trimmed_samples.append(str(samples_before[i])) - except: - pass - - perm_output = [] - bootstrap_results = [] - - if num_perm < 100: - suggestive = 0 - significant = 0 - else: - perm_output = genotype.permutation( - strains=trimmed_samples, trait=trimmed_values, nperm=num_perm) - suggestive = perm_output[int(num_perm * 0.37 - 1)] - significant = perm_output[int(num_perm * 0.95 - 1)] - # highly_significant = perm_output[int(num_perm*0.99-1)] #ZS: Currently not used, but leaving it here just in case - - json_data['suggestive'] = suggestive - json_data['significant'] = significant - - if control_marker != "" and do_control == "true": - reaper_results = genotype.regression(strains=trimmed_samples, - trait=trimmed_values, - control=str(control_marker)) - if bootCheck: - control_geno = [] - control_geno2 = [] - _FIND = 0 - for _chr in genotype: - for _locus in _chr: - if _locus.name == control_marker: - control_geno2 = _locus.genotype - _FIND = 1 - break - if _FIND: - break - if control_geno2: - _prgy = list(genotype.prgy) - for _strain in trimmed_samples: - _idx = _prgy.index(_strain) - control_geno.append(control_geno2[_idx]) - - bootstrap_results = genotype.bootstrap(strains=trimmed_samples, - trait=trimmed_values, - control=control_geno, - nboot=num_bootstrap) - else: - reaper_results = genotype.regression(strains=trimmed_samples, - trait=trimmed_values) - - if bootCheck: - bootstrap_results = genotype.bootstrap(strains=trimmed_samples, - trait=trimmed_values, - nboot=num_bootstrap) - - json_data['chr'] = [] - json_data['pos'] = [] - json_data['lod.hk'] = [] - json_data['markernames'] = [] - # if self.additive: - # self.json_data['additive'] = [] - - # Need to convert the QTL objects that qtl reaper returns into a json serializable dictionary - qtl_results = [] - for qtl in reaper_results: - reaper_locus = qtl.locus - # ZS: Convert chr to int - converted_chr = reaper_locus.chr - if reaper_locus.chr != "X" and reaper_locus.chr != "X/Y": - converted_chr = int(reaper_locus.chr) - json_data['chr'].append(converted_chr) - json_data['pos'].append(reaper_locus.Mb) - json_data['lod.hk'].append(qtl.lrs) - json_data['markernames'].append(reaper_locus.name) - # if self.additive: - # self.json_data['additive'].append(qtl.additive) - locus = {"name": reaper_locus.name, "chr": reaper_locus.chr, - "cM": reaper_locus.cM, "Mb": reaper_locus.Mb} - qtl = {"lrs_value": qtl.lrs, "chr": converted_chr, "Mb": reaper_locus.Mb, - "cM": reaper_locus.cM, "name": reaper_locus.name, "additive": qtl.additive, "dominance": qtl.dominance} - qtl_results.append(qtl) - return qtl_results, json_data, perm_output, suggestive, significant, bootstrap_results - - def natural_sort(marker_list): """ Function to naturally sort numbers + strings, adopted from user Mark Byers here: https://stackoverflow.com/questions/4836710/does-python-have-a-built-in-function-for-string-natural-sort diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 09afb8d1..1dca1b1b 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -39,7 +39,7 @@ def run_rqtl(trait_name, vals, samples, dataset, mapping_scale, model, method, n } if do_control == "true" and control_marker: - post_data["control_marker"] = control_marker + post_data["control"] = control_marker if not manhattan_plot: post_data["interval"] = True @@ -61,6 +61,7 @@ def get_hash_of_textio(the_file: TextIO) -> str: the_file.seek(0) hash_of_file = hashlib.md5(the_file.read().encode()).hexdigest() + hash_of_file = hash_of_file.replace("/", "_") # Replace / with _ to prevent issue with filenames being translated to directories return hash_of_file @@ -89,7 +90,7 @@ def write_phenotype_file(trait_name: str, for i, sample in enumerate(samples): this_row = [sample] if vals[i] != "x": - this_row.append(vals[i]) + this_row.append(str(round(float(vals[i]), 3))) else: this_row.append("NA") for cofactor in cofactor_data: @@ -126,7 +127,7 @@ def cofactors_to_dict(cofactors: str, dataset_ob, samples) -> Dict: sample_data = trait_ob.data for index, sample in enumerate(samples): if sample in sample_data: - sample_value = sample_data[sample].value + sample_value = str(round(float(sample_data[sample].value), 3)) cofactor_dict[cofactor_name].append(sample_value) else: cofactor_dict[cofactor_name].append("NA") diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index c5b980a7..290c4a14 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -75,6 +75,7 @@ class RunMapping: self.vals = [] self.samples = [] self.sample_vals = start_vars['sample_vals'] + self.vals_hash = start_vars['vals_hash'] sample_val_dict = json.loads(self.sample_vals) samples = sample_val_dict.keys() if (len(genofile_samplelist) != 0): @@ -103,9 +104,7 @@ class RunMapping: if "results_path" in start_vars: self.mapping_results_path = start_vars['results_path'] else: - mapping_results_filename = self.dataset.group.name + "_" + \ - ''.join(random.choice(string.ascii_uppercase + string.digits) - for _ in range(6)) + mapping_results_filename = "_".join([self.dataset.group.name, self.vals_hash]).replace("/", "_") self.mapping_results_path = "{}{}.csv".format( webqtlConfig.GENERATED_IMAGE_DIR, mapping_results_filename) @@ -220,7 +219,7 @@ class RunMapping: elif self.mapping_method == "rqtl_plink": results = self.run_rqtl_plink() elif self.mapping_method == "rqtl_geno": - perm_strata = [] + self.perm_strata = [] if "perm_strata" in start_vars and "categorical_vars" in start_vars: self.categorical_vars = start_vars["categorical_vars"].split( ",") @@ -229,7 +228,7 @@ class RunMapping: sample_names=self.samples, this_trait=self.this_trait) - perm_strata = get_perm_strata( + self.perm_strata = get_perm_strata( self.this_trait, primary_samples, self.categorical_vars, self.samples) self.score_type = "LOD" self.control_marker = start_vars['control_marker'] @@ -243,10 +242,10 @@ class RunMapping: # self.pair_scan = True if self.permCheck and self.num_perm > 0: self.perm_output, self.suggestive, self.significant, results = rqtl_mapping.run_rqtl( - self.this_trait.name, self.vals, self.samples, self.dataset, self.mapping_scale, self.model, self.method, self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.covariates) + self.this_trait.name, self.vals, self.samples, self.dataset, self.mapping_scale, self.model, self.method, self.num_perm, self.perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.covariates) else: results = rqtl_mapping.run_rqtl(self.this_trait.name, self.vals, self.samples, self.dataset, self.mapping_scale, self.model, self.method, - self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.covariates) + self.num_perm, self.perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.covariates) elif self.mapping_method == "reaper": if "startMb" in start_vars: # ZS: Check if first time page loaded, so it can default to ON if "additiveCheck" in start_vars: @@ -272,47 +271,32 @@ class RunMapping: self.bootCheck = False self.num_bootstrap = 0 - self.reaper_version = start_vars['reaper_version'] - self.control_marker = start_vars['control_marker'] self.do_control = start_vars['do_control'] logger.info("Running qtlreaper") - if self.reaper_version == "new": - self.first_run = True - self.output_files = None - # ZS: check if first run so existing result files can be used if it isn't (for example zooming on a chromosome, etc) - if 'first_run' in start_vars: - self.first_run = False - if 'output_files' in start_vars: - self.output_files = start_vars['output_files'].split( - ",") - - results, self.perm_output, self.suggestive, self.significant, self.bootstrap_results, self.output_files = qtlreaper_mapping.run_reaper(self.this_trait, - self.dataset, - self.samples, - self.vals, - self.json_data, - self.num_perm, - self.bootCheck, - self.num_bootstrap, - self.do_control, - self.control_marker, - self.manhattan_plot, - self.first_run, - self.output_files) - else: - results, self.json_data, self.perm_output, self.suggestive, self.significant, self.bootstrap_results = qtlreaper_mapping.run_original_reaper(self.this_trait, - self.dataset, - self.samples, - self.vals, - self.json_data, - self.num_perm, - self.bootCheck, - self.num_bootstrap, - self.do_control, - self.control_marker, - self.manhattan_plot) + self.first_run = True + self.output_files = None + # ZS: check if first run so existing result files can be used if it isn't (for example zooming on a chromosome, etc) + if 'first_run' in start_vars: + self.first_run = False + if 'output_files' in start_vars: + self.output_files = start_vars['output_files'].split( + ",") + + results, self.perm_output, self.suggestive, self.significant, self.bootstrap_results, self.output_files = qtlreaper_mapping.run_reaper(self.this_trait, + self.dataset, + self.samples, + self.vals, + self.json_data, + self.num_perm, + self.bootCheck, + self.num_bootstrap, + self.do_control, + self.control_marker, + self.manhattan_plot, + self.first_run, + self.output_files) elif self.mapping_method == "plink": self.score_type = "-logP" self.manhattan_plot = True @@ -421,8 +405,9 @@ class RunMapping: total_markers = len(self.qtl_results) with Bench("Exporting Results"): - export_mapping_results(self.dataset, self.this_trait, self.qtl_results, self.mapping_results_path, - self.mapping_scale, self.score_type, self.transform, self.covariates, self.n_samples) + export_mapping_results(self.dataset, self.this_trait, self.qtl_results, + self.mapping_results_path, self.mapping_scale, self.score_type, + self.transform, self.covariates, self.n_samples, self.vals_hash) with Bench("Trimming Markers for Figure"): if len(self.qtl_results) > 30000: @@ -540,13 +525,15 @@ class RunMapping: return trimmed_genotype_data -def export_mapping_results(dataset, trait, markers, results_path, mapping_scale, score_type, transform, covariates, n_samples): +def export_mapping_results(dataset, trait, markers, results_path, mapping_scale, score_type, transform, covariates, n_samples, vals_hash): with open(results_path, "w+") as output_file: output_file.write( "Time/Date: " + datetime.datetime.now().strftime("%x / %X") + "\n") output_file.write( "Population: " + dataset.group.species.title() + " " + dataset.group.name + "\n") output_file.write("Data Set: " + dataset.fullname + "\n") + output_file.write("Trait: " + trait.display_name + "\n") + output_file.write("Trait Hash: " + vals_hash + "\n") output_file.write("N Samples: " + str(n_samples) + "\n") if len(transform) > 0: transform_text = "Transform - " @@ -673,9 +660,9 @@ def trim_markers_for_table(markers): sorted_markers = sorted( markers, key=lambda k: k['lrs_value'], reverse=True) - # ZS: So we end up with a list of just 2000 markers - if len(sorted_markers) >= 2000: - trimmed_sorted_markers = sorted_markers[:2000] + #ZS: So we end up with a list of just 2000 markers + if len(sorted_markers) >= 10000: + trimmed_sorted_markers = sorted_markers[:10000] return trimmed_sorted_markers else: return sorted_markers @@ -765,9 +752,9 @@ def get_perm_strata(this_trait, sample_list, categorical_vars, used_samples): if sample in list(sample_list.sample_attribute_values.keys()): combined_string = "" for var in categorical_vars: - if var.lower() in sample_list.sample_attribute_values[sample]: + if var in sample_list.sample_attribute_values[sample]: combined_string += str( - sample_list.sample_attribute_values[sample][var.lower()]) + sample_list.sample_attribute_values[sample][var]) else: combined_string += "NA" else: diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index b28c1b04..c54dd0b3 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -8,8 +8,6 @@ from wqflask import app from utility.authentication_tools import check_owner_or_admin from utility.redis_tools import get_resource_info, get_group_info, get_groups_like_unique_column, get_user_id, get_user_by_unique_column, get_users_like_unique_column, add_access_mask, add_resource, change_resource_owner -from utility.logger import getLogger -logger = getLogger(__name__) @app.route("/resources/manage", methods=('GET', 'POST')) diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index 92cea550..ae30aa59 100644 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -32,7 +32,7 @@ class SampleList: for counter, sample_name in enumerate(sample_names, 1): sample_name = sample_name.replace("_2nd_", "") - # ZS: self.this_trait will be a list if it is a Temp trait + # self.this_trait will be a list if it is a Temp trait if isinstance(self.this_trait, list): sample = webqtlCaseData.webqtlCaseData(name=sample_name) if counter <= len(self.this_trait): @@ -47,7 +47,7 @@ class SampleList: name=sample_name, value=float(self.this_trait[counter - 1])) else: - # ZS - If there's no value for the sample/strain, + # If there's no value for the sample/strain, # create the sample object (so samples with no value # are still displayed in the table) try: @@ -63,29 +63,29 @@ class SampleList: sample.this_id = str(counter) - # ZS: For extra attribute columns; currently only used by + # For extra attribute columns; currently only used by # several datasets if self.sample_attribute_values: sample.extra_attributes = self.sample_attribute_values.get( sample_name, {}) - # ZS: Add a url so RRID case attributes can be displayed as links - if 'rrid' in sample.extra_attributes: + # Add a url so RRID case attributes can be displayed as links + if '36' in sample.extra_attributes: if self.dataset.group.species == "mouse": - if len(sample.extra_attributes['rrid'].split(":")) > 1: - the_rrid = sample.extra_attributes['rrid'].split(":")[ + if len(sample.extra_attributes['36'].split(":")) > 1: + the_rrid = sample.extra_attributes['36'].split(":")[ 1] - sample.extra_attributes['rrid'] = [ - sample.extra_attributes['rrid']] - sample.extra_attributes['rrid'].append( + sample.extra_attributes['36'] = [ + sample.extra_attributes['36']] + sample.extra_attributes['36'].append( webqtlConfig.RRID_MOUSE_URL % the_rrid) elif self.dataset.group.species == "rat": - if len(str(sample.extra_attributes['rrid'])): - the_rrid = sample.extra_attributes['rrid'].split("_")[ + if len(str(sample.extra_attributes['36'])): + the_rrid = sample.extra_attributes['36'].split("_")[ 1] - sample.extra_attributes['rrid'] = [ - sample.extra_attributes['rrid']] - sample.extra_attributes['rrid'].append( + sample.extra_attributes['36'] = [ + sample.extra_attributes['36']] + sample.extra_attributes['36'].append( webqtlConfig.RRID_RAT_URL % the_rrid) self.sample_list.append(sample) @@ -124,17 +124,19 @@ class SampleList: # Get attribute names and distinct values for each attribute results = g.db.execute(''' - SELECT DISTINCT CaseAttribute.Id, CaseAttribute.Name, CaseAttributeXRefNew.Value + SELECT DISTINCT CaseAttribute.Id, CaseAttribute.Name, CaseAttribute.Description, CaseAttributeXRefNew.Value FROM CaseAttribute, CaseAttributeXRefNew WHERE CaseAttributeXRefNew.CaseAttributeId = CaseAttribute.Id AND CaseAttributeXRefNew.InbredSetId = %s - ORDER BY lower(CaseAttribute.Name)''', (str(self.dataset.group.id),)) + ORDER BY CaseAttribute.Id''', (str(self.dataset.group.id),)) self.attributes = {} - for attr, values in itertools.groupby(results.fetchall(), lambda row: (row.Id, row.Name)): - key, name = attr + for attr, values in itertools.groupby(results.fetchall(), lambda row: (row.Id, row.Name, row.Description)): + key, name, description = attr self.attributes[key] = Bunch() + self.attributes[key].id = key self.attributes[key].name = name + self.attributes[key].description = description self.attributes[key].distinct_values = [ item.Value for item in values] self.attributes[key].distinct_values = natural_sort( @@ -168,10 +170,13 @@ class SampleList: for sample_name, items in itertools.groupby(results.fetchall(), lambda row: row.SampleName): attribute_values = {} + # Make a list of attr IDs without values (that have values for other samples) + valueless_attr_ids = [self.attributes[key].id for key in self.attributes.keys()] for item in items: + valueless_attr_ids.remove(item.Id) attribute_value = item.Value - # ZS: If it's an int, turn it into one for sorting + # If it's an int, turn it into one for sorting # (for example, 101 would be lower than 80 if # they're strings instead of ints) try: @@ -179,8 +184,10 @@ class SampleList: except ValueError: pass - attribute_values[self.attributes[item.Id].name.lower( - )] = attribute_value + attribute_values[str(item.Id)] = attribute_value + for attr_id in valueless_attr_ids: + attribute_values[str(attr_id)] = "" + self.sample_attribute_values[sample_name] = attribute_values def get_first_attr_col(self): diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index c07430dd..c4d1ae1c 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -1,3 +1,5 @@ +from typing import Dict + import string import datetime import uuid @@ -176,11 +178,11 @@ class ShowTrait: self.sample_group_types['samples_primary'] = self.dataset.group.name sample_lists = [group.sample_list for group in self.sample_groups] - categorical_var_list = [] + self.categorical_var_list = [] self.numerical_var_list = [] if not self.temp_trait: # ZS: Only using first samplelist, since I think mapping only uses those samples - categorical_var_list = get_categorical_variables( + self.categorical_var_list = get_categorical_variables( self.this_trait, self.sample_groups[0]) self.numerical_var_list = get_numerical_variables( self.this_trait, self.sample_groups[0]) @@ -192,6 +194,8 @@ class ShowTrait: [self.dataset.species.chromosomes.chromosomes[this_chr].name, i]) self.genofiles = self.dataset.group.get_genofiles() + study_samplelist_json = self.dataset.group.get_study_samplelists() + self.study_samplelists = [study["title"] for study in study_samplelist_json] # ZS: No need to grab scales from .geno file unless it's using # a mapping method that reads .geno files @@ -280,10 +284,11 @@ class ShowTrait: hddn['selected_chr'] = -1 hddn['mapping_display_all'] = True hddn['suggestive'] = 0 + hddn['study_samplelists'] = json.dumps(study_samplelist_json) hddn['num_perm'] = 0 hddn['categorical_vars'] = "" - if categorical_var_list: - hddn['categorical_vars'] = ",".join(categorical_var_list) + if self.categorical_var_list: + hddn['categorical_vars'] = ",".join(self.categorical_var_list) hddn['manhattan_plot'] = "" hddn['control_marker'] = "" if not self.temp_trait: @@ -295,7 +300,7 @@ class ShowTrait: hddn['compare_traits'] = [] hddn['export_data'] = "" hddn['export_format'] = "excel" - if len(self.scales_in_geno) < 2: + if len(self.scales_in_geno) < 2 and bool(self.scales_in_geno): hddn['mapping_scale'] = self.scales_in_geno[list( self.scales_in_geno.keys())[0]][0][0] @@ -318,7 +323,7 @@ class ShowTrait: has_num_cases=self.has_num_cases, attributes=self.sample_groups[0].attributes, categorical_attr_exists=self.categorical_attr_exists, - categorical_vars=",".join(categorical_var_list), + categorical_vars=",".join(self.categorical_var_list), num_values=self.num_values, qnorm_values=self.qnorm_vals, zscore_values=self.z_scores, @@ -520,6 +525,9 @@ class ShowTrait: sample_group_type='primary', header="%s Only" % (self.dataset.group.name)) self.sample_groups = (primary_samples,) + print("\nttttttttttttttttttttttttttttttttttttttttttttt\n") + print(self.sample_groups) + print("\nttttttttttttttttttttttttttttttttttttttttttttt\n") self.primary_sample_names = primary_sample_names self.dataset.group.allsamples = all_samples_ordered @@ -693,7 +701,7 @@ def get_categorical_variables(this_trait, sample_list) -> list: if len(sample_list.attributes) > 0: for attribute in sample_list.attributes: if len(sample_list.attributes[attribute].distinct_values) < 10: - categorical_var_list.append(sample_list.attributes[attribute].name) + categorical_var_list.append(str(sample_list.attributes[attribute].id)) return categorical_var_list @@ -799,3 +807,41 @@ def get_scales_from_genofile(file_location): return [["physic", "Mb"], ["morgan", "cM"]] else: return [["physic", "Mb"]] + + + +def get_diff_of_vals(new_vals: Dict, trait_id: str) -> Dict: + """ Get the diff between current sample values and the values in the DB + + Given a dict of the changed values and the trait/dataset ID, return a Dict + with keys corresponding to each sample with a changed value and a value + that is a dict with keys for the old_value and new_value + + """ + + trait_name = trait_id.split(":")[0] + dataset_name = trait_id.split(":")[1] + trait_ob = create_trait(name=trait_name, dataset_name=dataset_name) + + old_vals = {sample : trait_ob.data[sample].value for sample in trait_ob.data} + + shared_samples = set.union(set(new_vals.keys()), set(old_vals.keys())) + + diff_dict = {} + for sample in shared_samples: + try: + new_val = round(float(new_vals[sample]), 3) + except: + new_val = "x" + try: + old_val = round(float(old_vals[sample]), 3) + except: + old_val = "x" + + if new_val != old_val: + diff_dict[sample] = { + "new_val": new_val, + "old_val": old_val + } + + return diff_dict diff --git a/wqflask/wqflask/static/new/css/bootstrap-custom.css b/wqflask/wqflask/static/new/css/bootstrap-custom.css index 7c8549e1..a0d3ff6a 100644 --- a/wqflask/wqflask/static/new/css/bootstrap-custom.css +++ b/wqflask/wqflask/static/new/css/bootstrap-custom.css @@ -327,7 +327,7 @@ th { font-family: 'Glyphicons Halflings'; src: url('../fonts/glyphicons-halflings-regular.eot'); - src: url('../fonts/glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'), url('../fonts/glyphicons-halflings-regular.woff') format('woff'), url('../fonts/glyphicons-halflings-regular.ttf') format('truetype'), url('../fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular') format('svg'); + src: url('../fonts/glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'), url('../fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular') format('svg'); } .glyphicon { @@ -7554,5 +7554,3 @@ button.close { display: none !important; } } - -/*# sourceMappingURL=bootstrap.css.map */
\ No newline at end of file diff --git a/wqflask/wqflask/static/new/css/show_trait.css b/wqflask/wqflask/static/new/css/show_trait.css index 0486da4d..f5e8c22a 100644 --- a/wqflask/wqflask/static/new/css/show_trait.css +++ b/wqflask/wqflask/static/new/css/show_trait.css @@ -159,10 +159,10 @@ div.normalize-div { } div.mapping-main { - min-width: 1200px; + min-width: 1400px; } div.mapping-options { - min-width: 500px; + min-width: 700px; } div.covar-options { @@ -194,7 +194,7 @@ div.select-covar-div { .selected-covariates { overflow-y: scroll; resize: none; - width: 200px; + width: 400px; } .cofactor-input { @@ -259,3 +259,33 @@ input.trait-value-input { div.inline-div { display: inline; } + +/* div.colorbox_border { + border: 1px solid grey; +} */ +div#cboxContent { + /* box-shadow: + 0 2.8px 2.2px rgba(0, 0, 0, 0.034), + 0 6.7px 5.3px rgba(0, 0, 0, 0.048), + 0 12.5px 10px rgba(0, 0, 0, 0.06), + 0 22.3px 17.9px rgba(0, 0, 0, 0.072), + 0 41.8px 33.4px rgba(0, 0, 0, 0.086), + 0 100px 80px rgba(0, 0, 0, 0.12) */ + + padding: 10px 10px 5px 10px; + + -moz-box-shadow: 3px 3px 5px #535353; + -webkit-box-shadow: 3px 3px 5px #535353; + box-shadow: 3px 3px 5px #535353; + + -moz-border-radius: 6px 6px 6px 6px; + -webkit-border-radius: 6px; + border-radius: 6px 6px 6px 6px; + + /* border: 2px solid grey; */ +} + +#cboxClose { + margin-right: 5px; + margin-bottom: 2px; +} diff --git a/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js b/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js index 3e414034..00025a32 100644 --- a/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js +++ b/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js @@ -65,10 +65,8 @@ if ( ! $.fn.DataTable.isDataTable( '#collection_table' ) ) { collection_click = function() { var this_collection_url; - console.log("Clicking on:", $(this)); this_collection_url = $(this).find('.collection_name').prop("href"); this_collection_url += "&json"; - console.log("this_collection_url", this_collection_url); collection_list = $("#collections_holder").html(); return $.ajax({ dataType: "json", @@ -79,32 +77,57 @@ collection_click = function() { submit_click = function() { var covariates_string = ""; - var covariates_display_string = ""; + var covariates_as_set = new Set(); + $(".selected-covariates:first option").each(function() { + if ($(this).val() != ""){ + covariates_as_set.add($(this).val() + "," + $(this).text()); + } + }); $('#collections_holder').find('input[type=checkbox]:checked').each(function() { var this_dataset, this_trait; this_trait = $(this).parents('tr').find('.trait').text(); this_trait_display = $(this).parents('tr').find('.trait').data("display_name"); this_description = $(this).parents('tr').find('.description').text(); - console.log("this_trait is:", this_trait_display); this_dataset = $(this).parents('tr').find('.dataset').data("dataset"); - console.log("this_dataset is:", this_dataset); - covariates_string += this_trait + ":" + this_dataset + "," - //this_covariate_display_string = this_trait + ": " + this_description this_covariate_display_string = this_trait_display if (this_covariate_display_string.length > 50) { this_covariate_display_string = this_covariate_display_string.substring(0, 45) + "..." } - covariates_display_string += this_covariate_display_string + "\n" + covariates_as_set.add(this_trait + ":" + this_dataset + "," + this_covariate_display_string) + }); + + covariates_as_list = Array.from(covariates_as_set) + + // Removed the starting "No covariates selected" option before adding options for each covariate + if (covariates_as_list.length > 0){ + $(".selected-covariates option[value='']").each(function() { + $(this).remove(); + }); + } + + $(".selected-covariates option").each(function() { + $(this).remove(); }); - // Trim the last newline from display_string - covariates_display_string = covariates_display_string.replace(/\n$/, "") - // Trim the last comma - covariates_string = covariates_string.substring(0, covariates_string.length - 1) - //covariates_display_string = covariates_display_string.substring(0, covariates_display_string.length - 2) + covariate_list_for_form = [] + $.each(covariates_as_list, function (index, value) { + option_value = value.split(",")[0] + option_text = value.split(",")[1] + $(".selected-covariates").append($("<option/>", { + value: option_value, + text: option_text + })) + covariate_list_for_form.push(option_value) + }); - $("input[name=covariates]").val(covariates_string) - $(".selected-covariates").val(covariates_display_string) + $("input[name=covariates]").val(covariate_list_for_form.join(",")); + + cofactor_count = $(".selected-covariates:first option").length; + if (cofactor_count > 10){ + $(".selected-covariates").attr("size", 10); + } else { + $(".selected-covariates").attr("size", cofactor_count); + } return $.colorbox.close(); }; @@ -186,9 +209,8 @@ color_by_trait = function(trait_sample_data, textStatus, jqXHR) { process_traits = function(trait_data, textStatus, jqXHR) { var the_html, trait, _i, _len; console.log('in process_traits with trait_data:', trait_data); - the_html = "<button id='back_to_collections' class='btn btn-inverse btn-small'>"; - the_html += "<i class='icon-white icon-arrow-left'></i> Back </button>"; - the_html += " <button id='submit' class='btn btn-primary btn-small'> Submit </button>"; + the_html = "<button class='btn btn-success btn-small submit'> Submit </button>"; + the_html += "<button id='back_to_collections' class='btn btn-inverse btn-small' style='float: right;'>Back</button>"; the_html += "<table id='collection_table' style='padding-top: 10px;' class='table table-hover'>"; the_html += "<thead><tr><th></th><th>Record</th><th>Data Set</th><th>Description</th></tr></thead>"; the_html += "<tbody>"; @@ -221,6 +243,6 @@ back_to_collections = function() { }; $(".collection_line").on("click", collection_click); -$("#submit").on("click", submit_click); +$(".submit").on("click", submit_click); $(".trait").on("click", trait_click); -$("#back_to_collections").on("click", back_to_collections);
\ No newline at end of file +$("#back_to_collections").on("click", back_to_collections); diff --git a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js index 9fe61abe..897f79ff 100644 --- a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js +++ b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js @@ -132,16 +132,16 @@ build_columns = function() { } } - attr_keys = Object.keys(js_data.attributes).sort((a, b) => (js_data.attributes[a].name.toLowerCase() > js_data.attributes[b].name.toLowerCase()) ? 1 : -1) + attr_keys = Object.keys(js_data.attributes).sort((a, b) => (js_data.attributes[a].id > js_data.attributes[b].id) ? 1 : -1) for (i = 0; i < attr_keys.length; i++){ column_list.push( { - 'title': "<div style='text-align: " + js_data.attributes[attr_keys[i]].alignment + "'>" + js_data.attributes[attr_keys[i]].name + "</div>", + 'title': "<div title='" + js_data.attributes[attr_keys[i]].description + "' style='text-align: " + js_data.attributes[attr_keys[i]].alignment + "'>" + js_data.attributes[attr_keys[i]].name + "</div>", 'type': "natural", 'data': null, 'targets': attr_start + i, 'render': function(data, type, row, meta) { - attr_name = Object.keys(data.extra_attributes).sort()[meta.col - data.first_attr_col] + attr_name = Object.keys(data.extra_attributes).sort((a, b) => (parseInt(a) > parseInt(b)) ? 1 : -1)[meta.col - data.first_attr_col] if (attr_name != null && attr_name != undefined){ if (Array.isArray(data.extra_attributes[attr_name])){ diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index 77ef1720..f050d4ae 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -98,11 +98,54 @@ sample_group_types = js_data.sample_group_types; $(".select_covariates").click(function () { open_covariate_selection(); }); + $(".remove_covariates").click(function () { - $("input[name=covariates]").val("") - $(".selected-covariates").val("") + $(".selected-covariates option:selected").each(function() { + this_val = $(this).val(); + $(".selected-covariates option").each(function(){ + if ($(this).val() == this_val){ + $(this).remove(); + } + }) + cofactor_count = $(".selected-covariates:first option").length + if (cofactor_count > 2 && cofactor_count < 11){ + $(".selected-covariates").each(function() { + $(this).attr("size", $(".selected-covariates:first option").length) + }); + } else if (cofactor_count > 10) { + $(".selected-covariates").each(function() { + $(this).attr("size", 10) + }); + } else { + $(".selected-covariates").each(function() { + $(this).attr("size", 2) + }); + } + if (cofactor_count == 0){ + $(".selected-covariates").each(function() { + $(this).append($("<option/>", { + value: "", + text: "No covariates selected" + })) + }) + } + }); + + covariates_list = []; + $(".selected-covariates:first option").each(function() { + covariates_list.push($(this).val()); + }) + $("input[name=covariates]").val(covariates_list.join(",")) }); +$(".remove_all_covariates").click(function() { + $(".selected-covariates option").each(function() { + $(this).remove(); + }); + $(".selected-covariates").attr("size", 2) + $("input[name=covariates]").val(""); +}) + open_trait_selection = function() { return $('#collections_holder').load('/collections/list?color_by_trait #collections_list', (function(_this) { return function() { @@ -608,13 +651,14 @@ $(".corr_compute").on("click", (function(_this) { create_value_dropdown = function(value) { return "<option val=" + value + ">" + value + "</option>"; }; + populate_sample_attributes_values_dropdown = function() { var attribute_info, key, sample_attributes, selected_attribute, value, _i, _len, _ref, _ref1, _results; $('#attribute_values').empty(); sample_attributes = []; var attributes_as_list = Object.keys(js_data.attributes).map(function(key) { - return [key, js_data.attributes[key].name.toLowerCase()]; + return [key, js_data.attributes[key].id]; }); attributes_as_list.sort(function(first, second) { @@ -628,7 +672,7 @@ populate_sample_attributes_values_dropdown = function() { }); for (i=0; i < attributes_as_list.length; i++) { - attribute_info = js_data.attributes[attributes_as_list[i][0]] + attribute_info = js_data.attributes[attributes_as_list[i][1]] sample_attributes.push(attribute_info.distinct_values); } @@ -667,11 +711,13 @@ block_by_attribute_value = function() { let exclude_val_nodes = table_api.column(attribute_start_pos + parseInt(exclude_column)).nodes().to$(); for (i = 0; i < exclude_val_nodes.length; i++) { - let this_col_value = exclude_val_nodes[i].childNodes[0].data; - let this_val_node = val_nodes[i].childNodes[0]; + if (exclude_val_nodes[i].hasChildNodes()) { + let this_col_value = exclude_val_nodes[i].childNodes[0].data; + let this_val_node = val_nodes[i].childNodes[0]; - if (this_col_value == exclude_by_value){ - this_val_node.value = "x"; + if (this_col_value == exclude_by_value){ + this_val_node.value = "x"; + } } } @@ -713,10 +759,34 @@ block_by_index = function() { for (_k = 0, _len1 = index_list.length; _k < _len1; _k++) { index = index_list[_k]; val_nodes[index - 1].childNodes[0].value = "x"; - } }; +filter_by_study = function() { + let this_study = $('#filter_study').val(); + + let study_sample_data = JSON.parse($('input[name=study_samplelists]').val()) + let filter_samples = study_sample_data[parseInt(this_study)]['samples'] + + if ($('#filter_study_group').length){ + let block_group = $('#filter_study_group').val(); + if (block_group === "other") { + table_api = $('#samples_other').DataTable(); + } else { + table_api = $('#samples_primary').DataTable(); + } + } + + let sample_nodes = table_api.column(2).nodes().to$(); + let val_nodes = table_api.column(3).nodes().to$(); + for (i = 0; i < sample_nodes.length; i++) { + this_sample = sample_nodes[i].childNodes[0].innerText; + if (!filter_samples.includes(this_sample)){ + val_nodes[i].childNodes[0].value = "x"; + } + } +} + filter_by_value = function() { let filter_logic = $('#filter_logic').val(); let filter_column = $('#filter_column').val(); @@ -748,7 +818,7 @@ filter_by_value = function() { var this_col_value = filter_val_nodes[i].childNodes[0].value; } else { if (filter_val_nodes[i].childNodes[0] !== undefined){ - var this_col_value = filter_val_nodes[i].childNodes[0].data; + var this_col_value = filter_val_nodes[i].innerText; } else { continue } @@ -1690,6 +1760,11 @@ $('#block_by_index').click(function(){ edit_data_change(); }); +$('#filter_by_study').click(function(){ + filter_by_study(); + edit_data_change(); +}) + $('#filter_by_value').click(function(){ filter_by_value(); edit_data_change(); diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js index 09e9d024..e42fe8c4 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js +++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js @@ -141,11 +141,11 @@ $('input[name=display_all]').change((function(_this) { })(this)); //ZS: This is a list of inputs to be passed to the loading page, since not all inputs on the trait page are relevant to mapping -var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form_url', 'method', 'transform', 'trimmed_markers', 'selected_chr', 'chromosomes', 'mapping_scale', 'sample_vals', - 'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars', 'num_bootstrap', 'bootCheck', 'bootstrap_results', - 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker', 'do_control', 'genofile', - 'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', 'haplotypeAnalystCheck', - 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'reaper_version', 'primary_samples'] +var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form_url', 'method', 'transform', 'trimmed_markers', 'selected_chr', 'chromosomes', 'mapping_scale', + 'sample_vals', 'vals_hash', 'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars', + 'num_bootstrap', 'bootCheck', 'bootstrap_results', 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker', + 'do_control', 'genofile', 'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', + 'haplotypeAnalystCheck', 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'primary_samples'] $(".rqtl-geno-tab, #rqtl_geno_compute").on("click", (function(_this) { return function() { diff --git a/wqflask/wqflask/templates/admin/group_manager.html b/wqflask/wqflask/templates/admin/group_manager.html index c0b99e75..692a7abc 100644 --- a/wqflask/wqflask/templates/admin/group_manager.html +++ b/wqflask/wqflask/templates/admin/group_manager.html @@ -81,7 +81,7 @@ <tr> <td><input type="checkbox" name="read" value="{{ group.id }}"></td> <td>{{ loop.index }}</td> - <td>{{ group.name }}</td> + <td><a href="/groups/view?id={{ group.id }}">{{ group.name }}</a></td> <td>{{ group.admins|length + group.members|length }}</td> <td>{{ group.created_timestamp }}</td> <td>{{ group.changed_timestamp }}</td> diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index 1a0335b6..d30c575a 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -87,6 +87,7 @@ <li><a href="https://systems-genetics.org/">Systems Genetics PheWAS</a></li> <li><a href="http://ucscbrowser.genenetwork.org/">Genome Browser</a></li> <li><a href="http://power.genenetwork.org">BXD Power Calculator</a></li> + <li><a href="http://notebook.genenetwork.org/">Jupyter Notebook Launcher</a></li> <li><a href="http://datafiles.genenetwork.org">Interplanetary File System</a></li> </ul> </li> @@ -208,7 +209,7 @@ <a href="http://joss.theoj.org/papers/10.21105/joss.00025"><img src="https://camo.githubusercontent.com/846b750f582ae8f1d0b4f7e8fee78bed705c88ba/687474703a2f2f6a6f73732e7468656f6a2e6f72672f7061706572732f31302e32313130352f6a6f73732e30303032352f7374617475732e737667" alt="JOSS" data-canonical-src="http://joss.theoj.org/papers/10.21105/joss.00025/status.svg" style="max-width:100%;"></a> </p> <p> - Development and source code on <a href="https://github.com/genenetwork/">github</a> with <a href="https://github.com/genenetwork/genenetwork2/issues">issue tracker</a> and <a href="https://github.com/genenetwork/genenetwork2/blob/master/README.md">documentation</a>. Join the <a href="http://listserv.uthsc.edu/mailman/listinfo/genenetwork-dev">mailing list</a> and find us on <span class="broken_link" href="https://webchat.freenode.net/">IRC</span> (#genenetwork channel). + Development and source code on <a href="https://github.com/genenetwork/">github</a> with <a href="https://github.com/genenetwork/genenetwork2/issues">issue tracker</a> and <a href="https://github.com/genenetwork/genenetwork2/blob/master/README.md">documentation</a>. Join the <span class="broken_link" href="http://listserv.uthsc.edu/mailman/listinfo/genenetwork-dev">mailing list</span> and find us on <a href="https://webchat.freenode.net#genenetwork">IRC</a> (#genenetwork channel). {% if version: %} <p><small>GeneNetwork {{ version }}</small></p> {% endif %} diff --git a/wqflask/wqflask/templates/collections/add.html b/wqflask/wqflask/templates/collections/add.html index 0398c6e4..8640fdb8 100644 --- a/wqflask/wqflask/templates/collections/add.html +++ b/wqflask/wqflask/templates/collections/add.html @@ -5,7 +5,7 @@ or add to an existing collection.</p> </div> <div class="modal-body" style="margin-left: 20px;"> - <form action="/collections/new" target="_blank" data-validate="parsley" id="add_form"> + <form action="/collections/new" target="_blank" data-validate="parsley" id="add_form" class="form-inline"> {% if traits is defined %} <input type="hidden" name="traits" value="{{ traits }}" /> {% else %} @@ -14,10 +14,8 @@ {% if collections|length > 0 %} <fieldset> <legend>1. Add to an existing collection</legend> - <div style="margin-left: 20px;"> - <!--<label>Existing collection name:</label>--> - <select name="existing_collection" class="form-control"> - <!--<option selected disabled>Select Collection</option>--> + <div style="margin-left: 20px;"> + <select name="existing_collection" class="form-control" style="width: 80%;"> {% for col in collections %} {% if loop.index == 1 %} <option value="{{ col.id }}:{{ col.name }}" selected>{{ col.name }}</option> @@ -26,8 +24,9 @@ {% endif %} {% endfor %} </select> - <br /> - <button type="submit" name="add_to_existing" class="btn btn-primary">Add to existing collection</button> + <input type="button" style="display: inline;" id="make_default" value="Make Default"> + <br><br> + <button type="submit" name="add_to_existing" class="btn btn-primary">Add</button> </div> </fieldset> {% endif %} @@ -35,7 +34,6 @@ <fieldset> <legend>{% if collections|length > 0 %}2. {% else %}{% endif %}Create a new collection</legend> <div style="margin-left: 20px;"> - <!--<label>Collection name:</label>--> <input type="text" name="new_collection" placeholder=" Name of new collection..." data-trigger="change" data-minlength="5" data-maxlength="50" style="width: 100%"> <button type="submit" name="create_new" class="btn btn-primary" style="margin-top: 20px;">Create collection</button> @@ -54,6 +52,21 @@ parent.jQuery.colorbox.close(); }); + make_default = function() { + alert("The current collection is now your default collection.") + let uc_id = $('[name=existing_collection] option:selected').val().split(":")[0] + $.cookie('default_collection', uc_id, { + expires: 365, + path: '/' + }); + + let default_collection_id = $.cookie('default_collection'); + }; + + $("#make_default").on("click", function(){ + make_default(); + }); + apply_default = function() { let default_collection_id = $.cookie('default_collection'); if (default_collection_id) { diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index 9ec98ab1..a3090bcf 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -49,7 +49,7 @@ <input type="text" id="select_top" class="form-control" style="width: 200px; display: inline; padding-bottom: 9px;" placeholder="Select Top ..."> <button class="btn btn-default" id="deselect_all" type="button"><span class="glyphicon glyphicon-remove"></span> Deselect</button> <button id="remove" class="btn btn-danger" data-url="/collections/remove" type="button" disabled><i class="icon-minus-sign"></i> Delete Rows</button> - <button id="delete" class="btn btn-danger submit_special" data-url="/collections/delete" title="Delete this collection" > Delete Collection</button> + <button id="delete" class="btn btn-danger submit_special" data-url="/collections/delete" type="button" title="Delete this collection" > Delete Collection</button> </form> </div> <div style="margin-top: 10px; margin-bottom: 5px;"> diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html index 4cad2749..f66eb4bd 100644 --- a/wqflask/wqflask/templates/correlation_page.html +++ b/wqflask/wqflask/templates/correlation_page.html @@ -17,9 +17,9 @@ <hr style="height: 1px; background-color: #A9A9A9;"> </div> <div style="max-width: 100%;"> - <p>Values of record {{ this_trait.name }} in the <a href="http://genenetwork.org/webqtl/main.py?FormID=sharinginfo&{% if dataset.accession_id != 'None' %}GN_AccessionId={{ dataset.accession_id }}{% else %}InfoPageName={{ dataset.name }}{% endif %}">{{ dataset.fullname }}</a> + <p>Values of record {{ this_trait.name }} in the <a href="http://genenetwork.org/webqtl/main.py?FormID=sharinginfo&{% if this_dataset.accession_id != 'None' %}GN_AccessionId={{ this_dataset.accession_id }}{% else %}InfoPageName={{ this_dataset.name }}{% endif %}">{{ this_dataset.fullname }}</a> dataset were compared to all records in the <a href="http://genenetwork.org/webqtl/main.py?FormID=sharinginfo&{% if target_dataset.accession_id != 'None' %}GN_AccessionId={{ target_dataset.accession_id }}{% else %}InfoPageName={{ target_dataset.name }}{% endif %}">{{ target_dataset.fullname }}</a> - dataset. The top {{ return_number }} correlations ranked by the {{ formatted_corr_type }} are displayed. + dataset. The top {{ return_results }} correlations ranked by the {{ formatted_corr_type }} are displayed. You can resort this list by clicking the headers. Select the Record ID to open the trait data and analysis page. </p> @@ -30,7 +30,7 @@ <input type="hidden" name="form_url" value="" /> <input type="hidden" name="trait_list" id="trait_list" value= " {% for this_trait in trait_list %} - {{ this_trait.name }}:{{ this_trait.dataset }}, + {{ this_trait }}:{{ this_dataset.name }}, {% endfor %}" > {% include 'tool_buttons.html' %} </form> @@ -43,7 +43,7 @@ <button class="btn btn-success" id="add" type="button" disabled><span class="glyphicon glyphicon-plus-sign"></span> Add</button> <input type="hidden" name="database_name" id="database_name" value="None"> <input type="hidden" name="export_data" id="export_data" value=""> - <input type="hidden" name="file_name" id="file_name" value="{{ this_trait.name }}_{{ dataset.name }}_correlation"> + <input type="hidden" name="file_name" id="file_name" value="{{ this_trait.name }}_{{ this_dataset.name }}_correlation"> <input type="text" id="searchbox" class="form-control" style="width: 200px; display: inline;" placeholder="Search Table For ..."> <input type="text" id="select_top" class="form-control" style="width: 200px; display: inline;" placeholder="Select Top ..."> <button class="btn btn-default" id="deselect_all" type="button"><span class="glyphicon glyphicon-remove"></span> Deselect</button> @@ -146,7 +146,7 @@ <script type="text/javascript" charset="utf-8"> - var table_json = {{ json_results | safe }} + var table_json = {{ table_json | safe }} </script> <script type="text/javascript" charset="utf-8"> @@ -313,7 +313,7 @@ 'orderSequence': [ "desc", "asc"], 'render': function(data, type, row, meta) { if (data.sample_r != "N/A") { - return "<a target\"_blank\" href=\"corr_scatter_plot?method={% if corr_method == 'spearman' %}spearman{% else %}pearson{% endif %}&dataset_1={% if dataset.name == 'Temp' %}Temp_{{ dataset.group.name }}{% else %}{{ dataset.name }}{% endif %}&dataset_2=" + data.dataset + "&trait_1={{ this_trait.name }}&trait_2=" + data.trait_id + "\">" + data.sample_r + "</a>" + return "<a target\"_blank\" href=\"corr_scatter_plot?method={% if corr_method == 'spearman' %}spearman{% else %}pearson{% endif %}&dataset_1={% if this_dataset.name == 'Temp' %}Temp_{{ this_dataset.group }}{% else %}{{ this_dataset.name }}{% endif %}&dataset_2=" + data.dataset + "&trait_1={{ this_trait.name }}&trait_2=" + data.trait_id + "\">" + data.sample_r + "</a>" } else { return data.sample_r } @@ -441,7 +441,7 @@ 'orderSequence': [ "desc", "asc"], 'render': function(data, type, row, meta) { if (data.sample_r != "N/A") { - return "<a target\"_blank\" href=\"corr_scatter_plot?method={% if corr_method == 'spearman' %}spearman{% else %}pearson{% endif %}&dataset_1={% if dataset.name == 'Temp' %}Temp_{{ dataset.group.name }}{% else %}{{ dataset.name }}{% endif %}&dataset_2=" + data.dataset + "&trait_1={{ this_trait.name }}&trait_2=" + data.trait_id + "\">" + data.sample_r + "</a>" + return "<a target\"_blank\" href=\"corr_scatter_plot?method={% if corr_method == 'spearman' %}spearman{% else %}pearson{% endif %}&dataset_1={% if this_dataset.name== 'Temp' %}Temp_{{ this_dataset.group }}{% else %}{{ this_dataset.name }}{% endif %}&dataset_2=" + data.dataset + "&trait_1={{ this_trait.name }}&trait_2=" + data.trait_id + "\">" + data.sample_r + "</a>" } else { return data.sample_r } @@ -495,7 +495,7 @@ 'orderSequence': [ "desc", "asc"], 'render': function(data, type, row, meta) { if (data.sample_r != "N/A") { - return "<a target\"_blank\" href=\"corr_scatter_plot?method={% if corr_method == 'spearman' %}spearman{% else %}pearson{% endif %}&dataset_1={% if dataset.name == 'Temp' %}Temp_{{ dataset.group.name }}{% else %}{{ dataset.name }}{% endif %}&dataset_2=" + data.dataset + "&trait_1={{ this_trait.name }}&trait_2=" + data.trait_id + "\">" + data.sample_r + "</a>" + return "<a target\"_blank\" href=\"corr_scatter_plot?method={% if corr_method == 'spearman' %}spearman{% else %}pearson{% endif %}&dataset_1={% if this_dataset.name == 'Temp' %}Temp_{{ this_dataset.group }}{% else %}{{ this_dataset.name }}{% endif %}&dataset_2=" + data.dataset + "&trait_1={{ this_trait.name }}&trait_2=" + data.trait_id + "\">" + data.sample_r + "</a>" } else { return data.sample_r } diff --git a/wqflask/wqflask/templates/display_files_admin.html b/wqflask/wqflask/templates/display_files_admin.html new file mode 100644 index 00000000..4b4babc4 --- /dev/null +++ b/wqflask/wqflask/templates/display_files_admin.html @@ -0,0 +1,32 @@ +{% extends "base.html" %} +{% block title %}Trait Submission{% endblock %} +{% block content %} +<!-- Start of body --> +{% with messages = get_flashed_messages(with_categories=true) %} +{% if messages %} +{% for category, message in messages %} +<div class="container-fluid bg-{{ category }}"> + <p>{{ message }}</p> +</div> +{% endfor %} +{% endif %} +{% endwith %} +Show files for approval + +<div> + <ul> + {% for file in files %} + <li><a href="/display-file/{{ file }}" target="_blank">{{ file }}</a><br/> + <button><a href="/data-samples/approve/{{ file }}">Approve</a></button> + <button><a href="/data-samples/reject/{{ file }}">Reject</a></button></li> + {% endfor %} + </ul> +</div> +{%endblock%} + +{% block js %} +<script> + gn_server_url = "{{ gn_server_url }}"; + +</script> +{% endblock %} diff --git a/wqflask/wqflask/templates/display_files_user.html b/wqflask/wqflask/templates/display_files_user.html new file mode 100644 index 00000000..b6bab709 --- /dev/null +++ b/wqflask/wqflask/templates/display_files_user.html @@ -0,0 +1,31 @@ +{% extends "base.html" %} +{% block title %}Trait Submission{% endblock %} +{% block content %} +<!-- Start of body --> +{% with messages = get_flashed_messages(with_categories=true) %} +{% if messages %} +{% for category, message in messages %} +<div class="container-fluid bg-{{ category }}"> + <p>{{ message }}</p> +</div> +{% endfor %} +{% endif %} +{% endwith %} +Show files for approval + +<div> + <ul> + {% for file in files %} + <li><a href="/display-file/{{ file }}" target="_blank">{{ file }}</a><br/> + <button><a href="/data-samples/reject/{{ file }}">Reject</a></button></li> + {% endfor %} + </ul> +</div> +{%endblock%} + +{% block js %} +<script> + gn_server_url = "{{ gn_server_url }}"; + +</script> +{% endblock %} diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_phenotype.html index 7d4c65f8..7a841793 100644 --- a/wqflask/wqflask/templates/edit_trait.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -2,8 +2,18 @@ {% block title %}Trait Submission{% endblock %} {% block content %} <!-- Start of body --> -Edit Trait for Published Database -Submit Trait | Reset +{% with messages = get_flashed_messages(with_categories=true) %} +{% if messages %} +{% for category, message in messages %} +<div class="container-fluid bg-{{ category }}"> + <p>{{ message }}</p> +</div> +{% endfor %} +{% endif %} +{% endwith %} +<div class="page-header text-center"> + <h1>Edit Trait for Published Database</h1> +</div> {% if diff %} @@ -53,7 +63,7 @@ Submit Trait | Reset {% endif %} -<form id="edit-form" class="form-horizontal" method="post" action="/trait/update"> +<form id="edit-form" class="form-horizontal" method="post" action="/trait/update" enctype=multipart/form-data> <h2 class="text-center">Trait Information:</h2> <div class="form-group"> <label for="pubmed-id" class="col-sm-2 control-label">Pubmed ID:</label> @@ -207,10 +217,18 @@ Submit Trait | Reset <input name="old_sequence" class="changed" type="hidden" value="{{ publication.sequence |default('', true) }}"/> </div> </div> - <div class="controls" style="display:block; margin-left: 40%; margin-right: 20%;"> + <div style="margin-left: 13%;"> + <a href="/trait/{{ publish_xref.id_ }}/sampledata/{{ publish_xref.phenotype_id }}" class="btn btn-link btn-sm"> + Sample Data(CSV Download) + </a> + </div> + <div class="form-group"> + <input type = "file" class="col-sm-4 control-label" name = "file" /> + </div> + <div class="controls center-block" style="width: max-content;"> <input name="dataset-name" class="changed" type="hidden" value="{{ publish_xref.id_ }}"/> - <input name="phenotype-id" class="changed" type="hidden" value="{{ publish_xref.phenotype_id }}"/> <input name="inbred-set-id" class="changed" type="hidden" value="{{ publish_xref.inbred_set_id }}"/> + <input name="phenotype-id" class="changed" type="hidden" value="{{ publish_xref.phenotype_id }}"/> <input name="comments" class="changed" type="hidden" value="{{ publish_xref.comments }}"/> <input type="submit" style="width: 125px; margin-right: 25px;" class="btn btn-primary form-control col-xs-2 changed" value="Submit Change"> <input type="reset" style="width: 110px;" class="btn btn-primary form-control col-xs-2 changed" onClick="window.location.reload();" value="Reset"> diff --git a/wqflask/wqflask/templates/edit_probeset.html b/wqflask/wqflask/templates/edit_probeset.html new file mode 100644 index 00000000..85d49561 --- /dev/null +++ b/wqflask/wqflask/templates/edit_probeset.html @@ -0,0 +1,239 @@ +{% extends "base.html" %} +{% block title %}Trait Submission{% endblock %} +{% block content %} +<!-- Start of body --> +Edit Trait for Probeset +Submit Trait | Reset + +{% if diff %} + +<div class="container"> + <details class="col-sm-12 col-md-10 col-lg-12"> + <summary> + <h2>Update History</h2> + </summary> + <table class="table"> + <tbody> + <tr> + <th>Timestamp</th> + <th>Editor</th> + <th>Field</th> + <th>Diff</th> + </tr> + {% set ns = namespace(display_cell=True) %} + + {% for timestamp, group in diff %} + {% set ns.display_cell = True %} + {% for i in group %} + <tr> + {% if ns.display_cell and i.timestamp == timestamp %} + + {% set author = i.author %} + {% set timestamp_ = i.timestamp %} + + {% else %} + + {% set author = "" %} + {% set timestamp_ = "" %} + + {% endif %} + <td>{{ timestamp_ }}</td> + <td>{{ author }}</td> + <td>{{ i.diff.field }}</td> + <td><pre>{{ i.diff.diff }}</pre></td> + {% set ns.display_cell = False %} + </tr> + {% endfor %} + {% endfor %} + </tbody> + </table> + </details> + +</div> + +{% endif %} + +<form id="edit-form" class="form-horizontal" method="post" action="/probeset/update"> + <h2 class="text-center">Probeset Information:</h2> + <div class="form-group"> + <label for="symbol" class="col-sm-2 control-label">Symbol:</label> + <div class="col-sm-4"> + <textarea name="symbol" class="form-control" rows="1">{{ probeset.symbol |default('', true) }}</textarea> + <input name="old_symbol" class="changed" type="hidden" value="{{ probeset.symbol |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="description" class="col-sm-2 control-label">Description:</label> + <div class="col-sm-5"> + <textarea name="description" class="form-control" rows="3">{{ probeset.description |default('', true) }}</textarea> + <input name="old_description" class="changed" type="hidden" value="{{ probeset.description |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_target_description" class="col-sm-2 control-label">Probe Target Description:</label> + <div class="col-sm-4"> + <textarea name="probe_target_description" class="form-control" rows="4">{{ probeset.probe_target_description |default('', true) }}</textarea> + <input name="old_probe_target_description" class="changed" type="hidden" value="{{ probeset.probe_target_description |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="chr" class="col-sm-2 control-label">Chr:</label> + <div class="col-sm-4"> + <textarea name="chr" class="form-control" rows="1">{{ probeset.chr_ |default('', true) }}</textarea> + <input name="old_chr_" class="changed" type="hidden" value="{{ probeset.chr_ |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="mb" class="col-sm-2 control-label">Mb:</label> + <div class="col-sm-4"> + <textarea name="mb" class="form-control" rows="1">{{ probeset.mb |default('', true) }}</textarea> + <input name="old_mb" class="changed" type="hidden" value="{{ probeset.mb |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="alias" class="col-sm-2 control-label"> + Alias: + </label> + <div class="col-sm-4"> + <textarea name="alias" class="form-control" rows="1">{{ probeset.alias |default('', true) }}</textarea> + <input name="old_alias" class="changed" type="hidden" value="{{ probeset.alias |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="geneid" class="col-sm-2 control-label"> + Gene Id: + </label> + <div class="col-sm-4"> + <textarea name="geneid" class="form-control" rows="1">{{ probeset.geneid |default('', true) }}</textarea> + <input name="old_geneid" class="changed" type="hidden" value="{{ probeset.geneid |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="homologeneid" class="col-sm-2 control-label"> + Homolegene Id: + </label> + <div class="col-sm-4"> + <textarea name="homologeneid" class="form-control" rows="1">{{ probeset.homologeneid |default('', true) }}</textarea> + <input name="old_homologeneid" class="changed" type="hidden" value="{{ probeset.homologeneid |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="unigeneid" class="col-sm-2 control-label"> + Unigene Id: + </label> + <div class="col-sm-4"> + <textarea name="unigeneid" class="form-control" rows="1">{{ probeset.unigeneid |default('', true) }}</textarea> + <input name="old_unigeneid" class="changed" type="hidden" value="{{ probeset.unigeneid |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="omim" class="col-sm-2 control-label">OMIM:</label> + <div class="col-sm-4"> + <textarea name="omim" class="form-control" rows="1">{{ probeset.omim |default('', true) }}</textarea> + <input name="old_omim" class="changed" type="hidden" value="{{ probeset.omim |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="refseq_transcriptid" class="col-sm-2 control-label"> + Refseq TranscriptId: + </label> + <div class="col-sm-4"> + <textarea name="refseq_transcriptid" class="form-control" rows="1">{{ probeset.refseq_transcriptid |default('', true) }}</textarea> + <input name="old_refseq_transcriptid" class="changed" type="hidden" value="{{ probeset.refseq_transcriptid |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="blatseq" class="col-sm-2 control-label">BlatSeq:</label> + <div class="col-sm-8"> + <textarea name="blatseq" class="form-control" rows="6">{{ probeset.blatseq |default('', true) }}</textarea> + <input name="old_blatseq" class="changed" type="hidden" value="{{ probeset.blatseq |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="targetseq" class="col-sm-2 control-label">TargetSeq:</label> + <div class="col-sm-8"> + <textarea name="targetseq" class="form-control" rows="6">{{ probeset.targetseq |default('', true) }}</textarea> + <input name="old_targetseq" class="changed" type="hidden" value="{{ probeset.targetseq |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="strand_probe" class="col-sm-2 control-label">Strand Probe:</label> + <div class="col-sm-2"> + <textarea name="strand_probe" class="form-control" rows="1">{{ probeset.strand_probe |default('', true) }}</textarea> + <input name="old_strand_probe" class="changed" type="hidden" value="{{ probeset.strand_probe |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_set_target_region" class="col-sm-2 control-label">Probe Set Target Region:</label> + <div class="col-sm-8"> + <textarea name="probe_set_target_region" class="form-control" rows="1">{{ probeset.probe_set_target_region |default('', true) }}</textarea> + <input name="old_probe_set_target_region" class="changed" type="hidden" value="{{ probeset.probe_set_target_region_ |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_set_specificity" class="col-sm-2 control-label">Probeset Specificity:</label> + <div class="col-sm-8"> + <textarea name="probe_set_specificity" class="form-control" rows="1">{{ probeset.probe_set_specificity |default('', true) }}</textarea> + <input name="old_probe_set_specificity" class="changed" type="hidden" value="{{ probeset.probe_set_specificity |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_set_blat_score" class="col-sm-2 control-label">Probeset Blat Score:</label> + <div class="col-sm-8"> + <textarea name="probe_set_blat_score" class="form-control" rows="1">{{ probeset.probe_set_blat_score |default('', true) }}</textarea> + <input name="old_probe_set_blat_score" class="changed" type="hidden" value="{{ probeset.probe_set_blat_score |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_set_blat_mb_start" class="col-sm-2 control-label"> + Probeset Blat Mb Start:</label> + <div class="col-sm-8"> + <textarea name="probe_set_blat_mb_start" class="form-control" rows="1">{{ probeset.probe_set_blat_mb_start |default('', true) }}</textarea> + <input name="old_probe_set_blat_mb_start" class="changed" type="hidden" value="{{ probeset.probe_set_blat_mb_start |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_set_blat_mb_end" class="col-sm-2 control-label">Probeset Blat Mb End:</label> + <div class="col-sm-8"> + <textarea name="probe_set_blat_mb_end" class="form-control" rows="6">{{ probeset.probe_set_blat_mb_end |default('', true) }}</textarea> + <input name="old_probe_set_blat_mb_end" class="changed" type="hidden" value="{{ probeset.probe_set_blat_mb_end |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_set_strand" class="col-sm-2 control-label">Probeset Strand:</label> + <div class="col-sm-8"> + <textarea name="probe_set_strand" class="form-control" rows="6">{{ probeset.probe_set_strand |default('', true) }}</textarea> + <input name="old_probe_set_strand" class="changed" type="hidden" value="{{ probeset.probe_set_strand |default('', true) }}"/> + </div> + </div> + <div class="form-group"> + <label for="probe_set_note_by_rw" class="col-sm-2 control-label">Probeset Strand:</label> + <div class="col-sm-8"> + <textarea name="probe_set_note_by_rw" class="form-control" rows="6">{{ probeset.probe_set_note_by_rw |default('', true) }}</textarea> + <input name="old_probe_set_note_by_rw" class="changed" type="hidden" value="{{ probeset.probe_set_note_by_rw |default('', true) }}"/> + </div> + </div> + <div class="controls" style="display:block; margin-left: 40%; margin-right: 20%;"> + <input name="id" class="changed" type="hidden" value="{{ probeset.id_ }}"/> + <input name="old_id_" class="changed" type="hidden" value="{{ probeset.id_ }}"/> + <input name="probeset_name" class="changed" type="hidden" value="{{ probeset.name }}"/> + <input type="submit" style="width: 125px; margin-right: 25px;" class="btn btn-primary form-control col-xs-2 changed" value="Submit Change"> + <input type="reset" style="width: 110px;" class="btn btn-primary form-control col-xs-2 changed" onClick="window.location.reload();" value="Reset"> + </div> +</form> + +{%endblock%} + +{% block js %} +<script> + gn_server_url = "{{ gn_server_url }}"; + function MarkAsChanged(){ + $(this).addClass("changed"); + } + $(":input").blur(MarkAsChanged).change(MarkAsChanged); + + $("input[type=submit]").click(function(){ + $(":input:not(.changed)").attr("disabled", "disabled"); + }); +</script> +{% endblock %} diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 6d6136ac..ccf810b0 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -12,6 +12,8 @@ {% if start_vars.tool_used == "Mapping" %} <h1>Computing the Maps</h1> <br> + <b>Time Elapsed:</b> <span class="timer"></span> + <br> <b>Trait Metadata</b> <br> species = <b><i>{{ start_vars.species[0] | upper }}{{ start_vars.species[1:] }}</i></b> @@ -25,6 +27,8 @@ <br> transformation = <b><i>{{ start_vars.transform }}</i></b> {% endif %} + <br> + hash of sample values = <b><i>{{ start_vars.vals_hash }}</i></b> <br><br> <b>Mapping Metadata</b> <br> @@ -68,6 +72,29 @@ <div style="text-align: center;"> <img align="center" src="/static/gif/89.gif"> </div> + {% if start_vars.vals_diff|length != 0 and start_vars.transform == "" %} + <br><br> + <button id="show_full_diff">Show Full Diff</button> + <br> + <div id="diff_table_container" style="display: none; height:200px; overflow:auto;"> + <table class="table table-hover"> + <thead> + <th>Sample</th> + <th>New Value</th> + <th>Old Value</th> + </thead> + <tbody> + {% for sample in start_vars.vals_diff %} + <tr> + <td>{{ sample }}</td> + <td>{{ start_vars.vals_diff[sample].new_val }}</td> + <td>{{ start_vars.vals_diff[sample].old_val }}</td> + </tr> + {% endfor %} + </tbody> + </table> + </div> + {% endif %} </div> </div> </div> @@ -76,6 +103,30 @@ <script src="{{ url_for('js', filename='jquery/jquery.min.js') }}" type="text/javascript"></script> <script src="{{ url_for('js', filename='bootstrap/js/bootstrap.min.js') }}" type="text/javascript"></script> <script type="text/javascript"> +$('#show_full_diff').click(function() { + if ($('#diff_table_container').is(':visible')){ + $('#diff_table_container').hide(); + } else { + $('#diff_table_container').show(); + } +}) + +var start = new Date; + +setInterval(function() { + minutes = Math.floor((new Date - start) / 1000 / 60) + seconds = Math.round(((new Date - start) / 1000) % 60) + if (seconds < 10 && minutes >= 1){ + seconds_text = "0" + seconds.toString() + } else { + seconds_text = seconds.toString() + } + if (minutes < 1) { + $('.timer').text(seconds_text + " seconds"); + } else { + $('.timer').text(minutes.toString() + ":" + seconds_text); + } +}, 100); $("#loading_form").attr("action", "{{ start_vars.form_url }}"); setTimeout(function(){ $("#loading_form").submit()}, 350); diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index d6fc6e37..f2d11e89 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -34,6 +34,7 @@ <input type="hidden" name="results_path" value="{{ mapping_results_path }}"> <input type="hidden" name="method" value="{{ mapping_method }}"> <input type="hidden" name="sample_vals" value="{{ sample_vals }}"> + <input type="hidden" name="vals_hash" value="{{ vals_hash }}"> <input type="hidden" name="n_samples" value="{{ n_samples }}"> <input type="hidden" name="maf" value="{{ maf }}"> <input type="hidden" name="use_loco" value="{{ use_loco }}"> @@ -44,7 +45,12 @@ {% endif %} <input type="hidden" name="num_perm" value="{{ nperm }}"> <input type="hidden" name="perm_info" value=""> - <input type="hidden" name="perm_strata" value="{{ perm_strata }}"> + {% if categorical_vars is defined %} + <input type="hidden" name="categorical_vars" value="{{ categorical_vars|join(',') }}"> + {% endif %} + {% if perm_strata is defined %} + <input type="hidden" name="perm_strata" value="True"> + {% endif %} <input type="hidden" name="num_bootstrap" value="{{ nboot }}"> <input type="hidden" name="do_control" value="{{ doControl }}"> <input type="hidden" name="control_marker" value="{{ controlLocus }}"> @@ -62,15 +68,16 @@ <h2>Map Viewer: Whole Genome</h2><br> <b>Population:</b> {{ dataset.group.species|capitalize }} {{ dataset.group.name }}<br> <b>Database:</b> {{ dataset.fullname }}<br> - {% if dataset.type == "ProbeSet" %}<b>Trait ID:</b>{% else %}<b>Record ID:</b>{% endif %} <a href="/show_trait?trait_id={{ this_trait.name }}&dataset={{ dataset.name }}">{{ this_trait.name }}</a><br> + {% if dataset.type == "ProbeSet" %}<b>Trait ID:</b>{% else %}<b>Record ID:</b>{% endif %} <a href="/show_trait?trait_id={{ this_trait.name }}&dataset={{ dataset.name }}">{{ this_trait.display_name }}</a><br> + <b>Trait Hash: </b> {{ vals_hash }}<br> {% if dataset.type == "ProbeSet" %} <b>Gene Symbol:</b> <i>{{ this_trait.symbol }}</i><br> <b>Location:</b> Chr {{ this_trait.chr }} @ {{ this_trait.mb }} Mb<br> {% endif %} - {% if genofile_string is defined %} - <b>Genotypes:</b> {{ genofile_string.split(":")[1] }} + {% if genofile_string != "" %} + <b>Genotypes:</b> {{ genofile_string.split(":")[1] }}<br> {% endif %} - <br> + <b>Current Date/Time:</b> {{ current_datetime }}<br> <br> <a class="export_mapping_results" href="#" target="_blank" >Download Full Results</a> </div> @@ -357,7 +364,9 @@ {% endif %} <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.dataTables.min.js') }}"></script> - <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/buttons/js/dataTables.buttons.min.js') }}"></script> + <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/buttons/js/dataTables.buttons.min.js') }}"></script> + <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/scroller/js/dataTables.scroller.min.js') }}"></script> + <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/plugins/sorting/scientific.js') }}"></script> <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/plugins/sorting/natural.js') }}"></script> <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='purescript-genome-browser/js/purescript-genetics-browser.js') }}"></script> @@ -409,13 +418,12 @@ "info": "Showing from _START_ to _END_ of " + js_data.total_markers + " records", }, "order": [[1, "asc" ]], - "sDom": "iRZtir", - "iDisplayLength": -1, - "autoWidth": false, - "deferRender": true, + "sDom": "itir", + "autoWidth": true, "bSortClasses": false, - "scrollCollapse": false, - "paging": false + "scrollY": "100vh", + "scroller": true, + "scrollCollapse": true } ); {% elif selectedChr != -1 and plotScale =="physic" and (dataset.group.species == 'mouse' or dataset.group.species == 'rat') %} $('#trait_table').dataTable( { @@ -523,7 +531,7 @@ }); {% endif %} - {% if mapping_method != "gemma" and mapping_method != "plink" %} + {% if mapping_method != "gemma" and mapping_method != "plink" and nperm > 0 and permChecked == "ON" %} $('#download_perm').click(function(){ perm_info_dict = { perm_data: js_data.perm_results, diff --git a/wqflask/wqflask/templates/new_security/_scripts.html b/wqflask/wqflask/templates/new_security/_scripts.html deleted file mode 100644 index 5fefe305..00000000 --- a/wqflask/wqflask/templates/new_security/_scripts.html +++ /dev/null @@ -1 +0,0 @@ -<!--<script type="text/javascript" src="/static/new/javascript/login.js"></script>--> diff --git a/wqflask/wqflask/templates/new_security/forgot_password.html b/wqflask/wqflask/templates/new_security/forgot_password.html index e5c42a45..60a221da 100644 --- a/wqflask/wqflask/templates/new_security/forgot_password.html +++ b/wqflask/wqflask/templates/new_security/forgot_password.html @@ -48,6 +48,5 @@ {% endblock %} {% block js %} - {% include "new_security/_scripts.html" %} {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/forgot_password_step2.html b/wqflask/wqflask/templates/new_security/forgot_password_step2.html index b4bf41c7..1835fd4c 100644 --- a/wqflask/wqflask/templates/new_security/forgot_password_step2.html +++ b/wqflask/wqflask/templates/new_security/forgot_password_step2.html @@ -20,7 +20,6 @@ {% endblock %} {% block js %} - {% include "new_security/_scripts.html" %} <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='zxcvbn/zxcvbn.js') }}"></script> <script type="text/javascript" src="/static/new/javascript/password_strength.js"></script> {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index 095036f0..88eab6bc 100644 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -114,31 +114,5 @@ label.error,div.error{ {% endblock %} {% block js %} - <!-- Disable plugin, see https://github.com/genenetwork/genenetwork2/issues/310 - - <script type="text/javascript" src="/static/new/packages/ValidationPlugin/dist/jquery.validate.min.js"></script> - <script> - $(document).ready(function () { - $("#loginUserForm").validate({ - onkeyup: false, - onsubmit: true, - onfocusout: function(element) { $(element).valid(); }, - rules: { - email_address: { - required: true, - email: true - }, - password: { - required: true - } - } - }); - }); - - </script> - - --> - - {% include "new_security/_scripts.html" %} {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/password_reset.html b/wqflask/wqflask/templates/new_security/password_reset.html index 684c12b1..e21f075c 100644 --- a/wqflask/wqflask/templates/new_security/password_reset.html +++ b/wqflask/wqflask/templates/new_security/password_reset.html @@ -73,7 +73,6 @@ {% block js %} - {% include "new_security/_scripts.html" %} <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='zxcvbn/zxcvbn.js') }}"></script> <script type="text/javascript" src="/static/new/javascript/password_strength.js"></script> {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/register_user.html b/wqflask/wqflask/templates/new_security/register_user.html index 3ae4488b..c2895517 100644 --- a/wqflask/wqflask/templates/new_security/register_user.html +++ b/wqflask/wqflask/templates/new_security/register_user.html @@ -100,7 +100,6 @@ {% block js %} - {% include "new_security/_scripts.html" %} <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='zxcvbn/zxcvbn.js') }}"></script> <script type="text/javascript" src="/static/new/javascript/password_strength.js"></script> {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/registered.html b/wqflask/wqflask/templates/new_security/registered.html index f2f58ec1..29889a97 100644 --- a/wqflask/wqflask/templates/new_security/registered.html +++ b/wqflask/wqflask/templates/new_security/registered.html @@ -19,7 +19,6 @@ {% block js %} - {% include "new_security/_scripts.html" %} <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='zxcvbn/zxcvbn.js') }}"></script> <script type="text/javascript" src="/static/new/javascript/password_strength.js"></script> {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/thank_you.html b/wqflask/wqflask/templates/new_security/thank_you.html index 0ff7ee8d..d4f5e574 100644 --- a/wqflask/wqflask/templates/new_security/thank_you.html +++ b/wqflask/wqflask/templates/new_security/thank_you.html @@ -18,7 +18,6 @@ {% endblock %} {% block js %} - {% include "new_security/_scripts.html" %} <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='zxcvbn/zxcvbn.js') }}"></script> <script type="text/javascript" src="/static/new/javascript/password_strength.js"></script> {% endblock %} diff --git a/wqflask/wqflask/templates/new_security/verification_still_needed.html b/wqflask/wqflask/templates/new_security/verification_still_needed.html index dc0f9e68..1f91fd8d 100644 --- a/wqflask/wqflask/templates/new_security/verification_still_needed.html +++ b/wqflask/wqflask/templates/new_security/verification_still_needed.html @@ -21,7 +21,6 @@ {% endblock %} {% block js %} - {% include "new_security/_scripts.html" %} <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='zxcvbn/zxcvbn.js') }}"></script> <script type="text/javascript" src="/static/new/javascript/password_strength.js"></script> {% endblock %} diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index e5eea4d1..72a4b560 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -54,6 +54,7 @@ A total of {{ results|count }} records were found. </p> + {% if results|count > 0 %} {% if go_term is not none %} <p><b>The associated genes include:</b><br><br>{% for word in search_terms %}{{ word.search_term[0] }}{% endfor %}</p> {% endif %} @@ -134,8 +135,11 @@ </div> </div> {% endif %} + {% else %} + <br> + <button type="button" onclick="window.location.href='/'">Return To Index Page</button> + {% endif %} </div> - <div id="myModal"></div> <!-- End of body --> @@ -172,6 +176,7 @@ return params; }; + {% if results|count > 0 %} var tableId = "trait_table"; var width_change = 0; //ZS: For storing the change in width so overall table width can be adjusted by that amount @@ -501,6 +506,7 @@ var table = $('#' + tableId).DataTable(); table.colReorder.reset() }); + {% endif %} submit_special = function(url) { $("#trait_submission_form").attr("action", url); diff --git a/wqflask/wqflask/templates/show_trait_calculate_correlations.html b/wqflask/wqflask/templates/show_trait_calculate_correlations.html index e623a968..16a819fa 100644 --- a/wqflask/wqflask/templates/show_trait_calculate_correlations.html +++ b/wqflask/wqflask/templates/show_trait_calculate_correlations.html @@ -7,8 +7,10 @@ <div class="col-xs-3 controls"> <select name="corr_type" class="form-control"> <option value="sample">Sample r</option> + {% if dataset.type == 'ProbeSet' %} <option value="lit">Literature r</option> <option value="tissue">Tissue r</option> + {% endif %} </select> </div> </div> @@ -70,7 +72,7 @@ <select name="corr_sample_method" class="form-control"> <option value="pearson">Pearson</option> <option value="spearman">Spearman Rank</option> - <!-- <option value="bicor">Biweight Midcorrelation</option> --> + <option value="bicor">Biweight Midcorrelation</option> </select> </div> </div> diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 83f7b0ac..2a21dd24 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -235,7 +235,16 @@ {% endif %} <button type="button" id="view_in_gn1" class="btn btn-primary" title="View Trait in GN1" onclick="window.open('http://gn1.genenetwork.org/webqtl/main.py?cmd=show&db={{ this_trait.dataset.name }}&probeset={{ this_trait.name }}', '_blank')">Go to GN1</button> {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} - <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('/trait/{{ this_trait.name }}/edit/{{ this_trait.dataset.id }}', '_blank')">Edit</button> + {% if this_trait.dataset.type == 'Publish' %} + <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('/trait/{{ this_trait.name }}/edit/inbredset-id/{{ this_trait.dataset.id }}', '_blank')">Edit</button> + {% endif %} + + {% if this_trait.dataset.type == 'ProbeSet' %} + <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('/trait/edit/probeset-name/{{ this_trait.name }}', '_blank')">Edit</button> + {% endif %} + {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} + <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('./resources/manage?resource_id={{ resource_id }}', '_blank')">Edit Privileges</button> + {% endif %} {% endif %} </div> </div> diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index 3dd44c85..3af94ed6 100755 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -74,17 +74,20 @@ No collections available. Please add traits to a collection to use them as covariates. {% else %} <div class="select-covar-div"> - <button type="button" class="btn btn-default select-covar-button select_covariates">Select</button> + <button type="button" class="btn btn-success select-covar-button select_covariates">Select</button> <button type="button" class="btn btn-default select-covar-button remove_covariates">Remove</button> + <button type="button" class="btn btn-danger select-covar-button remove_all_covariates">Clear</button> </div> - <textarea rows="3" cols="50" readonly placeholder="No covariates selected" class="selected-covariates"></textarea> + <select size="2" name="selected_covariates_gemma" class="form-control selected-covariates" multiple> + <option value="">No covariates selected</option> + </select> {% endif %} </div> </div> <div class="mapping_method_fields form-group"> <label class="col-xs-3 control-label"></label> <div class="col-xs-6"> - <button id="gemma_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression" value="Compute">Compute</button> + <button id="gemma_compute" type="button" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression" value="Compute">Compute</button> </div> </div> </div> @@ -93,15 +96,6 @@ <div class="tab-pane" id="interval_mapping"> <div class="form-horizontal section-form-div"> <div class="mapping_method_fields form-group"> - <label for="reaper_version" class="col-xs-3 control-label">Version<sup><a href="https://github.com/chfi/rust-qtlreaper" target="_blank" title="'New' is the new qtlreaper implementation written in RUST by Christian Fischer. 'Original' corresponds to the original version written in C.">?</a></sup></label> - <div class="col-xs-3 controls"> - <select name="reaper_version" class="form-control reaper-ver-select"> - <option value="new">New</option> - <option value="original">Original</option> - </select> - </div> - </div> - <div class="mapping_method_fields form-group"> <label for="chr_select" class="col-xs-3 control-label">Chromosome</label> <div class="col-xs-2 controls"> <select id="chr_reaper" class="form-control chr-select"> @@ -187,7 +181,7 @@ <div class="mapping_method_fields form-group"> <label class="col-xs-3 control-label"></label> <div class="col-xs-6"> - <button id="interval_mapping_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Interval Mapping" value="Compute">Compute</button> + <button id="interval_mapping_compute" type="button" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Interval Mapping" value="Compute">Compute</button> </div> </div> </div> @@ -226,6 +220,17 @@ </select> </div> </div> + {% else %} + <div class="mapping_method_fields form-group"> + <label for="scale_select" class="col-xs-3 control-label">Map Scale</label> + <div class="col-xs-2 controls"> + <select id="scale_rqtl_geno" class="form-control scale-select"> + {% for item in scales_in_geno[dataset.group.name + ".geno"] %} + <option value="{{ item[0] }}">{{ item[1] }}</option> + {% endfor %} + </select> + </div> + </div> {% endif %} <div class="mapping_method_fields form-group"> <label for="mapping_permutations" class="col-xs-3 control-label">Permutations</label> @@ -249,21 +254,6 @@ </div> {% endif %} <div class="mapping_method_fields form-group"> - <label for="control_for" class="col-xs-3 control-label">Control for</label> - <div class="col-xs-6 controls"> - <input name="control_rqtl_geno" value="{% if dataset.type == 'ProbeSet' and this_trait.locus_chr != '' %}{{ nearest_marker }}{% endif %}" type="text" class="form-control cofactor-input" /> - <label class="radio-inline"> - <input type="radio" name="do_control_rqtl" value="true"> - Yes - </label> - <label class="radio-inline"> - <input type="radio" name="do_control_rqtl" value="false" checked=""> - No - </label> - </div> - </div> - - <div class="mapping_method_fields form-group"> <label for="mapmodel_rqtl_geno" class="col-xs-3 control-label">Model</label> <div class="col-xs-4 controls"> <select id="mapmodel_rqtl_geno" name="mapmodel_rqtl_geno" class="form-control"> @@ -317,17 +307,20 @@ No collections available. Please add traits to a collection to use them as covariates. {% else %} <div class="select-covar-div"> - <button type="button" class="btn btn-default select-covar-button select_covariates">Select</button> + <button type="button" class="btn btn-success select-covar-button select_covariates">Select</button> <button type="button" class="btn btn-default select-covar-button remove_covariates">Remove</button> + <button type="button" class="btn btn-danger select-covar-button remove_all_covariates">Clear</button> </div> - <textarea rows="3" cols="50" readonly placeholder="No covariates selected" class="selected-covariates"></textarea> + <select size="2" name="selected_covariates_rqtl" class="form-control selected-covariates" multiple> + <option value="">No covariates selected</option> + </select> {% endif %} </div> </div> <div class="mapping_method_fields form-group"> <label class="col-xs-3 control-label"></label> <div class="col-xs-6 controls"> - <button id="rqtl_geno_compute" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression" value="Compute">Compute</button> + <button id="rqtl_geno_compute" type="button" class="btn submit_special btn-success" data-url="/marker_regression" title="Compute Marker Regression" value="Compute">Compute</button> </div> </div> </div> diff --git a/wqflask/wqflask/templates/show_trait_transform_and_filter.html b/wqflask/wqflask/templates/show_trait_transform_and_filter.html index 20f78b48..5e6ed2cf 100644 --- a/wqflask/wqflask/templates/show_trait_transform_and_filter.html +++ b/wqflask/wqflask/templates/show_trait_transform_and_filter.html @@ -25,7 +25,7 @@ <label for="exclude_column">Block samples by group:</label> <select id="exclude_column" size=1> {% for attribute in sample_groups[0].attributes %} - {% if sample_groups[0].attributes[attribute].distinct_values|length <= 10 %} + {% if sample_groups[0].attributes[attribute].distinct_values|length <= 10 and sample_groups[0].attributes[attribute].distinct_values|length > 1 %} <option value="{{ loop.index }}"> {{ sample_groups[0].attributes[attribute].name }} </option> @@ -45,6 +45,27 @@ <input type="button" id="exclude_by_attr" class="btn btn-danger" value="Block"> </div> {% endif %} + {% if study_samplelists|length > 0 %} + <div id="filterMenuSpan" class="input-append block-div-2"> + <label for="filter_study_select">Filter samples by study: </label> + <select id="filter_study"> + {% for study in study_samplelists %} + <option value="{{ loop.index - 1 }}">{{ study }}</option> + {% endfor %} + </select> + {% if sample_groups|length != 1 %} + <select id="filter_study_group" size="1"> + <option value="primary"> + {{ sample_group_types['samples_primary'] }} + </option> + <option value="other"> + {{ sample_group_types['samples_other'] }} + </option> + </select> + {% endif %} + <input type="button" id="filter_by_study" class="btn btn-danger" value="Filter"> + </div> + {% endif %} <div id="filterMenuSpan" class="input-append block-div-2"> <label for="filter_samples_field">Filter samples by {% if (numerical_var_list|length == 0) and (not js_data.se_exists) %}value{% endif %} </label> {% if (numerical_var_list|length > 0) or js_data.se_exists %} @@ -53,10 +74,12 @@ {% if js_data.se_exists %} <option value="stderr">SE</option> {% endif %} - {% for attribute in numerical_var_list %} + {% for attribute in sample_groups[0].attributes %} + {% if sample_groups[0].attributes[attribute].name in numerical_var_list %} <option value="{{ loop.index }}"> - {{ attribute }} + {{ sample_groups[0].attributes[attribute].name }} </option> + {% endif %} {% endfor %} </select> {% endif %} diff --git a/wqflask/wqflask/templates/test_correlation_page.html b/wqflask/wqflask/templates/test_correlation_page.html index 0809b65e..991773a2 100644 --- a/wqflask/wqflask/templates/test_correlation_page.html +++ b/wqflask/wqflask/templates/test_correlation_page.html @@ -113,7 +113,7 @@ console.log(correlationResults) {"data":corr_type=="sample"?null:"fd","width":"25px"}, { "data": "index","width":"120px","title":"Index" }, { "data": "trait_name","title":"TraitName"}, - { "data": "corr_coeffient","defaultContent": "--"}, + { "data": "corr_coefficient","defaultContent": "--"}, { "data": "p_value","defaultContent":"--"}, { "data": "num_overlap","defaultContent":"--"}, {"data":"tissue_corr","defaultContent":"--","title":"Tissue r"}, diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index b9181368..5067ca0e 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -27,6 +27,8 @@ from zipfile import ZIP_DEFLATED from wqflask import app +from gn3.commands import run_cmd +from gn3.computations.gemma import generate_hash_of_string from gn3.db import diff_from_dict from gn3.db import fetchall from gn3.db import fetchone @@ -34,12 +36,17 @@ from gn3.db import insert from gn3.db import update from gn3.db.metadata_audit import MetadataAudit from gn3.db.phenotypes import Phenotype +from gn3.db.phenotypes import Probeset from gn3.db.phenotypes import Publication from gn3.db.phenotypes import PublishXRef +from gn3.db.phenotypes import probeset_mapping +from gn3.db.traits import get_trait_csv_sample_data +from gn3.db.traits import update_sample_data from flask import current_app from flask import g +from flask import flash from flask import Response from flask import request from flask import make_response @@ -57,6 +64,7 @@ from wqflask import server_side from base.data_set import create_dataset # Used by YAML in marker_regression from wqflask.show_trait import show_trait from wqflask.show_trait import export_trait_data +from wqflask.show_trait.show_trait import get_diff_of_vals from wqflask.heatmap import heatmap from wqflask.external_tools import send_to_bnw from wqflask.external_tools import send_to_webgestalt @@ -65,7 +73,7 @@ from wqflask.comparison_bar_chart import comparison_bar_chart from wqflask.marker_regression import run_mapping from wqflask.marker_regression import display_mapping_results from wqflask.network_graph import network_graph -from wqflask.correlation import show_corr_results +from wqflask.correlation.show_corr_results import set_template_vars from wqflask.correlation.correlation_gn3_api import compute_correlation from wqflask.correlation_matrix import show_corr_matrix from wqflask.correlation import corr_scatter_plot @@ -77,7 +85,7 @@ from wqflask.export_traits import export_search_results_csv from wqflask.gsearch import GSearch from wqflask.update_search_results import GSearch as UpdateGSearch from wqflask.docs import Docs, update_text -from wqflask.decorators import admin_login_required +from wqflask.decorators import edit_access_required from wqflask.db_info import InfoPage from utility import temp_data @@ -152,28 +160,37 @@ def shutdown_session(exception=None): @app.errorhandler(Exception) -def handle_bad_request(e): +def handle_generic_exceptions(e): + import werkzeug err_msg = str(e) - logger.error(err_msg) - logger.error(request.url) - # get the stack trace and send it to the logger - exc_type, exc_value, exc_traceback = sys.exc_info() - logger.error(traceback.format_exc()) now = datetime.datetime.utcnow() time_str = now.strftime('%l:%M%p UTC %b %d, %Y') - formatted_lines = [request.url - + " (" + time_str + ")"] + traceback.format_exc().splitlines() - + # get the stack trace and send it to the logger + exc_type, exc_value, exc_traceback = sys.exc_info() + formatted_lines = {f"{request.url} ({time_str}) " + f" {traceback.format_exc().splitlines()}"} + + _message_templates = { + werkzeug.exceptions.NotFound: ("404: Not Found: " + f"{time_str}: {request.url}"), + werkzeug.exceptions.BadRequest: ("400: Bad Request: " + f"{time_str}: {request.url}"), + werkzeug.exceptions.RequestTimeout: ("408: Request Timeout: " + f"{time_str}: {request.url}")} + # Default to the lengthy stack trace! + logger.error(_message_templates.get(exc_type, + formatted_lines)) # Handle random animations # Use a cookie to have one animation on refresh animation = request.cookies.get(err_msg[:32]) if not animation: - list = [fn for fn in os.listdir( - "./wqflask/static/gif/error") if fn.endswith(".gif")] - animation = random.choice(list) + animation = random.choice([fn for fn in os.listdir( + "./wqflask/static/gif/error") if fn.endswith(".gif")]) resp = make_response(render_template("error.html", message=err_msg, - stack=formatted_lines, error_image=animation, version=GN_VERSION)) + stack=formatted_lines, + error_image=animation, + version=GN_VERSION)) # logger.error("Set cookie %s with %s" % (err_msg, animation)) resp.set_cookie(err_msg[:32], animation) @@ -300,6 +317,7 @@ def gsearchact(): elif type == "phenotype": return render_template("gsearch_pheno.html", **result) + @app.route("/gsearch_table", methods=('GET',)) def gsearchtable(): logger.info(request.url) @@ -314,6 +332,7 @@ def gsearchtable(): return flask.jsonify(current_page) + @app.route("/gsearch_updating", methods=('POST',)) def gsearch_updating(): logger.info("REQUEST ARGS:", request.values) @@ -357,20 +376,6 @@ def wcgna_setup(): return render_template("wgcna_setup.html", **request.form) -# @app.route("/wgcna_results", methods=('POST',)) -# def wcgna_results(): -# logger.info("In wgcna, request.form is:", request.form) -# logger.info(request.url) -# # Start R, load the package and pointers and create the analysis -# wgcna = wgcna_analysis.WGCNA() -# # Start the analysis, a wgcnaA object should be a separate long running thread -# wgcnaA = wgcna.run_analysis(request.form) -# # After the analysis is finished store the result -# result = wgcna.process_results(wgcnaA) -# # Display them using the template -# return render_template("wgcna_results.html", **result) - - @app.route("/ctl_setup", methods=('POST',)) def ctl_setup(): # We are going to get additional user input for the analysis @@ -380,20 +385,6 @@ def ctl_setup(): return render_template("ctl_setup.html", **request.form) -# @app.route("/ctl_results", methods=('POST',)) -# def ctl_results(): -# logger.info("In ctl, request.form is:", request.form) -# logger.info(request.url) -# # Start R, load the package and pointers and create the analysis -# ctl = ctl_analysis.CTL() -# # Start the analysis, a ctlA object should be a separate long running thread -# ctlA = ctl.run_analysis(request.form) -# # After the analysis is finished store the result -# result = ctl.process_results(ctlA) -# # Display them using the template -# return render_template("ctl_results.html", **result) - - @app.route("/intro") def intro(): doc = Docs("intro", request.args) @@ -428,9 +419,9 @@ def submit_trait_form(): version=GN_VERSION) -@app.route("/trait/<name>/edit/<inbred_set_id>") -@admin_login_required -def edit_trait(name, inbred_set_id): +@app.route("/trait/<name>/edit/inbredset-id/<inbredset_id>") +@edit_access_required +def edit_phenotype(name, inbredset_id): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), passwd=current_app.config.get("DB_PASS"), @@ -439,7 +430,7 @@ def edit_trait(name, inbred_set_id): conn=conn, table="PublishXRef", where=PublishXRef(id_=name, - inbred_set_id=inbred_set_id)) + inbred_set_id=inbredset_id)) phenotype_ = fetchone( conn=conn, table="Phenotype", @@ -476,7 +467,7 @@ def edit_trait(name, inbred_set_id): if len(diff_data) > 0: diff_data_ = groupby(diff_data, lambda x: x.timestamp) return render_template( - "edit_trait.html", + "edit_phenotype.html", diff=diff_data_, publish_xref=publish_xref, phenotype=phenotype_, @@ -485,13 +476,119 @@ def edit_trait(name, inbred_set_id): ) +@app.route("/trait/edit/probeset-name/<dataset_name>") +@edit_access_required +def edit_probeset(dataset_name): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + probeset_ = fetchone(conn=conn, + table="ProbeSet", + columns=list(probeset_mapping.values()), + where=Probeset(name=dataset_name)) + json_data = fetchall( + conn, + "metadata_audit", + where=MetadataAudit(dataset_id=probeset_.id_)) + Edit = namedtuple("Edit", ["field", "old", "new", "diff"]) + Diff = namedtuple("Diff", ["author", "diff", "timestamp"]) + diff_data = [] + for data in json_data: + json_ = json.loads(data.json_data) + timestamp = json_.get("timestamp") + author = json_.get("author") + for key, value in json_.items(): + if isinstance(value, dict): + for field, data_ in value.items(): + diff_data.append( + Diff(author=author, + diff=Edit(field, + data_.get("old"), + data_.get("new"), + "\n".join(difflib.ndiff( + [data_.get("old")], + [data_.get("new")]))), + timestamp=timestamp)) + diff_data_ = None + if len(diff_data) > 0: + diff_data_ = groupby(diff_data, lambda x: x.timestamp) + return render_template( + "edit_probeset.html", + diff=diff_data_, + probeset=probeset_) + + @app.route("/trait/update", methods=["POST"]) -def update_trait(): +@edit_access_required +def update_phenotype(): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), passwd=current_app.config.get("DB_PASS"), host=current_app.config.get("DB_HOST")) data_ = request.form.to_dict() + TMPDIR = current_app.config.get("TMPDIR") + author = g.user_session.record.get(b'user_name') + if 'file' not in request.files: + flash("No sample-data has been uploaded", "warning") + else: + file_ = request.files['file'] + trait_name = str(data_.get('dataset-name')) + phenotype_id = str(data_.get('phenotype-id', 35)) + SAMPLE_DATADIR = os.path.join(TMPDIR, "sample-data") + if not os.path.exists(SAMPLE_DATADIR): + os.makedirs(SAMPLE_DATADIR) + if not os.path.exists(os.path.join(SAMPLE_DATADIR, + "diffs")): + os.makedirs(os.path.join(SAMPLE_DATADIR, + "diffs")) + if not os.path.exists(os.path.join(SAMPLE_DATADIR, + "updated")): + os.makedirs(os.path.join(SAMPLE_DATADIR, + "updated")) + current_time = str(datetime.datetime.now().isoformat()) + new_file_name = (os.path.join(TMPDIR, + "sample-data/updated/", + (f"{author.decode('utf-8')}." + f"{trait_name}.{phenotype_id}." + f"{current_time}.csv"))) + uploaded_file_name = (os.path.join( + TMPDIR, + "sample-data/updated/", + (f"updated.{author.decode('utf-8')}." + f"{trait_name}.{phenotype_id}." + f"{current_time}.csv"))) + file_.save(new_file_name) + publishdata_id = "" + lines = [] + with open(new_file_name, "r") as f: + lines = f.read() + first_line = lines.split('\n', 1)[0] + publishdata_id = first_line.split("Id:")[-1].strip() + with open(new_file_name, "w") as f: + f.write(lines.split("\n\n")[-1]) + csv_ = get_trait_csv_sample_data(conn=conn, + trait_name=str(trait_name), + phenotype_id=str(phenotype_id)) + with open(uploaded_file_name, "w") as f_: + f_.write(csv_.split("\n\n")[-1]) + r = run_cmd(cmd=("csvdiff " + f"'{uploaded_file_name}' '{new_file_name}' " + "--format json")) + diff_output = (f"{TMPDIR}/sample-data/diffs/" + f"{trait_name}.{author.decode('utf-8')}." + f"{phenotype_id}.{current_time}.json") + with open(diff_output, "w") as f: + dict_ = json.loads(r.get("output")) + dict_.update({ + "author": author.decode('utf-8'), + "publishdata_id": publishdata_id, + "dataset_id": data_.get("dataset-name"), + "timestamp": datetime.datetime.now().strftime( + "%Y-%m-%d %H:%M:%S") + }) + f.write(json.dumps(dict_)) + flash("Sample-data has been successfully uploaded", "success") # Run updates: phenotype_ = { "pre_pub_description": data_.get("pre-pub-desc"), @@ -533,7 +630,6 @@ def update_trait(): diff_data.update({"Publication": diff_from_dict(old={ k: data_.get(f"old_{k}") for k, v in publication_.items() if v is not None}, new=publication_)}) - author = g.user_session.record.get(b'user_name') if diff_data: diff_data.update({"dataset_id": data_.get("dataset-name")}) diff_data.update({"author": author.decode('utf-8')}) @@ -544,7 +640,66 @@ def update_trait(): data=MetadataAudit(dataset_id=data_.get("dataset-name"), editor=author.decode("utf-8"), json_data=json.dumps(diff_data))) - return redirect("/trait/10007/edit/1") + flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success") + return redirect(f"/trait/{data_.get('dataset-name')}" + f"/edit/inbredset-id/{data_.get('inbred-set-id')}") + + +@app.route("/probeset/update", methods=["POST"]) +@edit_access_required +def update_probeset(): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + data_ = request.form.to_dict() + probeset_ = { + "id_": data_.get("id"), + "symbol": data_.get("symbol"), + "description": data_.get("description"), + "probe_target_description": data_.get("probe_target_description"), + "chr_": data_.get("chr"), + "mb": data_.get("mb"), + "alias": data_.get("alias"), + "geneid": data_.get("geneid"), + "homologeneid": data_.get("homologeneid"), + "unigeneid": data_.get("unigeneid"), + "omim": data_.get("OMIM"), + "refseq_transcriptid": data_.get("refseq_transcriptid"), + "blatseq": data_.get("blatseq"), + "targetseq": data_.get("targetseq"), + "strand_probe": data_.get("Strand_Probe"), + "probe_set_target_region": data_.get("probe_set_target_region"), + "probe_set_specificity": data_.get("probe_set_specificity"), + "probe_set_blat_score": data_.get("probe_set_blat_score"), + "probe_set_blat_mb_start": data_.get("probe_set_blat_mb_start"), + "probe_set_blat_mb_end": data_.get("probe_set_blat_mb_end"), + "probe_set_strand": data_.get("probe_set_strand"), + "probe_set_note_by_rw": data_.get("probe_set_note_by_rw"), + "flag": data_.get("flag") + } + updated_probeset = update( + conn, "ProbeSet", + data=Probeset(**probeset_), + where=Probeset(id_=data_.get("id"))) + + diff_data = {} + author = g.user_session.record.get(b'user_name') + if updated_probeset: + diff_data.update({"Probeset": diff_from_dict(old={ + k: data_.get(f"old_{k}") for k, v in probeset_.items() + if v is not None}, new=probeset_)}) + if diff_data: + diff_data.update({"probeset_name": data_.get("probeset_name")}) + diff_data.update({"author": author.decode('utf-8')}) + diff_data.update({"timestamp": datetime.datetime.now().strftime( + "%Y-%m-%d %H:%M:%S")}) + insert(conn, + table="metadata_audit", + data=MetadataAudit(dataset_id=data_.get("id"), + editor=author.decode("utf-8"), + json_data=json.dumps(diff_data))) + return redirect(f"/trait/edit/probeset-name/{data_.get('probeset_name')}") @app.route("/create_temp_trait", methods=('POST',)) @@ -852,16 +1007,16 @@ def loading_page(): if key in wanted: start_vars[key] = value + sample_vals_dict = json.loads(start_vars['sample_vals']) if 'n_samples' in start_vars: n_samples = int(start_vars['n_samples']) else: - sample_vals_dict = json.loads(start_vars['sample_vals']) if 'group' in start_vars: dataset = create_dataset( start_vars['dataset'], group_name=start_vars['group']) else: dataset = create_dataset(start_vars['dataset']) - samples = start_vars['primary_samples'].split(",") + samples = dataset.group.samplelist if 'genofile' in start_vars: if start_vars['genofile'] != "": genofile_string = start_vars['genofile'] @@ -877,6 +1032,10 @@ def loading_page(): n_samples += 1 start_vars['n_samples'] = n_samples + start_vars['vals_hash'] = generate_hash_of_string(str(sample_vals_dict)) + if start_vars['dataset'] != "Temp": # Currently can't get diff for temp traits + start_vars['vals_diff'] = get_diff_of_vals(sample_vals_dict, str(start_vars['trait_id'] + ":" + str(start_vars['dataset']))) + start_vars['wanted_inputs'] = initial_start_vars['wanted_inputs'] start_vars_container['start_vars'] = start_vars @@ -901,6 +1060,7 @@ def mapping_results_page(): 'samples', 'vals', 'sample_vals', + 'vals_hash', 'first_run', 'output_files', 'geno_db_exists', @@ -917,7 +1077,6 @@ def mapping_results_page(): 'num_perm', 'permCheck', 'perm_strata', - 'strat_var', 'categorical_vars', 'perm_output', 'num_bootstrap', @@ -947,7 +1106,6 @@ def mapping_results_page(): 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', - 'reaper_version', 'n_samples', 'transform' ) @@ -1006,9 +1164,8 @@ def mapping_results_page(): gn1_template_vars = display_mapping_results.DisplayMappingResults( result).__dict__ - with Bench("Rendering template"): - rendered_template = render_template( - "mapping_results.html", **gn1_template_vars) + rendered_template = render_template( + "mapping_results.html", **gn1_template_vars) return rendered_template @@ -1021,7 +1178,7 @@ def export_mapping_results(): results_csv = open(file_path, "r").read() response = Response(results_csv, mimetype='text/csv', - headers={"Content-Disposition": "attachment;filename=mapping_results.csv"}) + headers={"Content-Disposition": "attachment;filename=" + os.path.basename(file_path)}) return response @@ -1082,22 +1239,17 @@ def network_graph_page(): @app.route("/corr_compute", methods=('POST',)) def corr_compute_page(): - logger.info("In corr_compute, request.form is:", pf(request.form)) - logger.info(request.url) - template_vars = show_corr_results.CorrelationResults(request.form) - return render_template("correlation_page.html", **template_vars.__dict__) - - # to test/disable the new correlation api uncomment these lines - - # correlation_results = compute_correlation(request.form) - # return render_template("test_correlation_page.html", correlation_results=correlation_results) + correlation_results = compute_correlation(request.form, compute_all=True) + correlation_results = set_template_vars(request.form, correlation_results) + return render_template("correlation_page.html", **correlation_results) @app.route("/test_corr_compute", methods=["POST"]) def test_corr_compute_page(): - correlation_data = compute_correlation(request.form) + correlation_data = compute_correlation(request.form, compute_all=True) return render_template("test_correlation_page.html", **correlation_data) - + + @app.route("/corr_matrix", methods=('POST',)) def corr_matrix_page(): logger.info("In corr_matrix, request.form is:", pf(request.form)) @@ -1195,8 +1347,6 @@ def browser_inputs(): return flask.jsonify(file_contents) -########################################################################## - def json_default_handler(obj): """Based on http://stackoverflow.com/a/2680060/1175849""" @@ -1212,3 +1362,112 @@ def json_default_handler(obj): else: raise TypeError('Object of type %s with value of %s is not JSON serializable' % ( type(obj), repr(obj))) + + +@app.route("/trait/<trait_name>/sampledata/<phenotype_id>") +def get_sample_data_as_csv(trait_name: int, phenotype_id: int): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + csv_ = get_trait_csv_sample_data(conn, str(trait_name), + str(phenotype_id)) + return Response( + csv_, + mimetype="text/csv", + headers={"Content-disposition": + "attachment; filename=myplot.csv"} + ) + + +@app.route("/admin/data-sample/diffs/") +@edit_access_required +def display_diffs_admin(): + TMPDIR = current_app.config.get("TMPDIR") + DIFF_DIR = f"{TMPDIR}/sample-data/diffs" + files = [] + if os.path.exists(DIFF_DIR): + files = os.listdir(DIFF_DIR) + files = filter(lambda x: not(x.endswith((".approved", ".rejected"))), + files) + return render_template("display_files_admin.html", + files=files) + + +@app.route("/user/data-sample/diffs/") +def display_diffs_users(): + TMPDIR = current_app.config.get("TMPDIR") + DIFF_DIR = f"{TMPDIR}/sample-data/diffs" + files = [] + author = g.user_session.record.get(b'user_name').decode("utf-8") + if os.path.exists(DIFF_DIR): + files = os.listdir(DIFF_DIR) + files = filter(lambda x: not(x.endswith((".approved", ".rejected"))) \ + and author in x, + files) + return render_template("display_files_user.html", + files=files) + + +@app.route("/data-samples/approve/<name>") +def approve_data(name): + sample_data = {} + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + TMPDIR = current_app.config.get("TMPDIR") + with open(os.path.join(f"{TMPDIR}/sample-data/diffs", + name), 'r') as myfile: + sample_data = json.load(myfile) + PUBLISH_ID = sample_data.get("publishdata_id") + modifications = [d for d in sample_data.get("Modifications")] + row_counts = len(modifications) + for modification in modifications: + if modification.get("Current"): + (strain_id, + strain_name, + value, se, count) = modification.get("Current").split(",") + update_sample_data( + conn=conn, + strain_name=strain_name, + strain_id=int(strain_id), + publish_data_id=int(PUBLISH_ID), + value=value, + error=se, + count=count + ) + insert(conn, + table="metadata_audit", + data=MetadataAudit( + dataset_id=name.split(".")[0], # use the dataset name + editor=sample_data.get("author"), + json_data=json.dumps(sample_data))) + if modifications: + # Once data is approved, rename it! + os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", name), + os.path.join(f"{TMPDIR}/sample-data/diffs", + f"{name}.approved")) + flash((f"Just updated data from: {name}; {row_counts} " + "row(s) modified!"), + "success") + return redirect("/admin/data-sample/diffs/") + + +@app.route("/data-samples/reject/<name>") +def reject_data(name): + TMPDIR = current_app.config.get("TMPDIR") + os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", name), + os.path.join(f"{TMPDIR}/sample-data/diffs", + f"{name}.rejected")) + flash(f"{name} has been rejected!", "success") + return redirect("/admin/data-sample/diffs/") + + +@app.route("/display-file/<name>") +def display_file(name): + TMPDIR = current_app.config.get("TMPDIR") + with open(os.path.join(f"{TMPDIR}/sample-data/diffs", + name), 'r') as myfile: + content = myfile.read() + return Response(content, mimetype='text/json') |