diff options
| author | zsloan | 2022-01-14 18:22:32 +0000 |
|---|---|---|
| committer | zsloan | 2022-01-14 18:22:32 +0000 |
| commit | 68ac19153b128f60b660e11365e5fd4304c95300 (patch) | |
| tree | 198e03522af43a2d41f3c02cf3785bcfd4635fc4 | |
| parent | f588ad96ae5045499860fa6e2740e101ad4410d7 (diff) | |
| parent | 9ab0c3b6cc146e1711f1478242d4198eed720e4c (diff) | |
| download | genenetwork2-68ac19153b128f60b660e11365e5fd4304c95300.tar.gz | |
Merge branch 'testing' of github.com:genenetwork/genenetwork2 into feature/add_rqtl_pairscan
97 files changed, 4142 insertions, 3500 deletions
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8e2c7966..0cf4557f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,6 +44,8 @@ jobs: /gn2-profile/bin/screen -dm bash -c "env GN2_PROFILE=/gn2-profile \ TMPDIR=/tmp SERVER_PORT=5004 \ WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \ + GN_PROXY_URL='http://localhost:8080' \ + GN3_LOCAL_URL='http://localhost:8081' \ GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \ etc/default_settings.py" @@ -52,6 +54,8 @@ jobs: env GN2_PROFILE=/gn2-profile \ TMPDIR=/tmp SERVER_PORT=5004 \ WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \ + GN_PROXY_URL='http://localhost:8080' \ + GN3_LOCAL_URL='http://localhost:8081' \ GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \ etc/default_settings.py -c -m unittest discover -v @@ -8,34 +8,42 @@ This repository contains the current source code for GeneNetwork (GN) (https://www.genenetwork.org/ (version 2). GN2 is a Web 2.0-style framework that includes data and computational tools for online genetics and genomic analysis of many different populations and many types of molecular, cellular, and physiological data. -The system is used by scientists and clinians in the field of precision health care and systems genetics. +The system is used by scientists and clinicians in the field of precision health care and systems genetics. GN and its predecessors have been in operation since Jan 1994, making it one of the longest-lived web services in biomedical research (https://en.wikipedia.org/wiki/GeneNetwork, and see a partial list of publications using GN and its predecessor, WebQTL (https://genenetwork.org/references/). -## Install - -The recommended installation is with GNU Guix which allows you to -deploy GN2 and dependencies as a self contained unit on any machine. -The database can be run separately as well as the source tree (for -developers). See the [installation docs](doc/README.org). - ## Run -Once having installed GN2 it can be run through a browser -interface +We recommend you use GNU Guix. GNU Guix allows you to deploy +GeneNetwork2 and dependencies as a self contained unit on any machine. +The database can be run separately as well as the source tree (for +developers). +Make sure you have the +[guix-bioinformatics](https://git.genenetwork.org/guix-bioinformatics/guix-bioinformatics) +channel set up. Then, to drop into a development environment with all +dependencies, run ```sh -genenetwork2 +guix shell -Df guix.scm +``` +Or, to drop into a development environment in a container, run +``` +guix shell -C --network -Df guix.scm ``` -A quick example is - +In the development environment, start GeneNetwork2 by running, for +example, ```sh -env GN2_PROFILE=~/opt/gn-latest SERVER_PORT=5300 GENENETWORK_FILES=~/data/gn2_data/ ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev +env SERVER_PORT=5300 \ + GENENETWORK_FILES=~/data/gn2_data/ \ + GN_PROXY_URL="http://localhost:8080"\ + GN3_LOCAL_URL="http://localhost:8081"\ + ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev ``` For full examples (you may need to set a number of environment variables), including running scripts and a Python REPL, also see the -startup script [./bin/genenetwork2](https://github.com/genenetwork/genenetwork2/blob/testing/bin/genenetwork2). +startup script +[./bin/genenetwork2](https://github.com/genenetwork/genenetwork2/blob/testing/bin/genenetwork2). Also mariadb and redis need to be running, see [INSTALL](./doc/README.org). @@ -59,14 +67,16 @@ We are building 'Mechanical Rob' automated testing using Python which can be run with: ```sh -env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py -a http://localhost:5003 +env ./bin/genenetwork2 \ + GN_PROXY_URL="http://localhost:8080" \ + GN3_LOCAL_URL="http://localhost:8081 "\ + ./etc/default_settings.py -c \ + ../test/requests/test-website.py -a http://localhost:5003 ``` -The GN2_PROFILE is the Guix profile that contains all -dependencies. The ./bin/genenetwork2 script sets up the environment -and executes test-website.py in a Python interpreter. The -a switch -says to run all tests and the URL points to the running GN2 http -server. +The ./bin/genenetwork2 script sets up the environment and executes +test-website.py in a Python interpreter. The -a switch says to run all +tests and the URL points to the running GN2 http server. #### Unit tests @@ -87,13 +97,25 @@ runcmd coverage html The `runcmd` and `runpython` are shell aliases defined in the following way: ```sh -alias runpython="env GN2_PROFILE=~/opt/gn-latest TMPDIR=/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ ./bin/genenetwork2 +alias runpython="env TMPDIR=/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 -alias runcmd="time env GN2_PROFILE=~/opt/gn-latest TMPDIR=//tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ ./bin/genenetwork2 ./etc/default_settings.py -cli" +alias runcmd="time env TMPDIR=//tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 ./etc/default_settings.py -cli" ``` Replace some of the env variables as per your use case. +### Troubleshooting + +If the menu does not pop up check your `GN2_BASE_URL`. E.g. + +``` +curl http://gn2-pjotr.genenetwork.org/api/v_pre1/gen_dropdown +``` + +check the logs. If there is ERROR 1054 (42S22): Unknown column +'InbredSet.Family' in 'field list' it may be you are trying the small +database. + ## Documentation User documentation can be found diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 2b94b2a2..ce3678e4 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,59 +1,49 @@ -#! /bin/bash +#! /bin/sh -e # # This is the startup script for GN2. It sets the environment variables to pick # up a Guix profile and allows for overriding parameters. # # Typical usage # -# env GN2_PROFILE=~/opt/genenetwork2-phewas ./bin/genenetwork2 ~/my_settings.py -# -# Where GN2_PROFILE points to the GNU Guix profile used for deployment. +# ./bin/genenetwork2 ~/my_settings.py # # This will run the GN2 server (with default settings if none -# supplied). Typically you need a GNU Guix profile which is set with -# an environment variable (this profile is dictated by the -# installation path of genenetwork). Say your profile is in -# ~/opt/gn-latest-guix -# -# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 -# -# You can pass in your own settings file, e.g. -# -# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ~/my_settings.py +# supplied). # # To run a maintenance python script with settings (instead of the # webserver) run from the base-dir with settings file and add that # script with a -c switch, e.g. # -# env GN2_PROFILE=/usr/local/guix-profiles/gn-latest-20190905 TMPDIR=/export/local/home/zas1024/gn2-zach/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5002 GENENETWORK_FILES=/export/local/home/zas1024/gn2-zach/genotype_files SQL_URI=mysql://webqtlout:webqtlout@lily.uthsc.edu/db_webqtl ./bin/genenetwork2 ./etc/default_settings.py -c ./maintenance/gen_select_dataset.py +# env TMPDIR=/export/local/home/zas1024/gn2-zach/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5002 GENENETWORK_FILES=/export/local/home/zas1024/gn2-zach/genotype_files SQL_URI=mysql://webqtlout:webqtlout@lily.uthsc.edu/db_webqtl ./bin/genenetwork2 ./etc/default_settings.py -c ./maintenance/gen_select_dataset.py # # To run any script in the environment # -# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli echo "HELLO WORLD" +# ./bin/genenetwork2 ./etc/default_settings.py -cli echo "HELLO WORLD" # # To get a python REPL(!) # -# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli python +# ./bin/genenetwork2 ./etc/default_settings.py -cli python # # For development you may want to run # -# env GN2_PROFILE=~/opt/gn-latest WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2 +# env WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2 # # For staging and production we use gunicorn. Run with something like # (note you have to provide the server port). Provide a settings file! # -# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-prod +# env SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-prod # # For development use # -# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev +# env SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev # # For extra flexibility you can also provide gunicorn parameters yourself with something like # -# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi" +# ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi" SCRIPT=$(realpath "$0") echo SCRIPT=$SCRIPT +export GN2_PROFILE=$GUIX_ENVIRONMENT echo GN2_PROFILE=$GN2_PROFILE GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT")) GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR) @@ -101,61 +91,15 @@ fi export GN2_SETTINGS=$settings # Python echo GN2_SETTINGS=$settings -# This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead -# if [ -z $ELASTICSEARCH_PROFILE ]; then -# echo -e "WARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE"; -# else -# PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python3.8/site-packages" -# fi - -if [ -z $GN2_PROFILE ] ; then - echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" - export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) - if [ -d $GN2_PROFILE ]; then - echo "Best guess is $GN2_PROFILE" - fi - echo "ERROR: always set GN2_PROFILE" - exit 1 -fi -if [ -z $GN2_PROFILE ]; then - read -p "PRESS [ENTER] TO CONTINUE..." -else - export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH="$GN2_PROFILE/lib/python3.8/site-packages" # never inject another PYTHONPATH!! - export R_LIBS_SITE=$GN2_PROFILE/site-library - export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript - export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" - export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" - export XDG_DATA_DIRS="$GN2_PROFILE/share" - export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" - export LC_ALL=C # FIXME - export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" - export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" - export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" - if [ -z $GEMMA_WRAPPER_COMMAND ]; then - export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" - fi - while IFS=":" read -ra PPATH; do - for PPART in "${PPATH[@]}"; do - if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi - done - done <<< "$PYTHONPATH" - if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi -fi -if [ -z $PYTHONPATH ] ; then - echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!" - exit 1 -fi -if [ ! -d $R_LIBS_SITE ] ; then - echo "ERROR R_LIBS_SITE has not been set correctly (we only allow one path) - use GN2_PROFILE!" - echo "Paste into your shell the output of (for example)" - echo "guix package -p \$GN2_PROFILE --search-paths" - exit 1 +export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript +export LC_ALL=C # FIXME +export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" +export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" +export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" +if [ -z $GEMMA_WRAPPER_COMMAND ]; then + export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" fi -# We may change this one: -export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$PYTHONPATH - # Our UNIX TMPDIR defaults to /tmp - change this on a shared server if [ -z $TMPDIR ]; then TMPDIR="/tmp" @@ -170,7 +114,6 @@ set|grep TMPDIR if [ "$1" = '-c' ] ; then cd $GN2_BASE_DIR/wqflask cmd=${2#wqflask/} - echo PYTHONPATH=$PYTHONPATH shift ; shift echo RUNNING COMMAND $cmd $* python $cmd $* @@ -181,7 +124,6 @@ fi if [ "$1" = "-cli" ] ; then cd $GN2_BASE_DIR/wqflask cmd=$2 - echo PYTHONPATH=$PYTHONPATH shift ; shift echo RUNNING COMMAND $cmd $* $cmd $* @@ -190,14 +132,12 @@ fi if [ "$1" = '-gunicorn' ] ; then cd $GN2_BASE_DIR/wqflask cmd=$2 - echo PYTHONPATH=$PYTHONPATH echo RUNNING gunicorn $cmd gunicorn $cmd exit $? fi if [ "$1" = '-gunicorn-dev' ] ; then cd $GN2_BASE_DIR/wqflask - echo PYTHONPATH=$PYTHONPATH if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --timeout 180 --reload wsgi" echo RUNNING gunicorn $cmd @@ -206,7 +146,6 @@ if [ "$1" = '-gunicorn-dev' ] ; then fi if [ "$1" = '-gunicorn-prod' ] ; then cd $GN2_BASE_DIR/wqflask - echo PYTHONPATH=$PYTHONPATH if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi PID=$TMPDIR/gunicorn.$USER.pid cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID --workers 20 --keep-alive 6000 --max-requests 100 --max-requests-jitter 30 --timeout 1200 wsgi" @@ -220,9 +159,6 @@ echo -n "dir $TMPDIR dbfilename gn2.rdb " | redis-server - & -# Overrides for packages that are not yet public (currently r-auwerx) -# export R_LIBS_SITE=$R_LIBS_SITE:$HOME/.Rlibs/das1i1pm54dj6lbdcsw5w0sdwhccyj1a-r-3.3.2/lib/R/lib - # Start the flask server running GN2 cd $GN2_BASE_DIR/wqflask echo "Starting with $settings" diff --git a/doc/API_readme.md b/doc/API_readme.md index be6668dc..17d10e44 100644 --- a/doc/API_readme.md +++ b/doc/API_readme.md @@ -1,169 +1,3 @@ # API Query Documentation # ---- -# Fetching Dataset/Trait info/data # ---- -## Fetch Species List ## -To get a list of species with data available in GN (and their associated names and ids): -``` -curl http://genenetwork.org/api/v_pre1/species -[ { "FullName": "Mus musculus", "Id": 1, "Name": "mouse", "TaxonomyId": 10090 }, ... { "FullName": "Populus trichocarpa", "Id": 10, "Name": "poplar", "TaxonomyId": 3689 } ] -``` - -Or to get a single species info: -``` -curl http://genenetwork.org/api/v_pre1/species/mouse -``` -OR -``` -curl http://genenetwork.org/api/v_pre1/species/mouse.json -``` - -*For all queries where the last field is a user-specified name/ID, there will be the option to append a file format type. Currently there is only JSON (and it will default to JSON if none is provided), but other formats will be added later* - -## Fetch Groups/RISets ## - -This query can optionally filter by species: - -``` -curl http://genenetwork.org/api/v_pre1/groups (for all species) -``` -OR -``` -curl http://genenetwork.org/api/v_pre1/groups/mouse (for just mouse groups/RISets) -[ { "DisplayName": "BXD", "FullName": "BXD RI Family", "GeneticType": "riset", "Id": 1, "MappingMethodId": "1", "Name": "BXD", "SpeciesId": 1, "public": 2 }, ... { "DisplayName": "AIL LGSM F34 and F39-43 (GBS)", "FullName": "AIL LGSM F34 and F39-43 (GBS)", "GeneticType": "intercross", "Id": 72, "MappingMethodId": "2", "Name": "AIL-LGSM-F34-F39-43-GBS", "SpeciesId": 1, "public": 2 } ] -``` - -## Fetch Genotypes for Group/RISet ## -``` -curl http://genenetwork.org/api/v_pre1/genotypes/bimbam/BXD -curl http://genenetwork.org/api/v_pre1/genotypes/BXD.bimbam -``` -Returns a group's genotypes in one of several formats - bimbam, rqtl2, or geno (a format used by qtlreaper which is just a CSV file consisting of marker positions and genotypes) - -Rqtl2 genotype queries can also include the dataset name and will return a zip of the genotypes, phenotypes, and gene map (marker names/positions). For example: -``` -curl http://genenetwork.org/api/v_pre1/genotypes/rqtl2/BXD/HC_M2_0606_P.zip -``` - -## Fetch Datasets ## -``` -curl http://genenetwork.org/api/v_pre1/datasets/bxd -``` -OR -``` -curl http://genenetwork.org/api/v_pre1/datasets/mouse/bxd -[ { "AvgID": 1, "CreateTime": "Fri, 01 Aug 2003 00:00:00 GMT", "DataScale": "log2", "FullName": "UTHSC/ETHZ/EPFL BXD Liver Polar Metabolites Extraction A, CD Cohorts (Mar 2017) log2", "Id": 1, "Long_Abbreviation": "BXDMicroArray_ProbeSet_August03", "ProbeFreezeId": 3, "ShortName": "Brain U74Av2 08/03 MAS5", "Short_Abbreviation": "Br_U_0803_M", "confidentiality": 0, "public": 0 }, ... { "AvgID": 3, "CreateTime": "Tue, 14 Aug 2018 00:00:00 GMT", "DataScale": "log2", "FullName": "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Aug18) RMA", "Id": 859, "Long_Abbreviation": "EPFLMouseLiverCDRMAApr18", "ProbeFreezeId": 181, "ShortName": "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Aug18) RMA", "Short_Abbreviation": "EPFLMouseLiverCDRMA0818", "confidentiality": 0, "public": 1 } ] -``` -(I added the option to specify species just in case we end up with the same group name across multiple species at some point, though it's currently unnecessary) - -## Fetch Individual Dataset Info ## -### For mRNA Assay/"ProbeSet" ### - -``` -curl http://genenetwork.org/api/v_pre1/dataset/HC_M2_0606_P -``` -OR -``` -curl http://genenetwork.org/api/v_pre1/dataset/bxd/HC_M2_0606_P``` -{ "confidential": 0, "data_scale": "log2", "dataset_type": "mRNA expression", "full_name": "Hippocampus Consortium M430v2 (Jun06) PDNN", "id": 112, "name": "HC_M2_0606_P", "public": 2, "short_name": "Hippocampus M430v2 BXD 06/06 PDNN", "tissue": "Hippocampus mRNA", "tissue_id": 9 } -``` -(This also has the option to specify group/riset) - -### For "Phenotypes" (basically non-mRNA Expression; stuff like weight, sex, etc) ### -For these traits, the query fetches publication info and takes the group and phenotype 'ID' as input. For example: -``` -curl http://genenetwork.org/api/v_pre1/dataset/bxd/10001 -{ "dataset_type": "phenotype", "description": "Central nervous system, morphology: Cerebellum weight, whole, bilateral in adults of both sexes [mg]", "id": 10001, "name": "CBLWT2", "pubmed_id": 11438585, "title": "Genetic control of the mouse cerebellum: identification of quantitative trait loci modulating size and architecture", "year": "2001" } -``` - -## Fetch Sample Data for Dataset ## -``` -curl http://genenetwork.org/api/v_pre1/sample_data/HSNIH-PalmerPublish.csv -``` - -Returns a CSV file with sample/strain names as the columns and trait IDs as rows - -## Fetch Sample Data for Single Trait ## -``` -curl http://genenetwork.org/api/v_pre1/sample_data/HC_M2_0606_P/1436869_at -[ { "data_id": 23415463, "sample_name": "129S1/SvImJ", "sample_name_2": "129S1/SvImJ", "se": 0.123, "value": 8.201 }, { "data_id": 23415463, "sample_name": "A/J", "sample_name_2": "A/J", "se": 0.046, "value": 8.413 }, { "data_id": 23415463, "sample_name": "AKR/J", "sample_name_2": "AKR/J", "se": 0.134, "value": 8.856 }, ... ] -``` - -## Fetch Trait List for Dataset ## -``` -curl http://genenetwork.org/api/v_pre1/traits/HXBBXHPublish.json -[ { "Additive": 0.0499967532467532, "Id": 10001, "LRS": 16.2831307029479, "Locus": "rs106114574", "PhenotypeId": 1449, "PublicationId": 319, "Sequence": 1 }, ... ] -``` - -Both JSON and CSV formats can be specified, with JSON as default. There is also an optional "ids_only" and "names_only" parameter that will only return a list of trait IDs or names, respectively. - -## Fetch Trait Info (Name, Description, Location, etc) ## -### For mRNA Expression/"ProbeSet" ### -``` -curl http://genenetwork.org/api/v_pre1/trait/HC_M2_0606_P/1436869_at -{ "additive": -0.214087568058076, "alias": "HHG1; HLP3; HPE3; SMMCI; Dsh; Hhg1", "chr": "5", "description": "sonic hedgehog (hedgehog)", "id": 99602, "locus": "rs8253327", "lrs": 12.7711275309832, "mb": 28.457155, "mean": 9.27909090909091, "name": "1436869_at", "p_value": 0.306, "se": null, "symbol": "Shh" } -``` - -### For "Phenotypes" ### -For phenotypes this just gets the max LRS, its location, and additive effect (as calculated by qtlreaper) - -Since each group/riset only has one phenotype "dataset", this query takes either the group/riset name or the group/riset name + "Publish" (for example "BXDPublish", which is the dataset name in the DB) as input -``` -curl http://genenetwork.org/api/v_pre1/trait/BXD/10001 -{ "additive": 2.39444435069444, "id": 4, "locus": "rs48756159", "lrs": 13.4974911471087 } -``` - ---- - -# Analyses # ---- -## Mapping ## -Currently two mapping tools can be used - GEMMA and R/qtl. qtlreaper will be added later with Christian Fischer's RUST implementation - https://github.com/chfi/rust-qtlreaper - -Each method's query takes the following parameters respectively (more will be added): -### GEMMA ### -* trait_id (*required*) - ID for trait being mapped -* db (*required*) - DB name for trait above (Short_Abbreviation listed when you query for datasets) -* use_loco - Whether to use LOCO (leave one chromosome out) method (default = false) -* maf - minor allele frequency (default = 0.01) - -Example query: -``` -curl http://genenetwork.org/api/v_pre1/mapping?trait_id=10015&db=BXDPublish&method=gemma&use_loco=true -``` - -### R/qtl ### -(See the R/qtl guide for information on some of these options - http://www.rqtl.org/manual/qtl-manual.pdf) -* trait_id (*required*) - ID for trait being mapped -* db (*required*) - DB name for trait above (Short_Abbreviation listed when you query for datasets) -* rqtl_method - hk (default) | ehk | em | imp | mr | mr-imp | mr-argmax ; Corresponds to the "method" option for the R/qtl scanone function. -* rqtl_model - normal (default) | binary | 2-part | np ; corresponds to the "model" option for the R/qtl scanone function -* num_perm - number of permutations; 0 by default -* control_marker - Name of marker to use as control; this relies on the user knowing the name of the marker they want to use as a covariate -* interval_mapping - Whether to use interval mapping; "false" by default -* pair_scan - *NYI* - -Example query: -``` -curl http://genenetwork.org/api/v_pre1/mapping?trait_id=1418701_at&db=HC_M2_0606_P&method=rqtl&num_perm=100 -``` - -Some combinations of methods/models may not make sense. The R/qtl manual should be referred to for any questions on its use (specifically the scanone function in this case) - -## Calculate Correlation ## -Currently only Sample and Tissue correlations are implemented - -This query currently takes the following parameters (though more will be added): -* trait_id (*required*) - ID for trait used for correlation -* db (*required*) - DB name for the trait above (this is the Short_Abbreviation listed when you query for datasets) -* target_db (*required*) - Target DB name to be correlated against -* type - sample (default) | tissue -* method - pearson (default) | spearman -* return - Number of results to return (default = 500) - -Example query: -``` -curl http://genenetwork.org/api/v_pre1/correlation?trait_id=1427571_at&db=HC_M2_0606_P&target_db=BXDPublish&type=sample&return_count=100 -[ { "#_strains": 6, "p_value": 0.004804664723032055, "sample_r": -0.942857142857143, "trait": 20511 }, { "#_strains": 6, "p_value": 0.004804664723032055, "sample_r": -0.942857142857143, "trait": 20724 }, { "#_strains": 12, "p_value": 1.8288943424888848e-05, "sample_r": -0.9233615170820528, "trait": 13536 }, { "#_strains": 7, "p_value": 0.006807187408935392, "sample_r": 0.8928571428571429, "trait": 10157 }, { "#_strains": 7, "p_value": 0.006807187408935392, "sample_r": -0.8928571428571429, "trait": 20392 }, ... ] -``` +This document has moved to [gn-docs](https://github.com/genenetwork/gn-docs/blob/master/api/GN2-REST-API.md)! diff --git a/doc/README.org b/doc/README.org index 1236016e..e1c6b614 100644 --- a/doc/README.org +++ b/doc/README.org @@ -26,7 +26,7 @@ * Introduction -Large system deployments can get very [[http://biogems.info/contrib/genenetwork/gn2.svg ][complex]]. In this document we +Large system deployments can get very [[http://genenetwork.org/environments/][complex]]. In this document we explain the GeneNetwork version 2 (GN2) reproducible deployment system which is based on GNU Guix (see also [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). The Guix system can be used to install GN with all its files and dependencies. @@ -81,14 +81,12 @@ GeneNetwork2 with : source ~/opt/guix-pull/etc/profile : git clone https://git.genenetwork.org/guix-bioinformatics/guix-bioinformatics.git ~/guix-bioinformatics : cd ~/guix-bioinformatics -: git pull : env GUIX_PACKAGE_PATH=$HOME/guix-bioinformatics guix package -i genenetwork2 -p ~/opt/genenetwork2 you probably also need guix-past (the upstream channel for older packages): : git clone https://gitlab.inria.fr/guix-hpc/guix-past.git ~/guix-past : cd ~/guix-past -: git pull : env GUIX_PACKAGE_PATH=$HOME/guix-bioinformatics:$HOME/guix-past/modules ~/opt/guix-pull/bin/guix package -i genenetwork2 -p ~/opt/genenetwork2 ignore the warnings. Guix should install the software without trying diff --git a/doc/elasticsearch.org b/doc/elasticsearch.org deleted file mode 100644 index 864a8363..00000000 --- a/doc/elasticsearch.org +++ /dev/null @@ -1,247 +0,0 @@ -* Elasticsearch - -** Introduction - -GeneNetwork uses elasticsearch (ES) for all things considered -'state'. One example is user collections, another is user management. - -** Example - -To get the right environment, first you can get a python REPL with something like - -: env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ../etc/default_settings.py -cli python - -(make sure to use the correct GN2_PROFILE!) - -Next try - -#+BEGIN_SRC python - -from elasticsearch import Elasticsearch, TransportError - -es = Elasticsearch([{ "host": 'localhost', "port": '9200' }]) - -# Dump all data - -es.search("*") - -# To fetch an E-mail record from the users index - -record = es.search( - index = 'users', doc_type = 'local', body = { - "query": { "match": { "email_address": "myname@email.com" } } - }) - -# It is also possible to do wild card matching - -q = { "query": { "wildcard" : { "full_name" : "pjot*" } }} -es.search(index = 'users', doc_type = 'local', body = q) - -# To get elements from that record: - -record['hits']['hits'][0][u'_source']['full_name'] -u'Pjotr' - -record['hits']['hits'][0][u'_source']['email_address'] -u"myname@email.com" - -#+END_SRC - -** Health - -ES provides support for checking its health: - -: curl -XGET http://localhost:9200/_cluster/health?pretty=true - -#+BEGIN_SRC json - - - { - "cluster_name" : "asgard", - "status" : "yellow", - "timed_out" : false, - "number_of_nodes" : 1, - "number_of_data_nodes" : 1, - "active_primary_shards" : 5, - "active_shards" : 5, - "relocating_shards" : 0, - "initializing_shards" : 0, - "unassigned_shards" : 5 - } - -#+END_SRC - -Yellow means just one instance is running (no worries). - -To get full cluster info - -: curl -XGET "localhost:9200/_cluster/stats?human&pretty" - -#+BEGIN_SRC json -{ - "_nodes" : { - "total" : 1, - "successful" : 1, - "failed" : 0 - }, - "cluster_name" : "elasticsearch", - "timestamp" : 1529050366452, - "status" : "yellow", - "indices" : { - "count" : 3, - "shards" : { - "total" : 15, - "primaries" : 15, - "replication" : 0.0, - "index" : { - "shards" : { - "min" : 5, - "max" : 5, - "avg" : 5.0 - }, - "primaries" : { - "min" : 5, - "max" : 5, - "avg" : 5.0 - }, - "replication" : { - "min" : 0.0, - "max" : 0.0, - "avg" : 0.0 - } - } - }, - "docs" : { - "count" : 14579, - "deleted" : 0 - }, - "store" : { - "size" : "44.7mb", - "size_in_bytes" : 46892794 - }, - "fielddata" : { - "memory_size" : "0b", - "memory_size_in_bytes" : 0, - "evictions" : 0 - }, - "query_cache" : { - "memory_size" : "0b", - "memory_size_in_bytes" : 0, - "total_count" : 0, - "hit_count" : 0, - "miss_count" : 0, - "cache_size" : 0, - "cache_count" : 0, - "evictions" : 0 - }, - "completion" : { - "size" : "0b", - "size_in_bytes" : 0 - }, - "segments" : { - "count" : 24, - "memory" : "157.3kb", - "memory_in_bytes" : 161112, - "terms_memory" : "122.6kb", - "terms_memory_in_bytes" : 125569, - "stored_fields_memory" : "15.3kb", - "stored_fields_memory_in_bytes" : 15728, - "term_vectors_memory" : "0b", - "term_vectors_memory_in_bytes" : 0, - "norms_memory" : "10.8kb", - "norms_memory_in_bytes" : 11136, - "points_memory" : "111b", - "points_memory_in_bytes" : 111, - "doc_values_memory" : "8.3kb", - "doc_values_memory_in_bytes" : 8568, - "index_writer_memory" : "0b", - "index_writer_memory_in_bytes" : 0, - "version_map_memory" : "0b", - "version_map_memory_in_bytes" : 0, - "fixed_bit_set" : "0b", - "fixed_bit_set_memory_in_bytes" : 0, - "max_unsafe_auto_id_timestamp" : -1, - "file_sizes" : { } - } - }, - "nodes" : { - "count" : { - "total" : 1, - "data" : 1, - "coordinating_only" : 0, - "master" : 1, - "ingest" : 1 - }, - "versions" : [ - "6.2.1" - ], - "os" : { - "available_processors" : 16, - "allocated_processors" : 16, - "names" : [ - { - "name" : "Linux", - "count" : 1 - } - ], - "mem" : { - "total" : "125.9gb", - "total_in_bytes" : 135189286912, - "free" : "48.3gb", - "free_in_bytes" : 51922628608, - "used" : "77.5gb", - "used_in_bytes" : 83266658304, - "free_percent" : 38, - "used_percent" : 62 - } - }, - "process" : { - "cpu" : { - "percent" : 0 - }, - "open_file_descriptors" : { - "min" : 415, - "max" : 415, - "avg" : 415 - } - }, - "jvm" : { - "max_uptime" : "1.9d", - "max_uptime_in_millis" : 165800616, - "versions" : [ - { - "version" : "9.0.4", - "vm_name" : "OpenJDK 64-Bit Server VM", - "vm_version" : "9.0.4+11", - "vm_vendor" : "Oracle Corporation", - "count" : 1 - } - ], - "mem" : { - "heap_used" : "1.1gb", - "heap_used_in_bytes" : 1214872032, - "heap_max" : "23.8gb", - "heap_max_in_bytes" : 25656426496 - }, - "threads" : 110 - }, - "fs" : { - "total" : "786.4gb", - "total_in_bytes" : 844400918528, - "free" : "246.5gb", - "free_in_bytes" : 264688160768, - "available" : "206.5gb", - "available_in_bytes" : 221771468800 - }, - "plugins" : [ ], - "network_types" : { - "transport_types" : { - "netty4" : 1 - }, - "http_types" : { - "netty4" : 1 - } - } - } -} -#+BEGIN_SRC json diff --git a/doc/heatmap-generation.org b/doc/heatmap-generation.org new file mode 100644 index 00000000..a697c70b --- /dev/null +++ b/doc/heatmap-generation.org @@ -0,0 +1,34 @@ +#+STARTUP: inlineimages +#+TITLE: Heatmap Generation +#+AUTHOR: Muriithi Frederick Muriuki + +* Generating Heatmaps + +Like a lot of other features, the heatmap generation requires an existing collection. If none exists, see [[][Creating a new collection]] for how to create a new collection. + +Once you have a collection, you can navigate to the collections page by clicking on the "Collections" link in the header + + +[[./images/gn2_header_collections.png]] + +From that page, pick the collection that you want to work with by clicking on its name on the collections table. + +That takes you to that collection's page, where you can select the data that you want to use to generate the heatmap. + +** Selecting Orientation + +Once you have selected the data, select the orientation of the heatmap you want generated. You do this by selecting either *"vertical"* or *"horizontal"* in the heatmaps form: + +[[./images/heatmap_form.png]] + +Once you have selected the orientation, click on the "Generate Heatmap" button as in the image above. + +The heatmap generation might take a while, but once it is done, an image shows up above the data table. + +** Downloading the PNG copy of the Heatmap + +Once the heatmap image is shown, hovering over it, displays some tools to interact with the image. + +To download, hover over the heatmap image, and click on the "Download plot as png" icon as shown. + +[[./images/heatmap_with_hover_tools.png]] diff --git a/doc/images/gn2_header_collections.png b/doc/images/gn2_header_collections.png Binary files differnew file mode 100644 index 00000000..ac23f9c1 --- /dev/null +++ b/doc/images/gn2_header_collections.png diff --git a/doc/images/heatmap_form.png b/doc/images/heatmap_form.png Binary files differnew file mode 100644 index 00000000..163fbb60 --- /dev/null +++ b/doc/images/heatmap_form.png diff --git a/doc/images/heatmap_with_hover_tools.png b/doc/images/heatmap_with_hover_tools.png Binary files differnew file mode 100644 index 00000000..4ab79f99 --- /dev/null +++ b/doc/images/heatmap_with_hover_tools.png diff --git a/doc/joss/2016/2020.12.23.424047v1.full.pdf b/doc/joss/2016/2020.12.23.424047v1.full.pdf Binary files differnew file mode 100644 index 00000000..491dddf3 --- /dev/null +++ b/doc/joss/2016/2020.12.23.424047v1.full.pdf diff --git a/etc/default_settings.py b/etc/default_settings.py index a194b10e..8636f4db 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -25,7 +25,12 @@ import os import sys GN_VERSION = open("../etc/VERSION", "r").read() -GN_SERVER_URL = "http://localhost:8880/" # REST API server + +# Redis +REDIS_URL = "redis://:@localhost:6379/0" + +# gn2-proxy +GN2_PROXY = "http://localhost:8080" # ---- MySQL diff --git a/guix.scm b/guix.scm new file mode 100644 index 00000000..9352c7c5 --- /dev/null +++ b/guix.scm @@ -0,0 +1,24 @@ +;; Make sure you have the +;; https://git.genenetwork.org/guix-bioinformatics/guix-bioinformatics +;; channel set up. +;; +;; To drop into a development environment, run +;; +;; guix shell -Df guix.scm +;; +;; To get a development environment in a container, run +;; +;; guix shell -C -Df guix.scm + +(use-modules (gn packages genenetwork) + (guix gexp) + (guix git-download) + (guix packages)) + +(define %source-dir (dirname (current-filename))) + +(package + (inherit genenetwork3) + (source (local-file %source-dir "genenetwork3-checkout" + #:recursive? #t + #:select? (git-predicate %source-dir)))) diff --git a/scripts/authentication/editors.py b/scripts/authentication/editors.py new file mode 100755 index 00000000..dc3b1075 --- /dev/null +++ b/scripts/authentication/editors.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Manually add editors users""" +import redis +import json +import uuid +import datetime + +if __name__ == "__main__": + conn = redis.Redis(decode_responses=True) + group_uid = "" + for guid in conn.hgetall("groups"): + group_details = json.loads(conn.hget("groups", guid)) + if group_details.get("name") == "editors": + group_uid = guid + break + + if not group_uid: + group_uid = str(uuid.uuid4()) + timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + conn.hset( + "groups", + group_uid, + json.dumps( + { + "name": "editors", + "admins": [], + "members": ["8ad942fe-490d-453e-bd37-56f252e41603"], + "created_timestamp": timestamp, + "changed_timestamp": timestamp, + })) + + for resource in conn.hgetall("resources"): + _resource = json.loads(conn.hget("resources", resource)) + _resource["default_mask"] = { + 'data': 'view', + 'metadata': 'view', + 'admin': 'not-admin', + } + _resource["group_masks"] = { + group_uid: { + 'metadata': 'edit', + 'data': 'edit', + 'admin': 'edit-admins', + }} + conn.hset("resources", resource, json.dumps(_resource)) + print("Done adding editor's group to resources!") diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index c8c2caad..dd9ba808 100644..100755 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """A script for adding users to a specific group. Example: @@ -6,22 +7,40 @@ Assuming there are no groups and 'test@bonfacemunyoki.com' does not exist in Redis: .. code-block:: bash - python group.py -g "editors" -m "test@bonfacemunyoki.com" + python group.py -n "editors" \ + -m "me@bonfacemunyoki.com" results in:: - Successfully created the group: 'editors' - Data: '{"admins": [], "members": []}' + Successfully created the group: 'editors' + `HGET groups 0360449f-5a96-4940-8522-b22d62085da9`: {'name': 'editors', 'admins': [], 'members': ['8ad942fe-490d-453e-bd37-56f252e41603'], 'changed_timestamp': 'Oct 19 2021 09:34AM', 'created_timestamp': 'Oct 19 2021 09:34AM'} + -If 'me@bonfacemunyoki.com' exists in 'users' in Redis and we run: +Assuming we have a group's unique id: .. code-block:: bash - python group.py -g "editors" -m "me@bonfacemunyoki.com" + python group.py -n "editors" \ + -m "me@bonfacemunyoki.com" \ + -g "8ad942fe-490d-453e-bd37-56f252e41603" now results in:: - No new group was created. - Updated Data: {'admins': [], 'members': ['me@bonfacemunyoki.com']} + Successfully created the group: 'editors' + `HGET groups 8ad942fe-490d-453e-bd37-56f252e41603`: {'name': 'editors', 'admins': [], 'members': ['8ad942fe-490d-453e-bd37-56f252e41603'], 'changed_timestamp': 'Oct 19 2021 09:38AM', 'created_timestamp': 'Oct 19 2021 09:38AM'} + +If 'me@bonfacemunyoki.com' exists in 'users' in Redis for the above +command and we run: + +.. code-block:: bash + python group.py -n "editors" \ + -m "me@bonfacemunyoki.com" \ + -g "8ad942fe-490d-453e-bd37-56f252e41603" + +now results in:: + + No new group was created. + `HGET groups 8ad942fe-490d-453e-bd37-56f252e41603`: {'name': 'editors', 'admins': [], 'members': ['8ad942fe-490d-453e-bd37-56f252e41603'], 'changed_timestamp': 'Oct 19 2021 09:40AM'} + """ @@ -36,44 +55,48 @@ from typing import Dict, Optional, Set def create_group_data(users: Dict, target_group: str, members: Optional[str] = None, - admins: Optional[str] = None) -> Dict: - """Return a dictionary that contains the following keys: "key", - "field", and "value" that can be used in a redis hash as follows: - HSET key field value + admins: Optional[str] = None, + group_id: Optional[str] = None) -> Dict: + """Create group data which is isomorphic to a redis HSET i.e.: KEY, + FIELD and VALUE. If the group_id is not specified, a unique hash + will be generated. The "field" return value is a unique-id that is used to distinguish the groups. - Parameters: - - - `users`: a list of users for example: - - {'8ad942fe-490d-453e-bd37-56f252e41603': - '{"email_address": "me@test.com", - "full_name": "John Doe", - "organization": "Genenetwork", - "password": {"algorithm": "pbkdf2", - "hashfunc": "sha256", - "salt": "gJrd1HnPSSCmzB5veMPaVk2ozzDlS1Z7Ggcyl1+pciA=", - "iterations": 100000, "keylength": 32, - "created_timestamp": "2021-09-22T11:32:44.971912", - "password": "edcdaa60e84526c6"}, - "user_id": "8ad942fe", "confirmed": 1, - "registration_info": { - "timestamp": "2021-09-22T11:32:45.028833", - "ip_address": "127.0.0.1", - "user_agent": "Mozilla/5.0"}}'} - - - `target_group`: the group name that will be stored inside the - "groups" hash in Redis. - - - `members`: a comma-separated list of values that contain members - of the `target_group` e.g. "me@test1.com, me@test2.com, - me@test3.com" - - - `admins`: a comma-separated list of values that contain - administrators of the `target_group` e.g. "me@test1.com, - me@test2.com, me@test3.com" + Args: + - users: a list of users for example: + {'8ad942fe-490d-453e-bd37-56f252e41603': + '{"email_address": "me@test.com", + "full_name": "John Doe", + "organization": "Genenetwork", + "password": {"algorithm": "pbkdf2", + "hashfunc": "sha256", + "salt": "gJrd1HnPSSCmzB5veMPaVk2ozzDlS1Z7Ggcyl1+pciA=", + "iterations": 100000, "keylength": 32, + "created_timestamp": "2021-09-22T11:32:44.971912", + "password": "edcdaa60e84526c6"}, + "user_id": "8ad942fe", "confirmed": 1, + "registration_info": { + "timestamp": "2021-09-22T11:32:45.028833", + "ip_address": "127.0.0.1", + "user_agent": "Mozilla/5.0"}}'} + + - target_group: the group name that will be stored inside the + "groups" hash in Redis. + - members: an optional comma-separated list of values that + contain members of the `target_group` e.g. "me@test1.com, + me@test2.com, me@test3.com" + - admins: an optional comma-separated list of values that + contain administrators of the `target_group` + e.g. "me@test1.com, me@test2.com, me@test3.com" + - group_id: an optional unique identifier for a group. If not + set, a unique value will be auto-generated. + + Returns: + A dictionary that contains the following keys: "key", "field", + and "value" that can be used in a redis hash as follows: HSET key + field value """ # Emails @@ -95,11 +118,12 @@ def create_group_data(users: Dict, target_group: str, timestamp: str = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') return {"key": "groups", - "field": str(uuid.uuid4()), + "field": (group_id or str(uuid.uuid4())), "value": json.dumps({ "name": target_group, "admins": list(admin_ids), "members": list(member_ids), + "created_timestamp": timestamp, "changed_timestamp": timestamp, })} @@ -107,8 +131,10 @@ def create_group_data(users: Dict, target_group: str, if __name__ == "__main__": # Initialising the parser CLI arguments parser = argparse.ArgumentParser() - parser.add_argument("-g", "--group-name", + parser.add_argument("-n", "--group-name", help="This is the name of the GROUP mask") + parser.add_argument("-g", "--group-id", + help="[Optional] This is the name of the GROUP mask") parser.add_argument("-m", "--members", help="Members of the GROUP mask") parser.add_argument("-a", "--admins", @@ -132,7 +158,8 @@ if __name__ == "__main__": users=USERS, target_group=args.group_name, members=members, - admins=admins) + admins=admins, + group_id=args.group_id) if not REDIS_CONN.hget("groups", data.get("field")): updated_data = json.loads(data["value"]) @@ -143,11 +170,10 @@ if __name__ == "__main__": created_p = REDIS_CONN.hset(data.get("key", ""), data.get("field", ""), data.get("value", "")) - groups = json.loads(REDIS_CONN.hget("groups", data.get("field"))) # type: ignore if created_p: exit(f"\nSuccessfully created the group: '{args.group_name}'\n" - f"`HGETALL groups {args.group_name}`: {groups}\n") + f"`HGET groups {data.get('field')}`: {groups}\n") exit("\nNo new group was created.\n" - f"`HGETALL groups {args.group_name}`: {groups}\n") + f"`HGET groups {data.get('field')}`: {groups}\n") diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py index 4996f34c..4b8d801a 100644..100755 --- a/scripts/authentication/resource.py +++ b/scripts/authentication/resource.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """A script that: - Optionally restores data from a json file. @@ -37,14 +38,14 @@ from datetime import datetime def recover_hash(name: str, file_path: str, set_function) -> bool: """Recover back-ups using the `set_function` - Parameters: + Args: + - name: Redis hash where `file_path` will be restored + - file_path: File path where redis hash is sourced from + - set_function: Function used to do the Redis backup for + example: HSET - - `name`: Redis hash where `file_path` will be restored - - - `file_path`: File path where redis hash is sourced from - - - `set_function`: Function used to do the Redis backup for - example: HSET + Returns: + A boolean indicating whether the function ran successfully. """ try: @@ -97,7 +98,8 @@ if __name__ == "__main__": for resource_id, resource in RESOURCES.items(): _resource = json.loads(resource) # str -> dict conversion _resource["group_masks"] = {args.group_id: {"metadata": "edit", - "data": "edit"}} + "data": "edit", + "admin": "not-admin"}} REDIS_CONN.hset("resources", resource_id, json.dumps(_resource)) diff --git a/scripts/convert_dol_genotypes.py b/scripts/convert_dol_genotypes.py index 81b3bd6d..5cda2e9c 100644 --- a/scripts/convert_dol_genotypes.py +++ b/scripts/convert_dol_genotypes.py @@ -1,6 +1,8 @@ # This is just to convert the Rqtl2 format genotype files for DOL into a .geno file # Everything is hard-coded since I doubt this will be re-used and I just wanted to generate the file quickly +# This is to be used on the files generated as described by Karl Broman here - https://kbroman.org/qtl2/pages/prep_do_data.html + import os geno_dir = "/home/zas1024/gn2-zach/DO_genotypes/" @@ -30,9 +32,13 @@ for filename in os.listdir(geno_dir): if i < 3: continue elif not len(sample_names) and i == 3: - sample_names = [item.replace("TLB", "TB") for item in line_items[1:]] + sample_names_positions = [[item.replace("TLB", "TB").strip(), i] for i, item in enumerate(line_items[1:])] + sample_names_positions.sort(key = lambda x: x[0][2:]) + sample_names = [sample[0] for sample in sample_names_positions] elif i > 3: - marker_data[line_items[0]]['genotypes'] = ["X" if item.strip() == "-" else item.strip() for item in line_items[1:]] + genotypes = ["X" if item.strip() == "-" else item.strip() for item in line_items[1:]] + ordered_genotypes = [genotypes[i].strip() for i in [pos[1] for pos in sample_names_positions]] + marker_data[line_items[0]]['genotypes'] = ordered_genotypes # Generate list of marker obs to iterate through when writing to .geno file marker_list = [] @@ -46,6 +52,7 @@ for key, value in marker_data.items(): } marker_list.append(this_marker) + def sort_func(e): """For ensuring that X/Y chromosomes/mitochondria are sorted to the end correctly""" try: @@ -63,7 +70,7 @@ marker_list.sort(key=sort_func) # Write lines to .geno file with open(gn_geno_path, "w") as gn_geno_fh: - gn_geno_fh.write("\t".join((["Chr", "Locus", "cM", "Mb"] + sample_names))) + gn_geno_fh.write("\t".join((["Chr", "Locus", "cM", "Mb"] + sample_names)) + "\n") for marker in marker_list: row_contents = [ marker['chr'], @@ -72,3 +79,4 @@ with open(gn_geno_path, "w") as gn_geno_fh: marker['pos'] ] + marker['genotypes'] gn_geno_fh.write("\t".join(row_contents) + "\n") + diff --git a/test/requests/parametrized_test.py b/test/requests/parametrized_test.py deleted file mode 100644 index 50003850..00000000 --- a/test/requests/parametrized_test.py +++ /dev/null @@ -1,32 +0,0 @@ -import logging -import unittest -from wqflask import app -from utility.elasticsearch_tools import get_elasticsearch_connection, get_user_by_unique_column -from elasticsearch import Elasticsearch, TransportError - -class ParametrizedTest(unittest.TestCase): - - def __init__(self, methodName='runTest', gn2_url="http://localhost:5003", es_url="localhost:9200"): - super(ParametrizedTest, self).__init__(methodName=methodName) - self.gn2_url = gn2_url - self.es_url = es_url - - def setUp(self): - self.es = get_elasticsearch_connection() - self.es_cleanup = [] - - es_logger = logging.getLogger("elasticsearch") - es_logger.setLevel(app.config.get("LOG_LEVEL")) - es_logger.addHandler( - logging.FileHandler("/tmp/es_TestRegistrationInfo.log")) - es_trace_logger = logging.getLogger("elasticsearch.trace") - es_trace_logger.addHandler( - logging.FileHandler("/tmp/es_TestRegistrationTrace.log")) - - def tearDown(self): - from time import sleep - self.es.delete_by_query( - index="users" - , doc_type="local" - , body={"query":{"match":{"email_address":"test@user.com"}}}) - sleep(1) diff --git a/test/requests/test-website.py b/test/requests/test-website.py index 8bfb47c2..d619a7d5 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -43,7 +43,6 @@ def dummy(args_obj, parser): def integration_tests(args_obj, parser): gn2_url = args_obj.host - es_url = app.config.get("ELASTICSEARCH_HOST")+":"+str(app.config.get("ELASTICSEARCH_PORT")) run_integration_tests(gn2_url, es_url) def initTest(klass, gn2_url, es_url): diff --git a/test/requests/test_forgot_password.py b/test/requests/test_forgot_password.py deleted file mode 100644 index 346524bc..00000000 --- a/test/requests/test_forgot_password.py +++ /dev/null @@ -1,50 +0,0 @@ -import requests -from utility.elasticsearch_tools import get_user_by_unique_column -from parameterized import parameterized -from parametrized_test import ParametrizedTest - -passwork_reset_link = '' -forgot_password_page = None - -class TestForgotPassword(ParametrizedTest): - - def setUp(self): - super(TestForgotPassword, self).setUp() - self.forgot_password_url = self.gn2_url+"/n/forgot_password_submit" - def send_email(to_addr, msg, fromaddr="no-reply@genenetwork.org"): - print("CALLING: send_email_mock()") - email_data = { - "to_addr": to_addr - , "msg": msg - , "fromaddr": from_addr} - - data = { - "es_connection": self.es, - "email_address": "test@user.com", - "full_name": "Test User", - "organization": "Test Organisation", - "password": "test_password", - "password_confirm": "test_password" - } - - - def testWithoutEmail(self): - data = {"email_address": ""} - error_notification = '<div class="alert alert-danger">You MUST provide an email</div>' - result = requests.post(self.forgot_password_url, data=data) - self.assertEqual(result.url, self.gn2_url+"/n/forgot_password") - self.assertTrue( - result.content.find(error_notification) >= 0 - , "Error message should be displayed but was not") - - def testWithNonExistingEmail(self): - # Monkey patching doesn't work, so simply test that getting by email - # returns the correct data - user = get_user_by_unique_column(self.es, "email_address", "non-existent@domain.com") - self.assertTrue(user is None, "Should not find non-existent user") - - def testWithExistingEmail(self): - # Monkey patching doesn't work, so simply test that getting by email - # returns the correct data - user = get_user_by_unique_column(self.es, "email_address", "test@user.com") - self.assertTrue(user is not None, "Should find user") diff --git a/test/requests/test_login_github.py b/test/requests/test_login_github.py deleted file mode 100644 index 1bf4f695..00000000 --- a/test/requests/test_login_github.py +++ /dev/null @@ -1,47 +0,0 @@ -import uuid -import requests -from time import sleep -from wqflask import app -from parameterized import parameterized -from parametrized_test import ParametrizedTest - -login_link_text = '<a id="login_in" href="/n/login">Sign in</a>' -logout_link_text = '<a id="login_out" title="Signed in as ." href="/n/logout">Sign out</a>' -uid = str(uuid.uuid4()) - -class TestLoginGithub(ParametrizedTest): - - def setUp(self): - super(TestLoginGithub, self).setUp() - data = { - "user_id": uid - , "name": "A. T. Est User" - , "github_id": 693024 - , "user_url": "https://fake-github.com/atestuser" - , "login_type": "github" - , "organization": "" - , "active": 1 - , "confirmed": 1 - } - self.es.create(index="users", doc_type="local", body=data, id=uid) - sleep(1) - - def tearDown(self): - super(TestLoginGithub, self).tearDown() - self.es.delete(index="users", doc_type="local", id=uid) - - def testLoginUrl(self): - login_button_text = '<a href="https://github.com/login/oauth/authorize?client_id=' + app.config.get("GITHUB_CLIENT_ID") + '&client_secret=' + app.config.get("GITHUB_CLIENT_SECRET") + '" title="Login with GitHub" class="btn btn-info btn-group">Login with Github</a>' - result = requests.get(self.gn2_url+"/n/login") - index = result.content.find(login_button_text) - self.assertTrue(index >= 0, "Should have found `Login with Github` button") - - @parameterized.expand([ - ("1234", login_link_text, "Login should have failed with non-existing user") - , (uid, logout_link_text, "Login should have been successful with existing user") - ]) - def testLogin(self, test_uid, expected, message): - url = self.gn2_url+"/n/login?type=github&uid="+test_uid - result = requests.get(url) - index = result.content.find(expected) - self.assertTrue(index >= 0, message) diff --git a/test/requests/test_login_local.py b/test/requests/test_login_local.py deleted file mode 100644 index 6691d135..00000000 --- a/test/requests/test_login_local.py +++ /dev/null @@ -1,57 +0,0 @@ -import requests -from parameterized import parameterized -from parametrized_test import ParametrizedTest - -login_link_text = '<a id="login_in" href="/n/login">Sign in</a>' -logout_link_text = '<a id="login_out" title="Signed in as ." href="/n/logout">Sign out</a>' - -class TestLoginLocal(ParametrizedTest): - - def setUp(self): - super(TestLoginLocal, self).setUp() - self.login_url = self.gn2_url +"/n/login" - data = { - "es_connection": self.es, - "email_address": "test@user.com", - "full_name": "Test User", - "organization": "Test Organisation", - "password": "test_password", - "password_confirm": "test_password" - } - - - @parameterized.expand([ - ( - { - "email_address": "non@existent.email", - "password": "doesitmatter?" - }, login_link_text, "Login should have failed with the wrong user details."), - ( - { - "email_address": "test@user.com", - "password": "test_password" - }, logout_link_text, "Login should have been successful with correct user details and neither import_collections nor remember_me set"), - ( - { - "email_address": "test@user.com", - "password": "test_password", - "import_collections": "y" - }, logout_link_text, "Login should have been successful with correct user details and only import_collections set"), - ( - { - "email_address": "test@user.com", - "password": "test_password", - "remember_me": "y" - }, logout_link_text, "Login should have been successful with correct user details and only remember_me set"), - ( - { - "email_address": "test@user.com", - "password": "test_password", - "remember_me": "y", - "import_collections": "y" - }, logout_link_text, "Login should have been successful with correct user details, and both remember_me, and import_collections set") - ]) - def testLogin(self, data, expected, message): - result = requests.post(self.login_url, data=data) - index = result.content.find(expected) - self.assertTrue(index >= 0, message) diff --git a/test/requests/test_login_orcid.py b/test/requests/test_login_orcid.py deleted file mode 100644 index ea15642e..00000000 --- a/test/requests/test_login_orcid.py +++ /dev/null @@ -1,47 +0,0 @@ -import uuid -import requests -from time import sleep -from wqflask import app -from parameterized import parameterized -from parametrized_test import ParametrizedTest - -login_link_text = '<a id="login_in" href="/n/login">Sign in</a>' -logout_link_text = '<a id="login_out" title="Signed in as ." href="/n/logout">Sign out</a>' -uid = str(uuid.uuid4()) - -class TestLoginOrcid(ParametrizedTest): - - def setUp(self): - super(TestLoginOrcid, self).setUp() - data = { - "user_id": uid - , "name": "A. T. Est User" - , "orcid": 345872 - , "user_url": "https://fake-orcid.org/atestuser" - , "login_type": "orcid" - , "organization": "" - , "active": 1 - , "confirmed": 1 - } - self.es.create(index="users", doc_type="local", body=data, id=uid) - sleep(1) - - def tearDown(self): - super(TestLoginOrcid, self).tearDown() - self.es.delete(index="users", doc_type="local", id=uid) - - def testLoginUrl(self): - login_button_text = 'a href="https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=' + app.config.get("ORCID_CLIENT_ID") + '&client_secret=' + app.config.get("ORCID_CLIENT_SECRET") + '" title="Login with ORCID" class="btn btn-info btn-group">Login with ORCID</a>' - result = requests.get(self.gn2_url+"/n/login") - index = result.content.find(login_button_text) - self.assertTrue(index >= 0, "Should have found `Login with ORCID` button") - - @parameterized.expand([ - ("1234", login_link_text, "Login should have failed with non-existing user") - , (uid, logout_link_text, "Login should have been successful with existing user") - ]) - def testLogin(self, test_uid, expected, message): - url = self.gn2_url+"/n/login?type=orcid&uid="+test_uid - result = requests.get(url) - index = result.content.find(expected) - self.assertTrue(index >= 0, message) diff --git a/test/requests/test_registration.py b/test/requests/test_registration.py index 0047e8a6..5d08bf58 100644 --- a/test/requests/test_registration.py +++ b/test/requests/test_registration.py @@ -1,31 +1,25 @@ import sys import requests -from parametrized_test import ParametrizedTest class TestRegistration(ParametrizedTest): - def tearDown(self): - for item in self.es_cleanup: - self.es.delete(index="users", doc_type="local", id=item["_id"]) def testRegistrationPage(self): - if self.es.ping(): - data = { - "email_address": "test@user.com", - "full_name": "Test User", - "organization": "Test Organisation", - "password": "test_password", - "password_confirm": "test_password" - } - requests.post(self.gn2_url+"/n/register", data) - response = self.es.search( - index="users" - , doc_type="local" - , body={ - "query": {"match": {"email_address": "test@user.com"}}}) - self.assertEqual(len(response["hits"]["hits"]), 1) - else: - self.skipTest("The elasticsearch server is down") + data = { + "email_address": "test@user.com", + "full_name": "Test User", + "organization": "Test Organisation", + "password": "test_password", + "password_confirm": "test_password" + } + requests.post(self.gn2_url+"/n/register", data) + response = self.es.search( + index="users" + , doc_type="local" + , body={ + "query": {"match": {"email_address": "test@user.com"}}}) + self.assertEqual(len(response["hits"]["hits"]), 1) + def main(gn2, es): import unittest diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 8906ab69..af248659 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -20,7 +20,7 @@ from dataclasses import dataclass from dataclasses import field from dataclasses import InitVar -from typing import Optional, Dict +from typing import Optional, Dict, List from db.call import fetchall, fetchone, fetch1 from utility.logger import getLogger from utility.tools import USE_GN_SERVER, USE_REDIS, flat_files, flat_file_exists, GN2_BASE_URL @@ -39,6 +39,9 @@ from db import webqtlDatabaseFunction from base import species from base import webqtlConfig from flask import Flask, g +from base.webqtlConfig import TMPDIR +from urllib.parse import urlparse +from utility.tools import SQL_URI import os import math import string @@ -50,6 +53,8 @@ import requests import gzip import pickle as pickle import itertools +import hashlib +import datetime from redis import Redis @@ -397,7 +402,8 @@ class DatasetGroup: self.parlist = [maternal, paternal] def get_study_samplelists(self): - study_sample_file = locate_ignore_error(self.name + ".json", 'study_sample_lists') + study_sample_file = locate_ignore_error( + self.name + ".json", 'study_sample_lists') try: f = open(study_sample_file) except: @@ -423,8 +429,6 @@ class DatasetGroup: if result is not None: self.samplelist = json.loads(result) else: - logger.debug("Cache not hit") - genotype_fn = locate_ignore_error(self.name + ".geno", 'genotype') if genotype_fn: self.samplelist = get_group_samplelists.get_samplelist( @@ -447,7 +451,6 @@ class DatasetGroup: # genotype_1 is Dataset Object without parents and f1 # genotype_2 is Dataset Object with parents and f1 (not for intercross) - # reaper barfs on unicode filenames, so here we ensure it's a string if self.genofile: if "RData" in self.genofile: # ZS: This is a temporary fix; I need to change the way the JSON files that point to multiple genotype files are structured to point to other file types like RData @@ -726,7 +729,6 @@ class DataSet: data_results = self.chunk_dataset(query_results, len(sample_ids)) self.samplelist = sorted_samplelist self.trait_data = data_results - def get_trait_data(self, sample_list=None): if sample_list: @@ -745,66 +747,75 @@ class DataSet: and Species.name = '{}' """.format(create_in_clause(self.samplelist), *mescape(self.group.species)) results = dict(g.db.execute(query).fetchall()) - sample_ids = [results[item] for item in self.samplelist] + sample_ids = [results.get(item) + for item in self.samplelist if item is not None] # MySQL limits the number of tables that can be used in a join to 61, # so we break the sample ids into smaller chunks # Postgres doesn't have that limit, so we can get rid of this after we transition chunk_size = 50 number_chunks = int(math.ceil(len(sample_ids) / chunk_size)) - trait_sample_data = [] - for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): - if self.type == "Publish": - dataset_type = "Phenotype" - else: - dataset_type = self.type - temp = ['T%s.value' % item for item in sample_ids_step] - if self.type == "Publish": - query = "SELECT {}XRef.Id,".format(escape(self.type)) - else: - query = "SELECT {}.Name,".format(escape(dataset_type)) - data_start_pos = 1 - query += ', '.join(temp) - query += ' FROM ({}, {}XRef, {}Freeze) '.format(*mescape(dataset_type, - self.type, - self.type)) - - for item in sample_ids_step: - query += """ - left join {}Data as T{} on T{}.Id = {}XRef.DataId - and T{}.StrainId={}\n - """.format(*mescape(self.type, item, item, self.type, item, item)) - - if self.type == "Publish": - query += """ - WHERE {}XRef.InbredSetId = {}Freeze.InbredSetId - and {}Freeze.Name = '{}' - and {}.Id = {}XRef.{}Id - order by {}.Id - """.format(*mescape(self.type, self.type, self.type, self.name, - dataset_type, self.type, dataset_type, dataset_type)) - else: - query += """ - WHERE {}XRef.{}FreezeId = {}Freeze.Id - and {}Freeze.Name = '{}' - and {}.Id = {}XRef.{}Id - order by {}.Id - """.format(*mescape(self.type, self.type, self.type, self.type, - self.name, dataset_type, self.type, self.type, dataset_type)) - results = g.db.execute(query).fetchall() - trait_sample_data.append(results) + cached_results = fetch_cached_results(self.name, self.type) + if cached_results is None: + trait_sample_data = [] + for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): + if self.type == "Publish": + dataset_type = "Phenotype" + else: + dataset_type = self.type + temp = ['T%s.value' % item for item in sample_ids_step] + if self.type == "Publish": + query = "SELECT {}XRef.Id,".format(escape(self.type)) + else: + query = "SELECT {}.Name,".format(escape(dataset_type)) + data_start_pos = 1 + query += ', '.join(temp) + query += ' FROM ({}, {}XRef, {}Freeze) '.format(*mescape(dataset_type, + self.type, + self.type)) + + for item in sample_ids_step: + query += """ + left join {}Data as T{} on T{}.Id = {}XRef.DataId + and T{}.StrainId={}\n + """.format(*mescape(self.type, item, item, self.type, item, item)) + + if self.type == "Publish": + query += """ + WHERE {}XRef.InbredSetId = {}Freeze.InbredSetId + and {}Freeze.Name = '{}' + and {}.Id = {}XRef.{}Id + order by {}.Id + """.format(*mescape(self.type, self.type, self.type, self.name, + dataset_type, self.type, dataset_type, dataset_type)) + else: + query += """ + WHERE {}XRef.{}FreezeId = {}Freeze.Id + and {}Freeze.Name = '{}' + and {}.Id = {}XRef.{}Id + order by {}.Id + """.format(*mescape(self.type, self.type, self.type, self.type, + self.name, dataset_type, self.type, self.type, dataset_type)) - trait_count = len(trait_sample_data[0]) - self.trait_data = collections.defaultdict(list) + results = g.db.execute(query).fetchall() + trait_sample_data.append([list(result) for result in results]) - # put all of the separate data together into a dictionary where the keys are - # trait names and values are lists of sample values - for trait_counter in range(trait_count): - trait_name = trait_sample_data[0][trait_counter][0] - for chunk_counter in range(int(number_chunks)): - self.trait_data[trait_name] += ( - trait_sample_data[chunk_counter][trait_counter][data_start_pos:]) + trait_count = len(trait_sample_data[0]) + self.trait_data = collections.defaultdict(list) + + data_start_pos = 1 + for trait_counter in range(trait_count): + trait_name = trait_sample_data[0][trait_counter][0] + for chunk_counter in range(int(number_chunks)): + self.trait_data[trait_name] += ( + trait_sample_data[chunk_counter][trait_counter][data_start_pos:]) + + cache_dataset_results( + self.name, self.type, self.trait_data) + else: + + self.trait_data = cached_results class PhenotypeDataSet(DataSet): @@ -1242,3 +1253,65 @@ def geno_mrna_confidentiality(ob): if confidential: return True + + +def parse_db_url(): + parsed_db = urlparse(SQL_URI) + + return (parsed_db.hostname, parsed_db.username, + parsed_db.password, parsed_db.path[1:]) + + +def query_table_timestamp(dataset_type: str): + """function to query the update timestamp of a given dataset_type""" + + # computation data and actions + + fetch_db_name = parse_db_url() + query_update_time = f""" + SELECT UPDATE_TIME FROM information_schema.tables + WHERE TABLE_SCHEMA = '{fetch_db_name[-1]}' + AND TABLE_NAME = '{dataset_type}Data' + """ + + date_time_obj = g.db.execute(query_update_time).fetchone()[0] + return date_time_obj.strftime("%Y-%m-%d %H:%M:%S") + + +def generate_hash_file(dataset_name: str, dataset_type: str, dataset_timestamp: str): + """given the trait_name generate a unique name for this""" + string_unicode = f"{dataset_name}{dataset_timestamp}".encode() + md5hash = hashlib.md5(string_unicode) + return md5hash.hexdigest() + + +def cache_dataset_results(dataset_name: str, dataset_type: str, query_results: List): + """function to cache dataset query results to file + input dataset_name and type query_results(already processed in default dict format) + """ + # data computations actions + # store the file path on redis + + table_timestamp = query_table_timestamp(dataset_type) + + + file_name = generate_hash_file(dataset_name, dataset_type, table_timestamp) + file_path = os.path.join(TMPDIR, f"{file_name}.json") + + with open(file_path, "w") as file_handler: + json.dump(query_results, file_handler) + + +def fetch_cached_results(dataset_name: str, dataset_type: str): + """function to fetch the cached results""" + + table_timestamp = query_table_timestamp(dataset_type) + + file_name = generate_hash_file(dataset_name, dataset_type, table_timestamp) + file_path = os.path.join(TMPDIR, f"{file_name}.json") + try: + with open(file_path, "r") as file_handler: + + return json.load(file_handler) + except FileNotFoundError: + pass diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 96a09302..f0749858 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -7,7 +7,7 @@ from base.webqtlCaseData import webqtlCaseData from base.data_set import create_dataset from utility import hmac from utility.authentication_tools import check_resource_availability -from utility.tools import GN2_BASE_URL +from utility.tools import GN2_BASE_URL, GN_PROXY_URL from utility.redis_tools import get_redis_conn, get_resource_id from utility.db_tools import escape @@ -361,10 +361,10 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): resource_id = get_resource_id(dataset, trait.name) if dataset.type == 'Publish': - the_url = "http://localhost:8080/run-action?resource={}&user={}&branch=data&action=view".format( + the_url = GN_PROXY_URL + "run-action?resource={}&user={}&branch=data&action=view".format( resource_id, g.user_session.user_id) else: - the_url = "http://localhost:8080/run-action?resource={}&user={}&branch=data&action=view&trait={}".format( + the_url = GN_PROXY_URL + "run-action?resource={}&user={}&branch=data&action=view&trait={}".format( resource_id, g.user_session.user_id, trait.name) try: diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py index 0cc963e5..32780ca6 100644 --- a/wqflask/maintenance/quantile_normalize.py +++ b/wqflask/maintenance/quantile_normalize.py @@ -5,14 +5,10 @@ import urllib.parse import numpy as np import pandas as pd -from elasticsearch import Elasticsearch, TransportError -from elasticsearch.helpers import bulk from flask import Flask, g, request from wqflask import app -from utility.elasticsearch_tools import get_elasticsearch_connection -from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT, SQL_URI def parse_db_uri(): @@ -106,20 +102,6 @@ if __name__ == '__main__': Conn = MySQLdb.Connect(**parse_db_uri()) Cursor = Conn.cursor() - # es = Elasticsearch([{ - # "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT - # }], timeout=60) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None - - es = get_elasticsearch_connection(for_user=False) - - #input_filename = "/home/zas1024/cfw_data/" + sys.argv[1] + ".txt" - #input_df = create_dataframe(input_filename) - #output_df = quantileNormalize(input_df) - - #output_df.to_csv('quant_norm.csv', sep='\t') - - #out_filename = sys.argv[1][:-4] + '_quantnorm.txt' - success, _ = bulk(es, set_data(sys.argv[1])) response = es.search( diff --git a/wqflask/tests/unit/wqflask/test_markdown_routes.py b/wqflask/tests/unit/wqflask/api/test_markdown_routes.py index 90e0f17c..1c513ac5 100644 --- a/wqflask/tests/unit/wqflask/test_markdown_routes.py +++ b/wqflask/tests/unit/wqflask/api/test_markdown_routes.py @@ -1,10 +1,10 @@ -"""Test functions in markdown utils""" +"""Test functions for wqflask/api/markdown.py""" import unittest from unittest import mock from dataclasses import dataclass -from wqflask.markdown_routes import render_markdown +from wqflask.api.markdown import render_markdown @dataclass @@ -24,9 +24,9 @@ This is another sub-heading""" class TestMarkdownRoutesFunctions(unittest.TestCase): - """Test cases for functions in markdown_routes""" + """Test cases for functions in markdown""" - @mock.patch('wqflask.markdown_routes.requests.get') + @mock.patch('wqflask.api.markdown.requests.get') def test_render_markdown_when_fetching_locally(self, requests_mock): requests_mock.return_value = MockRequests404() markdown_content = render_markdown("general/glossary/glossary.md") @@ -38,7 +38,7 @@ class TestMarkdownRoutesFunctions(unittest.TestCase): self.assertRegexpMatches(markdown_content, "Content for general/glossary/glossary.md not available.") - @mock.patch('wqflask.markdown_routes.requests.get') + @mock.patch('wqflask.api.markdown.requests.get') def test_render_markdown_when_fetching_remotely(self, requests_mock): requests_mock.return_value = MockRequests200() markdown_content = render_markdown("general/glossary/glossary.md") diff --git a/wqflask/tests/unit/wqflask/test_resource_manager.py b/wqflask/tests/unit/wqflask/test_resource_manager.py new file mode 100644 index 00000000..f4335425 --- /dev/null +++ b/wqflask/tests/unit/wqflask/test_resource_manager.py @@ -0,0 +1,94 @@ +"""Test cases for wqflask/resource_manager.py""" +import json +import unittest + +from unittest import mock +from gn3.authentication import get_user_membership +from gn3.authentication import get_highest_user_access_role +from gn3.authentication import DataRole +from gn3.authentication import AdminRole + + +class TestGetUserMembership(unittest.TestCase): + """Test cases for `get_user_membership`""" + + def setUp(self): + conn = mock.MagicMock() + conn.hgetall.return_value = { + '7fa95d07-0e2d-4bc5-b47c-448fdc1260b2': ( + '{"name": "editors", ' + '"admins": ["8ad942fe-490d-453e-bd37-56f252e41604", "rand"], ' + '"members": ["8ad942fe-490d-453e-bd37-56f252e41603", ' + '"rand"], ' + '"changed_timestamp": "Oct 06 2021 06:39PM", ' + '"created_timestamp": "Oct 06 2021 06:39PM"}')} + self.conn = conn + + def test_user_is_group_member_only(self): + """Test that a user is only a group member""" + self.assertEqual( + get_user_membership( + conn=self.conn, + user_id="8ad942fe-490d-453e-bd37-56f252e41603", + group_id="7fa95d07-0e2d-4bc5-b47c-448fdc1260b2"), + {"member": True, + "admin": False}) + + def test_user_is_group_admin_only(self): + """Test that a user is a group admin only""" + self.assertEqual( + get_user_membership( + conn=self.conn, + user_id="8ad942fe-490d-453e-bd37-56f252e41604", + group_id="7fa95d07-0e2d-4bc5-b47c-448fdc1260b2"), + {"member": False, + "admin": True}) + + def test_user_is_both_group_member_and_admin(self): + """Test that a user is both an admin and member of a group""" + self.assertEqual( + get_user_membership( + conn=self.conn, + user_id="rand", + group_id="7fa95d07-0e2d-4bc5-b47c-448fdc1260b2"), + {"member": True, + "admin": True}) + + +class TestCheckUserAccessRole(unittest.TestCase): + """Test cases for `get_highest_user_access_role`""" + + @mock.patch("wqflask.resource_manager.requests.get") + def test_edit_access(self, requests_mock): + """Test that the right access roles are set""" + response = mock.PropertyMock(return_value=json.dumps( + { + 'data': ['no-access', 'view', 'edit', ], + 'metadata': ['no-access', 'view', 'edit', ], + 'admin': ['not-admin', 'edit-access', ], + } + )) + type(requests_mock.return_value).content = response + self.assertEqual(get_highest_user_access_role( + resource_id="0196d92e1665091f202f", + user_id="8ad942fe-490d-453e-bd37"), + {"data": DataRole.EDIT, + "metadata": DataRole.EDIT, + "admin": AdminRole.EDIT_ACCESS}) + + @mock.patch("wqflask.resource_manager.requests.get") + def test_no_access(self, requests_mock): + response = mock.PropertyMock(return_value=json.dumps( + { + 'data': ['no-access', ], + 'metadata': ['no-access', ], + 'admin': ['not-admin', ], + } + )) + type(requests_mock.return_value).content = response + self.assertEqual(get_highest_user_access_role( + resource_id="0196d92e1665091f202f", + user_id=""), + {"data": DataRole.NO_ACCESS, + "metadata": DataRole.NO_ACCESS, + "admin": AdminRole.NOT_ADMIN}) diff --git a/wqflask/tests/unit/wqflask/wgcna/__init__.py b/wqflask/tests/unit/wqflask/wgcna/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/wqflask/tests/unit/wqflask/wgcna/__init__.py diff --git a/wqflask/tests/unit/wqflask/wgcna/test_wgcna.py b/wqflask/tests/unit/wqflask/wgcna/test_wgcna.py new file mode 100644 index 00000000..8e947e2f --- /dev/null +++ b/wqflask/tests/unit/wqflask/wgcna/test_wgcna.py @@ -0,0 +1,50 @@ + +"""module contains for processing gn3 wgcna data""" +from unittest import TestCase + +from wqflask.wgcna.gn3_wgcna import process_wgcna_data + + +class DataProcessingTests(TestCase): + """class contains data processing tests""" + + def test_data_processing(self): + """test for parsing data for datatable""" + output = { + "input": { + "sample_names": ["BXD1", "BXD2", "BXD3", "BXD4", "BXD5", "BXD6"], + + }, + "output": { + "ModEigens": { + "MEturquoise": [ + 0.0646677768085351, + 0.137200224277058, + 0.63451113720732, + -0.544002665501479, + -0.489487590361863, + 0.197111117570427 + ], + "MEgrey": [ + 0.213, + 0.214, + 0.3141, + -0.545, + -0.423, + 0.156, + ] + }}} + + row_data = [['BXD1', 0.065, 0.213], + ['BXD2', 0.137, 0.214], + ['BXD3', 0.635, 0.314], + ['BXD4', -0.544, -0.545], + ['BXD5', -0.489, -0.423], + ['BXD6', 0.197, 0.156]] + + expected_results = { + "col_names": ["sample_names", "MEturquoise", "MEgrey"], + "mod_dataset": row_data + } + + self.assertEqual(process_wgcna_data(output), expected_results) diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py index 6802d689..afea69e1 100644 --- a/wqflask/utility/authentication_tools.py +++ b/wqflask/utility/authentication_tools.py @@ -4,11 +4,12 @@ import requests from flask import g from base import webqtlConfig - from utility.redis_tools import (get_redis_conn, get_resource_info, get_resource_id, add_resource) +from utility.tools import GN_PROXY_URL + Redis = get_redis_conn() def check_resource_availability(dataset, trait_id=None): @@ -24,19 +25,19 @@ def check_resource_availability(dataset, trait_id=None): if resource_id: resource_info = get_resource_info(resource_id) - # ZS: If resource isn't already in redis, add it with default + # If resource isn't already in redis, add it with default # privileges if not resource_info: resource_info = add_new_resource(dataset, trait_id) - # ZS: Check if super-user - we should probably come up with some + # Check if super-user - we should probably come up with some # way to integrate this into the proxy if g.user_session.user_id in Redis.smembers("super_users"): return webqtlConfig.SUPER_PRIVILEGES response = None - the_url = "http://localhost:8080/available?resource={}&user={}".format( + the_url = GN_PROXY_URL + "available?resource={}&user={}".format( resource_id, g.user_session.user_id) try: @@ -93,7 +94,7 @@ def get_group_code(dataset): def check_admin(resource_id=None): - the_url = "http://localhost:8080/available?resource={}&user={}".format( + the_url = GN_PROXY_URL + "available?resource={}&user={}".format( resource_id, g.user_session.user_id) try: response = json.loads(requests.get(the_url).content)['admin'] diff --git a/wqflask/utility/chunks.py b/wqflask/utility/chunks.py index 484b5de6..f6e88cbe 100644 --- a/wqflask/utility/chunks.py +++ b/wqflask/utility/chunks.py @@ -1,5 +1,4 @@ import math -import time def divide_into_chunks(the_list, number_chunks): diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py deleted file mode 100644 index eae3ba03..00000000 --- a/wqflask/utility/elasticsearch_tools.py +++ /dev/null @@ -1,121 +0,0 @@ -# Elasticsearch support -# -# Some helpful commands to view the database: -# -# You can test the server being up with -# -# curl -H 'Content-Type: application/json' http://localhost:9200 -# -# List all indices -# -# curl -H 'Content-Type: application/json' 'localhost:9200/_cat/indices?v' -# -# To see the users index 'table' -# -# curl http://localhost:9200/users -# -# To list all user ids -# -# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d ' -# { -# "query" : { -# "match_all" : {} -# }, -# "stored_fields": [] -# }' -# -# To view a record -# -# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d ' -# { -# "query" : { -# "match" : { "email_address": "pjotr2017@thebird.nl"} -# } -# }' -# -# -# To delete the users index and data (dangerous!) -# -# curl -XDELETE -H 'Content-Type: application/json' 'localhost:9200/users' - - -from elasticsearch import Elasticsearch, TransportError -import logging - -from utility.logger import getLogger -logger = getLogger(__name__) - -from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT - - -def test_elasticsearch_connection(): - es = Elasticsearch(['http://' + ELASTICSEARCH_HOST + \ - ":" + str(ELASTICSEARCH_PORT) + '/'], verify_certs=True) - if not es.ping(): - logger.warning("Elasticsearch is DOWN") - - -def get_elasticsearch_connection(for_user=True): - """Return a connection to ES. Returns None on failure""" - logger.info("get_elasticsearch_connection") - es = None - try: - assert(ELASTICSEARCH_HOST) - assert(ELASTICSEARCH_PORT) - logger.info("ES HOST", ELASTICSEARCH_HOST) - - es = Elasticsearch([{ - "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT - }], timeout=30, retry_on_timeout=True) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None - - if for_user: - setup_users_index(es) - - es_logger = logging.getLogger("elasticsearch") - es_logger.setLevel(logging.INFO) - es_logger.addHandler(logging.NullHandler()) - except Exception as e: - logger.error("Failed to get elasticsearch connection", e) - es = None - - return es - - -def setup_users_index(es_connection): - if es_connection: - index_settings = { - "properties": { - "email_address": { - "type": "keyword"}}} - - es_connection.indices.create(index='users', ignore=400) - es_connection.indices.put_mapping( - body=index_settings, index="users", doc_type="local") - - -def get_user_by_unique_column(es, column_name, column_value, index="users", doc_type="local"): - return get_item_by_unique_column(es, column_name, column_value, index=index, doc_type=doc_type) - - -def save_user(es, user, user_id): - es_save_data(es, "users", "local", user, user_id) - - -def get_item_by_unique_column(es, column_name, column_value, index, doc_type): - item_details = None - try: - response = es.search( - index=index, doc_type=doc_type, body={ - "query": {"match": {column_name: column_value}} - }) - if len(response["hits"]["hits"]) > 0: - item_details = response["hits"]["hits"][0]["_source"] - except TransportError as te: - pass - return item_details - - -def es_save_data(es, index, doc_type, data_item, data_id,): - from time import sleep - es.create(index, doc_type, body=data_item, id=data_id) - sleep(1) # Delay 1 second to allow indexing diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py index de9dde46..a6c5875f 100644 --- a/wqflask/utility/redis_tools.py +++ b/wqflask/utility/redis_tools.py @@ -57,30 +57,6 @@ def get_user_by_unique_column(column_name, column_value): return item_details -def get_users_like_unique_column(column_name, column_value): - """Like previous function, but this only checks if the input is a - subset of a field and can return multiple results - - """ - matched_users = [] - - if column_value != "": - user_list = Redis.hgetall("users") - if column_name != "user_id": - for key in user_list: - user_ob = json.loads(user_list[key]) - if "user_id" not in user_ob: - set_user_attribute(key, "user_id", key) - user_ob["user_id"] = key - if column_name in user_ob: - if column_value in user_ob[column_name]: - matched_users.append(user_ob) - else: - matched_users.append(load_json_from_redis(user_list, column_value)) - - return matched_users - - def set_user_attribute(user_id, column_name, column_value): user_info = json.loads(Redis.hget("users", user_id)) user_info[column_name] = column_value @@ -165,52 +141,6 @@ def get_group_info(group_id): return group_info -def get_group_by_unique_column(column_name, column_value): - """ Get group by column; not sure if there's a faster way to do this """ - - matched_groups = [] - - all_group_list = Redis.hgetall("groups") - for key in all_group_list: - group_info = json.loads(all_group_list[key]) - # ZS: Since these fields are lists, search in the list - if column_name == "admins" or column_name == "members": - if column_value in group_info[column_name]: - matched_groups.append(group_info) - else: - if group_info[column_name] == column_value: - matched_groups.append(group_info) - - return matched_groups - - -def get_groups_like_unique_column(column_name, column_value): - """Like previous function, but this only checks if the input is a - subset of a field and can return multiple results - - """ - matched_groups = [] - - if column_value != "": - group_list = Redis.hgetall("groups") - if column_name != "group_id": - for key in group_list: - group_info = json.loads(group_list[key]) - # ZS: Since these fields are lists, search in the list - if column_name == "admins" or column_name == "members": - if column_value in group_info[column_name]: - matched_groups.append(group_info) - else: - if column_name in group_info: - if column_value in group_info[column_name]: - matched_groups.append(group_info) - else: - matched_groups.append( - load_json_from_redis(group_list, column_value)) - - return matched_groups - - def create_group(admin_user_ids, member_user_ids=[], group_name="Default Group Name"): group_id = str(uuid.uuid4()) @@ -352,11 +282,3 @@ def add_access_mask(resource_id, group_id, access_mask): Redis.hset("resources", resource_id, json.dumps(the_resource)) return the_resource - - -def change_resource_owner(resource_id, new_owner_id): - the_resource = get_resource_info(resource_id) - the_resource['owner_id'] = new_owner_id - - Redis.delete("resource") - Redis.hset("resources", resource_id, json.dumps(the_resource)) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index e28abb48..db0b4320 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -194,7 +194,6 @@ def locate(name, subdir=None): if valid_path(base): lookfor = base + "/" + name if valid_file(lookfor): - logger.info("Found: file " + lookfor + "\n") return lookfor else: raise Exception("Can not locate " + lookfor) @@ -220,9 +219,7 @@ def locate_ignore_error(name, subdir=None): if valid_path(base): lookfor = base + "/" + name if valid_file(lookfor): - logger.debug("Found: file " + name + "\n") return lookfor - logger.info("WARNING: file " + name + " not found\n") return None @@ -266,6 +263,8 @@ WEBSERVER_MODE = get_setting('WEBSERVER_MODE') GN2_BASE_URL = get_setting('GN2_BASE_URL') GN2_BRANCH_URL = get_setting('GN2_BRANCH_URL') GN_SERVER_URL = get_setting('GN_SERVER_URL') +GN_PROXY_URL = get_setting('GN_PROXY_URL') +GN3_LOCAL_URL = get_setting('GN3_LOCAL_URL') SERVER_PORT = get_setting_int('SERVER_PORT') SQL_URI = get_setting('SQL_URI') LOG_LEVEL = get_setting('LOG_LEVEL') @@ -285,6 +284,7 @@ JS_GN_PATH = get_setting('JS_GN_PATH') GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') +GITHUB_AUTH_URL = "" if GITHUB_CLIENT_ID != 'UNKNOWN' and GITHUB_CLIENT_SECRET: GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id=" + \ GITHUB_CLIENT_ID + "&client_secret=" + GITHUB_CLIENT_SECRET @@ -299,10 +299,6 @@ if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET: "&redirect_uri=" + GN2_BRANCH_URL + "n/login/orcid_oauth2" ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') -ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') -ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') -# import utility.elasticsearch_tools as es -# es.test_elasticsearch_connection() SMTP_CONNECT = get_setting('SMTP_CONNECT') SMTP_USERNAME = get_setting('SMTP_USERNAME') diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index 8b2055cd..05e040ed 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -8,14 +8,23 @@ from flask import Flask from typing import Tuple from urllib.parse import urlparse from utility import formatting -from wqflask.markdown_routes import glossary_blueprint -from wqflask.markdown_routes import references_blueprint -from wqflask.markdown_routes import links_blueprint -from wqflask.markdown_routes import policies_blueprint -from wqflask.markdown_routes import environments_blueprint -from wqflask.markdown_routes import facilities_blueprint -from wqflask.markdown_routes import blogs_blueprint -from wqflask.markdown_routes import news_blueprint + +from gn3.authentication import DataRole, AdminRole + +from wqflask.group_manager import group_management +from wqflask.resource_manager import resource_management +from wqflask.metadata_edits import metadata_edit + +from wqflask.api.markdown import glossary_blueprint +from wqflask.api.markdown import references_blueprint +from wqflask.api.markdown import links_blueprint +from wqflask.api.markdown import policies_blueprint +from wqflask.api.markdown import environments_blueprint +from wqflask.api.markdown import facilities_blueprint +from wqflask.api.markdown import blogs_blueprint +from wqflask.api.markdown import news_blueprint + +from wqflask.jupyter_notebooks import jupyter_notebooks app = Flask(__name__) @@ -54,7 +63,11 @@ app.register_blueprint(environments_blueprint, url_prefix="/environments") app.register_blueprint(facilities_blueprint, url_prefix="/facilities") app.register_blueprint(blogs_blueprint, url_prefix="/blogs") app.register_blueprint(news_blueprint, url_prefix="/news") +app.register_blueprint(jupyter_notebooks, url_prefix="/jupyter_notebooks") +app.register_blueprint(resource_management, url_prefix="/resource-management") +app.register_blueprint(metadata_edit, url_prefix="/datasets/") +app.register_blueprint(group_management, url_prefix="/group-management") @app.before_request def before_request(): @@ -62,6 +75,16 @@ def before_request(): g.request_time = lambda: "%.5fs" % (time.time() - g.request_start_time) +@app.context_processor +def include_admin_role_class(): + return {'AdminRole': AdminRole} + + +@app.context_processor +def include_data_role_class(): + return {'DataRole': DataRole} + + from wqflask.api import router from wqflask import group_manager from wqflask import resource_manager diff --git a/wqflask/wqflask/api/correlation.py b/wqflask/wqflask/api/correlation.py index 870f3275..9b875c99 100644 --- a/wqflask/wqflask/api/correlation.py +++ b/wqflask/wqflask/api/correlation.py @@ -1,23 +1,13 @@ import collections - import scipy -from utility.db_tools import escape - -from flask import g - from base import data_set from base.trait import create_trait, retrieve_sample_data - -from wqflask.correlation.show_corr_results import generate_corr_json +from flask import g +from utility import corr_result_helpers +from utility.db_tools import escape from wqflask.correlation import correlation_functions -from utility import webqtlUtil, helper_functions, corr_result_helpers -from utility.benchmark import Bench - -import utility.logger -logger = utility.logger.getLogger(__name__) - def do_correlation(start_vars): assert('db' in start_vars) @@ -35,7 +25,6 @@ def do_correlation(start_vars): corr_results = calculate_results( this_trait, this_dataset, target_dataset, corr_params) - #corr_results = collections.OrderedDict(sorted(corr_results.items(), key=lambda t: -abs(t[1][0]))) final_results = [] for _trait_counter, trait in enumerate(list(corr_results.keys())[:corr_params['return_count']]): @@ -63,11 +52,7 @@ def do_correlation(start_vars): "#_strains": num_overlap, "p_value": sample_p } - final_results.append(result_dict) - - # json_corr_results = generate_corr_json(final_corr_results, this_trait, this_dataset, target_dataset, for_api = True) - return final_results diff --git a/wqflask/wqflask/api/gen_menu.py b/wqflask/wqflask/api/gen_menu.py index a699a484..5d239343 100644 --- a/wqflask/wqflask/api/gen_menu.py +++ b/wqflask/wqflask/api/gen_menu.py @@ -1,4 +1,8 @@ from gn3.db.species import get_all_species + +import utility.logger +logger = utility.logger.getLogger(__name__) + def gen_dropdown_json(conn): """Generates and outputs (as json file) the data for the main dropdown menus on the home page @@ -19,16 +23,16 @@ def get_groups(species, conn): with conn.cursor() as cursor: for species_name, _species_full_name in species: groups[species_name] = [] - cursor.execute( - ("SELECT InbredSet.Name, InbredSet.FullName, " + query = ("SELECT InbredSet.Name, InbredSet.FullName, " "IFNULL(InbredSet.Family, 'None') " "FROM InbredSet, Species WHERE Species.Name = '{}' " "AND InbredSet.SpeciesId = Species.Id GROUP by " "InbredSet.Name ORDER BY IFNULL(InbredSet.FamilyOrder, " "InbredSet.FullName) ASC, IFNULL(InbredSet.Family, " "InbredSet.FullName) ASC, InbredSet.FullName ASC, " - "InbredSet.MenuOrderId ASC") - .format(species_name)) + "InbredSet.MenuOrderId ASC").format(species_name) + # logger.debug(query) + cursor.execute(query) results = cursor.fetchall() for result in results: family_name = "Family:" + str(result[2]) diff --git a/wqflask/wqflask/markdown_routes.py b/wqflask/wqflask/api/markdown.py index 580b9ac0..580b9ac0 100644 --- a/wqflask/wqflask/markdown_routes.py +++ b/wqflask/wqflask/api/markdown.py diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 01274ba9..76ef5ca4 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -12,6 +12,7 @@ from flask import flash from wqflask import app from utility import hmac from utility.formatting import numify +from utility.tools import GN_SERVER_URL from utility.redis_tools import get_redis_conn from base.trait import create_trait @@ -218,8 +219,10 @@ def view_collection(): json_version.append(jsonable(trait_ob)) - collection_info = dict(trait_obs=trait_obs, - uc=uc) + collection_info = dict( + trait_obs=trait_obs, + uc=uc, + heatmap_data_url=f"{GN_SERVER_URL}heatmaps/clustered") if "json" in params: return json.dumps(json_version) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index a18bceaf..c2acd648 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -1,14 +1,18 @@ """module that calls the gn3 api's to do the correlation """ import json +import time +from functools import wraps from wqflask.correlation import correlation_functions - +from wqflask.correlation.pre_computes import fetch_precompute_results +from wqflask.correlation.pre_computes import cache_compute_results from base import data_set from base.trait import create_trait from base.trait import retrieve_sample_data from gn3.computations.correlations import compute_all_sample_correlation +from gn3.computations.correlations import fast_compute_all_sample_correlation from gn3.computations.correlations import map_shared_keys_to_values from gn3.computations.correlations import compute_all_lit_correlation from gn3.computations.correlations import compute_tissue_correlation @@ -19,9 +23,11 @@ def create_target_this_trait(start_vars): """this function creates the required trait and target dataset for correlation""" if start_vars['dataset'] == "Temp": - this_dataset = data_set.create_dataset(dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) + this_dataset = data_set.create_dataset( + dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) else: - this_dataset = data_set.create_dataset(dataset_name=start_vars['dataset']) + this_dataset = data_set.create_dataset( + dataset_name=start_vars['dataset']) target_dataset = data_set.create_dataset( dataset_name=start_vars['corr_dataset']) this_trait = create_trait(dataset=this_dataset, @@ -58,13 +64,19 @@ def test_process_data(this_trait, dataset, start_vars): return sample_data -def process_samples(start_vars, sample_names, excluded_samples=None): - """process samples""" +def process_samples(start_vars, sample_names=[], excluded_samples=[]): + """code to fetch correct samples""" sample_data = {} - if not excluded_samples: - excluded_samples = () - sample_vals_dict = json.loads(start_vars["sample_vals"]) + sample_vals_dict = json.loads(start_vars["sample_vals"]) + if sample_names: for sample in sample_names: + if sample in sample_vals_dict and sample not in excluded_samples: + val = sample_vals_dict[sample] + if not val.strip().lower() == "x": + sample_data[str(sample)] = float(val) + + else: + for sample in sample_vals_dict.keys(): if sample not in excluded_samples: val = sample_vals_dict[sample] if not val.strip().lower() == "x": @@ -147,6 +159,18 @@ def lit_for_trait_list(corr_results, this_dataset, this_trait): def fetch_sample_data(start_vars, this_trait, this_dataset, target_dataset): + corr_samples_group = start_vars["corr_samples_group"] + if corr_samples_group == "samples_primary": + sample_data = process_samples( + start_vars, this_dataset.group.all_samples_ordered()) + + elif corr_samples_group == "samples_other": + sample_data = process_samples( + start_vars, excluded_samples=this_dataset.group.samplelist) + + else: + sample_data = process_samples(start_vars) + sample_data = process_samples( start_vars, this_dataset.group.all_samples_ordered()) @@ -187,9 +211,9 @@ def compute_correlation(start_vars, method="pearson", compute_all=False): if corr_type == "sample": (this_trait_data, target_dataset_data) = fetch_sample_data( start_vars, this_trait, this_dataset, target_dataset) - correlation_results = compute_all_sample_correlation(corr_method=method, - this_trait=this_trait_data, - target_dataset=target_dataset_data) + + correlation_results = compute_all_sample_correlation( + corr_method=method, this_trait=this_trait_data, target_dataset=target_dataset_data) elif corr_type == "tissue": trait_symbol_dict = this_dataset.retrieve_genes("Symbol") @@ -290,7 +314,8 @@ def get_tissue_correlation_input(this_trait, trait_symbol_dict): """Gets tissue expression values for the primary trait and target tissues values""" primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( symbol_list=[this_trait.symbol]) - if this_trait.symbol.lower() in primary_trait_tissue_vals_dict: + if this_trait.symbol and this_trait.symbol.lower() in primary_trait_tissue_vals_dict: + primary_trait_tissue_values = primary_trait_tissue_vals_dict[this_trait.symbol.lower( )] corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py new file mode 100644 index 00000000..975a53b8 --- /dev/null +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -0,0 +1,158 @@ +import json +import os +import hashlib +from pathlib import Path + +from base.data_set import query_table_timestamp +from base.webqtlConfig import TMPDIR + + +def fetch_all_cached_metadata(dataset_name): + """in a gvein dataset fetch all the traits metadata""" + file_name = generate_filename(dataset_name, suffix="metadata") + + file_path = os.path.join(TMPDIR, file_name) + + try: + with open(file_path, "r+") as file_handler: + dataset_metadata = json.load(file_handler) + return (file_path, dataset_metadata) + + except FileNotFoundError: + Path(file_path).touch(exist_ok=True) + return (file_path, {}) + + +def cache_new_traits_metadata(dataset_metadata: dict, new_traits_metadata, file_path: str): + """function to cache the new traits metadata""" + + if bool(new_traits_metadata): + dataset_metadata.update(new_traits_metadata) + + with open(file_path, "w+") as file_handler: + json.dump(dataset_metadata, file_handler) + + +def generate_filename(*args, suffix="", file_ext="json"): + """given a list of args generate a unique filename""" + + string_unicode = f"{*args,}".encode() + return f"{hashlib.md5(string_unicode).hexdigest()}_{suffix}.{file_ext}" + + +def cache_compute_results(base_dataset_type, + base_dataset_name, + target_dataset_name, + corr_method, + correlation_results, + trait_name): + """function to cache correlation results for heavy computations""" + + base_timestamp = query_table_timestamp(base_dataset_type) + + target_dataset_timestamp = base_timestamp + + file_name = generate_filename( + base_dataset_name, target_dataset_name, + base_timestamp, target_dataset_timestamp, + suffix="corr_precomputes") + + file_path = os.path.join(TMPDIR, file_name) + + try: + with open(file_path, "r+") as json_file_handler: + data = json.load(json_file_handler) + + data[trait_name] = correlation_results + + json_file_handler.seek(0) + + json.dump(data, json_file_handler) + + json_file_handler.truncate() + + except FileNotFoundError: + with open(file_path, "w+") as file_handler: + data = {} + data[trait_name] = correlation_results + + json.dump(data, file_handler) + + +def fetch_precompute_results(base_dataset_name, + target_dataset_name, + dataset_type, + trait_name): + """function to check for precomputed results""" + + base_timestamp = target_dataset_timestamp = query_table_timestamp( + dataset_type) + file_name = generate_filename( + base_dataset_name, target_dataset_name, + base_timestamp, target_dataset_timestamp, + suffix="corr_precomputes") + + file_path = os.path.join(TMPDIR, file_name) + + try: + with open(file_path, "r+") as json_handler: + correlation_results = json.load(json_handler) + + return correlation_results.get(trait_name) + + except FileNotFoundError: + pass + + +def pre_compute_dataset_vs_dataset(base_dataset, + target_dataset, + corr_method): + """compute sample correlation between dataset vs dataset + wn:heavy function should be invoked less frequently + input:datasets_data(two dicts),corr_method + + output:correlation results for entire dataset against entire dataset + """ + dataset_correlation_results = {} + + target_traits_data, base_traits_data = get_datasets_data( + base_dataset, target_dataset_data) + + for (primary_trait_name, strain_values) in base_traits_data: + + this_trait_data = { + "trait_sample_data": strain_values, + "trait_id": primary_trait_name + } + + trait_correlation_result = compute_all_sample_correlation( + corr_method=corr_method, + this_trait=this_trait_data, + target_dataset=target_traits_data) + + dataset_correlation_results[primary_trait_name] = trait_correlation_result + + return dataset_correlation_results + + +def get_datasets_data(base_dataset, target_dataset_data): + """required to pass data in a given format to the pre compute + function + + (works for bxd only probeset datasets) + + output:two dicts for datasets with key==trait and value==strains + """ + samples_fetched = base_dataset.group.all_samples_ordered() + target_traits_data = target_dataset.get_trait_data( + samples_fetched) + + base_traits_data = base_dataset.get_trait_data( + samples_fetched) + + target_results = map_shared_keys_to_values( + samples_fetched, target_traits_data) + base_results = map_shared_keys_to_values( + samples_fetched, base_traits_data) + + return (target_results, base_results) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index d73965da..1c391386 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -19,10 +19,17 @@ # This module is used by GeneNetwork project (www.genenetwork.org) import json +import os +from pathlib import Path from base.trait import create_trait, jsonable from base.data_set import create_dataset +from base.webqtlConfig import TMPDIR +from wqflask.correlation.pre_computes import fetch_all_cached_metadata +from wqflask.correlation.pre_computes import cache_new_traits_metadata + +from utility.authentication_tools import check_resource_availability from utility import hmac @@ -31,7 +38,8 @@ def set_template_vars(start_vars, correlation_data): corr_method = start_vars['corr_sample_method'] if start_vars['dataset'] == "Temp": - this_dataset_ob = create_dataset(dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) + this_dataset_ob = create_dataset( + dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) else: this_dataset_ob = create_dataset(dataset_name=start_vars['dataset']) this_trait = create_trait(dataset=this_dataset_ob, @@ -82,13 +90,29 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe corr_results = correlation_data['correlation_results'] results_list = [] + + new_traits_metadata = {} + + (file_path, dataset_metadata) = fetch_all_cached_metadata( + target_dataset['name']) + for i, trait_dict in enumerate(corr_results): trait_name = list(trait_dict.keys())[0] trait = trait_dict[trait_name] - target_trait_ob = create_trait(dataset=target_dataset_ob, - name=trait_name, - get_qtl_info=True) - target_trait = jsonable(target_trait_ob, target_dataset_ob) + + target_trait = dataset_metadata.get(trait_name) + if target_trait is None: + target_trait_ob = create_trait(dataset=target_dataset_ob, + name=trait_name, + get_qtl_info=True) + target_trait = jsonable(target_trait_ob, target_dataset_ob) + new_traits_metadata[trait_name] = target_trait + else: + if target_dataset['type'] == "Publish": + permissions = check_resource_availability(target_dataset_ob, trait_name) + if permissions['metadata'] == "no-access": + continue + if target_trait['view'] == False: continue results_dict = {} @@ -163,6 +187,10 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe results_list.append(results_dict) + cache_new_traits_metadata(dataset_metadata, + new_traits_metadata, + file_path) + return json.dumps(results_list) diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py index 54aa6795..41d23084 100644 --- a/wqflask/wqflask/decorators.py +++ b/wqflask/wqflask/decorators.py @@ -1,36 +1,80 @@ """This module contains gn2 decorators""" -from flask import g +import redis + +from flask import current_app, g, redirect, request, url_for from typing import Dict +from urllib.parse import urljoin from functools import wraps -from utility.hmac import hmac_creation +from gn3.authentication import AdminRole +from gn3.authentication import DataRole import json import requests +def login_required(f): + """Use this for endpoints where login is required""" + @wraps(f) + def wrap(*args, **kwargs): + user_id = ((g.user_session.record.get(b"user_id") or + b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") + redis_conn = redis.from_url(current_app.config["REDIS_URL"], + decode_responses=True) + if not redis_conn.hget("users", user_id): + return "You need to be logged in!", 401 + return f(*args, **kwargs) + return wrap + + def edit_access_required(f): - """Use this for endpoints where admins are required""" + """Use this for endpoints where people with admin or edit privileges +are required""" @wraps(f) def wrap(*args, **kwargs): resource_id: str = "" - if kwargs.get("inbredset_id"): # data type: dataset-publish - resource_id = hmac_creation("dataset-publish:" - f"{kwargs.get('inbredset_id')}:" - f"{kwargs.get('name')}") - if kwargs.get("dataset_name"): # data type: dataset-probe - resource_id = hmac_creation("dataset-probeset:" - f"{kwargs.get('dataset_name')}") + if request.args.get("resource-id"): + resource_id = request.args.get("resource-id") + elif kwargs.get("resource_id"): + resource_id = kwargs.get("resource_id") + response: Dict = {} + try: + user_id = ((g.user_session.record.get(b"user_id") or + b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") + response = json.loads( + requests.get(urljoin( + current_app.config.get("GN2_PROXY"), + ("available?resource=" + f"{resource_id}&user={user_id}"))).content) + except: + response = {} + if max([DataRole(role) for role in response.get( + "data", ["no-access"])]) < DataRole.EDIT: + return redirect(url_for("no_access_page")) + return f(*args, **kwargs) + return wrap + + +def edit_admins_access_required(f): + """Use this for endpoints where ownership of a resource is required""" + @wraps(f) + def wrap(*args, **kwargs): + resource_id: str = kwargs.get("resource_id", "") response: Dict = {} try: - _user_id = g.user_session.record.get(b"user_id", - "").decode("utf-8") + user_id = ((g.user_session.record.get(b"user_id") or + b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") response = json.loads( - requests.get("http://localhost:8080/" - "available?resource=" - f"{resource_id}&user={_user_id}").content) + requests.get(urljoin( + current_app.config.get("GN2_PROXY"), + ("available?resource=" + f"{resource_id}&user={user_id}"))).content) except: response = {} - if "edit" not in response.get("data", []): - return "You need to be admin", 401 + if max([AdminRole(role) for role in response.get( + "admin", ["not-admin"])]) < AdminRole.EDIT_ADMINS: + return redirect(url_for("no_access_page")) return f(*args, **kwargs) return wrap diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 761ae326..99272ee3 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -81,16 +81,31 @@ class MrnaAssaySearch(DoSearch): DoSearch.search_types['ProbeSet'] = "MrnaAssaySearch" - base_query = """SELECT distinct ProbeSet.Name as TNAME, - 0 as thistable, - ProbeSetXRef.Mean as TMEAN, - ProbeSetXRef.LRS as TLRS, - ProbeSetXRef.PVALUE as TPVALUE, - ProbeSet.Chr_num as TCHR_NUM, - ProbeSet.Mb as TMB, - ProbeSet.Symbol as TSYMBOL, - ProbeSet.name_num as TNAME_NUM - FROM ProbeSetXRef, ProbeSet """ + base_query = """ + SELECT + ProbeSetFreeze.`Name`, + ProbeSetFreeze.`FullName`, + ProbeSet.`Name`, + ProbeSet.`Symbol`, + CAST(ProbeSet.`description` AS BINARY), + CAST(ProbeSet.`Probe_Target_Description` AS BINARY), + ProbeSet.`Chr`, + ProbeSet.`Mb`, + ProbeSetXRef.`Mean`, + ProbeSetXRef.`LRS`, + ProbeSetXRef.`Locus`, + ProbeSetXRef.`pValue`, + ProbeSetXRef.`additive`, + Geno.`Chr` as geno_chr, + Geno.`Mb` as geno_mb + FROM Species + INNER JOIN InbredSet ON InbredSet.`SpeciesId`= Species.`Id` + INNER JOIN ProbeFreeze ON ProbeFreeze.`InbredSetId` = InbredSet.`Id` + INNER JOIN Tissue ON ProbeFreeze.`TissueId` = Tissue.`Id` + INNER JOIN ProbeSetFreeze ON ProbeSetFreeze.`ProbeFreezeId` = ProbeFreeze.`Id` + INNER JOIN ProbeSetXRef ON ProbeSetXRef.`ProbeSetFreezeId` = ProbeSetFreeze.`Id` + INNER JOIN ProbeSet ON ProbeSet.`Id` = ProbeSetXRef.`ProbeSetId` + LEFT JOIN Geno ON ProbeSetXRef.`Locus` = Geno.`Name` AND Geno.`SpeciesId` = Species.`Id` """ header_fields = ['Index', 'Record', @@ -193,10 +208,25 @@ class PhenotypeSearch(DoSearch): DoSearch.search_types['Publish'] = "PhenotypeSearch" base_query = """SELECT PublishXRef.Id, - PublishFreeze.createtime as thistable, - Publication.PubMed_ID as Publication_PubMed_ID, - Phenotype.Post_publication_description as Phenotype_Name - FROM Phenotype, PublishFreeze, Publication, PublishXRef """ + CAST(Phenotype.`Pre_publication_description` AS BINARY), + CAST(Phenotype.`Post_publication_description` AS BINARY), + Publication.`Authors`, + Publication.`Year`, + Publication.`PubMed_ID`, + PublishXRef.`mean`, + PublishXRef.`LRS`, + PublishXRef.`additive`, + PublishXRef.`Locus`, + InbredSet.`InbredSetCode`, + Geno.`Chr`, + Geno.`Mb` + FROM Species + INNER JOIN InbredSet ON InbredSet.`SpeciesId` = Species.`Id` + INNER JOIN PublishXRef ON PublishXRef.`InbredSetId` = InbredSet.`Id` + INNER JOIN PublishFreeze ON PublishFreeze.`InbredSetId` = InbredSet.`Id` + INNER JOIN Publication ON Publication.`Id` = PublishXRef.`PublicationId` + INNER JOIN Phenotype ON Phenotype.`Id` = PublishXRef.`PhenotypeId` + LEFT JOIN Geno ON PublishXRef.Locus = Geno.Name AND Geno.SpeciesId = Species.Id """ search_fields = ('Phenotype.Post_publication_description', 'Phenotype.Pre_publication_description', @@ -382,12 +412,10 @@ class RifSearch(MrnaAssaySearch): DoSearch.search_types['ProbeSet_RIF'] = "RifSearch" def get_from_clause(self): - return ", GeneRIF_BASIC " + return f" INNER JOIN GeneRIF_BASIC ON GeneRIF_BASIC.`symbol` = { self.dataset.type }.`symbol` " def get_where_clause(self): - where_clause = """( %s.symbol = GeneRIF_BASIC.symbol and - MATCH (GeneRIF_BASIC.comment) - AGAINST ('+%s' IN BOOLEAN MODE)) """ % (self.dataset.type, self.search_term[0]) + where_clause = f"(MATCH (GeneRIF_BASIC.comment) AGAINST ('+{ self.search_term[0] }' IN BOOLEAN MODE)) " return where_clause @@ -487,10 +515,7 @@ class LrsSearch(DoSearch): self.search_term = converted_search_term - if len(self.search_term) > 2: - from_clause = ", Geno" - else: - from_clause = "" + from_clause = "" return from_clause diff --git a/wqflask/wqflask/export_traits.py b/wqflask/wqflask/export_traits.py index 5459dc31..f4e04a4b 100644 --- a/wqflask/wqflask/export_traits.py +++ b/wqflask/wqflask/export_traits.py @@ -48,7 +48,7 @@ def export_search_results_csv(targs): if targs['filter_term'] != "None": metadata.append(["Search Filter Terms: " + targs['filter_term']]) metadata.append(["Exported Row Number: " + str(len(table_rows))]) - metadata.append(["Funding for The GeneNetwork: NIGMS (R01 GM123489, 2017-2021), NIDA (P30 DA044223, 2017-2022), NIA (R01AG043930, 2013-2018), NIAAA (U01 AA016662, U01 AA013499, U24 AA013513, U01 AA014425, 2006-2017), NIDA/NIMH/NIAAA (P20-DA 21131, 2001-2012), NCI MMHCC (U01CA105417), NCRR/BIRN (U24 RR021760)"]) + metadata.append(["Funding for The GeneNetwork: NIGMS (R01 GM123489, 2017-2026), NIDA (P30 DA044223, 2017-2022), NIA (R01AG043930, 2013-2018), NIAAA (U01 AA016662, U01 AA013499, U24 AA013513, U01 AA014425, 2006-2017), NIDA/NIMH/NIAAA (P20-DA 21131, 2001-2012), NCI MMHCC (U01CA105417), NCRR/BIRN (U24 RR021760)"]) metadata.append([]) trait_list = [] diff --git a/wqflask/wqflask/group_manager.py b/wqflask/wqflask/group_manager.py index 04a100ba..3936e36e 100644 --- a/wqflask/wqflask/group_manager.py +++ b/wqflask/wqflask/group_manager.py @@ -1,175 +1,157 @@ -import random -import string - -from flask import (Flask, g, render_template, url_for, request, make_response, - redirect, flash) - -from wqflask import app -from wqflask.user_login import send_verification_email, send_invitation_email, basic_info, set_password - -from utility.redis_tools import get_user_groups, get_group_info, save_user, create_group, delete_group, add_users_to_group, remove_users_from_group, \ - change_group_name, save_verification_code, check_verification_code, get_user_by_unique_column, get_resources, get_resource_info - -from utility.logger import getLogger -logger = getLogger(__name__) - - -@app.route("/groups/manage", methods=('GET', 'POST')) -def manage_groups(): - params = request.form if request.form else request.args - if "add_new_group" in params: - return redirect(url_for('add_group')) - else: - admin_groups, member_groups = get_user_groups(g.user_session.user_id) - return render_template("admin/group_manager.html", admin_groups=admin_groups, member_groups=member_groups) - - -@app.route("/groups/view", methods=('GET', 'POST')) -def view_group(): - params = request.form if request.form else request.args - group_id = params['id'] - group_info = get_group_info(group_id) - admins_info = [] - user_is_admin = False - if g.user_session.user_id in group_info['admins']: - user_is_admin = True - for user_id in group_info['admins']: - if user_id: - user_info = get_user_by_unique_column("user_id", user_id) - admins_info.append(user_info) - members_info = [] - for user_id in group_info['members']: - if user_id: - user_info = get_user_by_unique_column("user_id", user_id) - members_info.append(user_info) - - # ZS: This whole part might not scale well with many resources - resources_info = [] - all_resources = get_resources() - for resource_id in all_resources: - resource_info = get_resource_info(resource_id) - group_masks = resource_info['group_masks'] - if group_id in group_masks: - this_resource = {} - privileges = group_masks[group_id] - this_resource['id'] = resource_id - this_resource['name'] = resource_info['name'] - this_resource['data'] = privileges['data'] - this_resource['metadata'] = privileges['metadata'] - this_resource['admin'] = privileges['admin'] - resources_info.append(this_resource) - - return render_template("admin/view_group.html", group_info=group_info, admins=admins_info, members=members_info, user_is_admin=user_is_admin, resources=resources_info) - - -@app.route("/groups/remove", methods=('POST',)) -def remove_groups(): - group_ids_to_remove = request.form['selected_group_ids'] - for group_id in group_ids_to_remove.split(":"): - delete_group(g.user_session.user_id, group_id) - - return redirect(url_for('manage_groups')) - - -@app.route("/groups/remove_users", methods=('POST',)) -def remove_users(): - group_id = request.form['group_id'] - admin_ids_to_remove = request.form['selected_admin_ids'] - member_ids_to_remove = request.form['selected_member_ids'] - - remove_users_from_group(g.user_session.user_id, admin_ids_to_remove.split( - ":"), group_id, user_type="admins") - remove_users_from_group(g.user_session.user_id, member_ids_to_remove.split( - ":"), group_id, user_type="members") - - return redirect(url_for('view_group', id=group_id)) - - -@app.route("/groups/add_<path:user_type>", methods=('POST',)) -def add_users(user_type='members'): - group_id = request.form['group_id'] - if user_type == "admins": - user_emails = request.form['admin_emails_to_add'].split(",") - add_users_to_group(g.user_session.user_id, group_id, - user_emails, admins=True) - elif user_type == "members": - user_emails = request.form['member_emails_to_add'].split(",") - add_users_to_group(g.user_session.user_id, group_id, - user_emails, admins=False) - - return redirect(url_for('view_group', id=group_id)) - - -@app.route("/groups/change_name", methods=('POST',)) -def change_name(): - group_id = request.form['group_id'] - new_name = request.form['new_name'] - group_info = change_group_name(g.user_session.user_id, group_id, new_name) - - return new_name - - -@app.route("/groups/create", methods=('GET', 'POST')) -def add_or_edit_group(): - params = request.form if request.form else request.args - if "group_name" in params: - member_user_ids = set() - admin_user_ids = set() - # ZS: Always add the user creating the group as an admin - admin_user_ids.add(g.user_session.user_id) - if "admin_emails_to_add" in params: - admin_emails = params['admin_emails_to_add'].split(",") - for email in admin_emails: - user_details = get_user_by_unique_column( - "email_address", email) - if user_details: - admin_user_ids.add(user_details['user_id']) - #send_group_invites(params['group_id'], user_email_list = admin_emails, user_type="admins") - if "member_emails_to_add" in params: - member_emails = params['member_emails_to_add'].split(",") - for email in member_emails: - user_details = get_user_by_unique_column( - "email_address", email) - if user_details: - member_user_ids.add(user_details['user_id']) - #send_group_invites(params['group_id'], user_email_list = user_emails, user_type="members") - - create_group(list(admin_user_ids), list( - member_user_ids), params['group_name']) - return redirect(url_for('manage_groups')) - else: - return render_template("admin/create_group.html") - -# ZS: Will integrate this later, for now just letting users be added directly - - -def send_group_invites(group_id, user_email_list=[], user_type="members"): - for user_email in user_email_list: - user_details = get_user_by_unique_column("email_address", user_email) - if user_details: - group_info = get_group_info(group_id) - # ZS: Probably not necessary since the group should normally always exist if group_id is being passed here, - # but it's technically possible to hit it if Redis is cleared out before submitting the new users or something - if group_info: - # ZS: Don't add user if they're already an admin or if they're being added a regular user and are already a regular user, - # but do add them if they're a regular user and are added as an admin - if (user_details['user_id'] in group_info['admins']) or \ - ((user_type == "members") and (user_details['user_id'] in group_info['members'])): - continue - else: - send_verification_email(user_details, template_name="email/group_verification.txt", - key_prefix="verification_code", subject="You've been invited to join a GeneNetwork user group") - else: - temp_password = ''.join(random.choice( - string.ascii_uppercase + string.digits) for _ in range(6)) - user_details = { - 'user_id': str(uuid.uuid4()), - 'email_address': user_email, - 'registration_info': basic_info(), - 'password': set_password(temp_password), - 'confirmed': 0 - } - save_user(user_details, user_details['user_id']) - send_invitation_email(user_email, temp_password) - -# @app.route() +import json +import redis +import datetime + +from flask import current_app +from flask import Blueprint +from flask import g +from flask import render_template +from flask import request +from flask import redirect +from flask import url_for +from gn3.authentication import get_groups_by_user_uid +from gn3.authentication import get_user_info_by_key +from gn3.authentication import create_group +from wqflask.decorators import login_required + +group_management = Blueprint("group_management", __name__) + + +@group_management.route("/groups") +@login_required +def display_groups(): + groups = get_groups_by_user_uid( + user_uid=(g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")), + conn=redis.from_url( + current_app.config["REDIS_URL"], + decode_responses=True)) + return render_template("admin/group_manager.html", + admin_groups=groups.get("admin"), + member_groups=groups.get("member")) + + +@group_management.route("/groups/create", methods=("GET",)) +@login_required +def view_create_group_page(): + return render_template("admin/create_group.html") + + +@group_management.route("/groups/create", methods=("POST",)) +@login_required +def create_new_group(): + conn = redis.from_url(current_app.config["REDIS_URL"], + decode_responses=True) + if group_name := request.form.get("group_name"): + members_uid, admins_uid = set(), set() + admins_uid.add(user_uid := ( + g.user_session.record.get( + b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", ""))) + if admin_string := request.form.get("admin_emails_to_add"): + for email in admin_string.split(","): + user_info = get_user_info_by_key(key="email_address", + value=email, + conn=conn) + if user_uid := user_info.get("user_id"): + admins_uid.add(user_uid) + if member_string := request.form.get("member_emails_to_add"): + for email in member_string.split(","): + user_info = get_user_info_by_key(key="email_address", + value=email, + conn=conn) + if user_uid := user_info.get("user_id"): + members_uid.add(user_uid) + + # Create the new group: + create_group(conn=conn, + group_name=group_name, + member_user_uids=list(members_uid), + admin_user_uids=list(admins_uid)) + return redirect(url_for('group_management.display_groups')) + return redirect(url_for('group_management.create_groups')) + + +@group_management.route("/groups/delete", methods=("POST",)) +@login_required +def delete_groups(): + conn = redis.from_url(current_app.config["REDIS_URL"], + decode_responses=True) + user_uid = (g.user_session.record.get(b"user_id", b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) + current_app.logger.info(request.form.get("selected_group_ids")) + for group_uid in request.form.get("selected_group_ids", "").split(":"): + if group_info := conn.hget("groups", group_uid): + group_info = json.loads(group_info) + # A user who is an admin can delete things + if user_uid in group_info.get("admins"): + conn.hdel("groups", group_uid) + return redirect(url_for('group_management.display_groups')) + + +@group_management.route("/groups/<group_id>") +@login_required +def view_group(group_id: str): + conn = redis.from_url(current_app.config["REDIS_URL"], + decode_responses=True) + user_uid = (g.user_session.record.get(b"user_id", b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) + + resource_info = [] + for resource_uid, resource in conn.hgetall("resources").items(): + resource = json.loads(resource) + if group_id in (group_mask := resource.get("group_masks")): + __dict = {} + for val in group_mask.values(): + __dict.update(val) + __dict.update({ + "id": resource_uid, + "name": resource.get("name"), + }) + resource_info.append(__dict) + group_info = json.loads(conn.hget("groups", + group_id)) + group_info["guid"] = group_id + + return render_template( + "admin/view_group.html", + group_info=group_info, + admins=[get_user_info_by_key(key="user_id", + value=user_id, + conn=conn) + for user_id in group_info.get("admins")], + members=[get_user_info_by_key(key="user_id", + value=user_id, + conn=conn) + for user_id in group_info.get("members")], + is_admin = (True if user_uid in group_info.get("admins") else False), + resources=resource_info) + + +@group_management.route("/groups/<group_id>", methods=("POST",)) +def update_group(group_id: str): + conn = redis.from_url(current_app.config["REDIS_URL"], + decode_responses=True) + user_uid = (g.user_session.record.get(b"user_id", b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) + group = json.loads(conn.hget("groups", group_id)) + timestamp = group["changed_timestamp"] + timestamp_ = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + if user_uid in group.get("admins"): + if name := request.form.get("new_name"): + group["name"] = name + group["changed_timestamp"] = timestamp_ + if admins := request.form.get("admin_emails_to_add"): + group["admins"] = list(set(admins.split(":") + + group.get("admins"))) + group["changed_timestamp"] = timestamp_ + if members := request.form.get("member_emails_to_add"): + print(f"\n+++++\n{members}\n+++++\n") + group["members"] = list(set(members.split(":") + + group.get("members"))) + group["changed_timestamp"] = timestamp_ + conn.hset("groups", group_id, json.dumps(group)) + return redirect(url_for('group_management.view_group', + group_id=group_id)) diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index 2516e4fb..53a124d0 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -82,13 +82,14 @@ class GSearch: this_trait['species'] = line[0] this_trait['group'] = line[1] this_trait['tissue'] = line[2] - this_trait['symbol'] = line[6] + this_trait['symbol'] = "N/A" + if line[6]: + this_trait['symbol'] = line[6] + this_trait['description'] = "N/A" if line[7]: this_trait['description'] = line[7].decode( 'utf-8', 'replace') - else: - this_trait['description'] = "N/A" - this_trait['location_repr'] = 'N/A' + this_trait['location_repr'] = "N/A" if (line[8] != "NULL" and line[8] != "") and (line[9] != 0): this_trait['location_repr'] = 'Chr%s: %.6f' % ( line[8], float(line[9])) @@ -118,7 +119,7 @@ class GSearch: this_trait['dataset_id'] = line[15] dataset_ob = SimpleNamespace( - id=this_trait["dataset_id"], type="ProbeSet", species=this_trait["species"]) + id=this_trait["dataset_id"], type="ProbeSet", name=this_trait["dataset"], species=this_trait["species"]) if dataset_ob.id not in dataset_to_permissions: permissions = check_resource_availability(dataset_ob) dataset_to_permissions[dataset_ob.id] = permissions diff --git a/wqflask/wqflask/jupyter_notebooks.py b/wqflask/wqflask/jupyter_notebooks.py new file mode 100644 index 00000000..7d76828e --- /dev/null +++ b/wqflask/wqflask/jupyter_notebooks.py @@ -0,0 +1,17 @@ +from flask import Blueprint, render_template + +jupyter_notebooks = Blueprint('jupyter_notebooks', __name__) + +@jupyter_notebooks.route("/launcher", methods=("GET",)) +def launcher(): + links = ( + { + "main_url": "http://notebook.genenetwork.org/34301/notebooks/genenetwork-api-using-r.ipynb", + "notebook_name": "R notebook showing how to query the GeneNetwork API.", + "src_link_url": "https://github.com/jgarte/genenetwork-api-r-jupyter-notebook"}, + { + "main_url": "http://notebook.genenetwork.org/57675/notebooks/genenetwork.ipynb", + "notebook_name": "Querying the GeneNetwork API declaratively with python.", + "src_link_url": "https://github.com/jgarte/genenetwork-jupyter-notebook-example"}) + + return render_template("jupyter_notebooks.html", links=links) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 6254b9b9..920a8d30 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -2471,12 +2471,6 @@ class DisplayMappingResults: thisLRSColor = self.colorCollection[0] if qtlresult['chr'] != previous_chr and self.selectedChr == -1: if self.manhattan_plot != True: - # im_drawer.polygon( - # xy=LRSCoordXY, - # outline=thisLRSColor - # # , closed=0, edgeWidth=lrsEdgeWidth, - # # clipX=(xLeftOffset, xLeftOffset + plotWidth) - # ) draw_open_polygon(canvas, xy=LRSCoordXY, outline=thisLRSColor, width=lrsEdgeWidth) @@ -2497,25 +2491,21 @@ class DisplayMappingResults: im_drawer.line( xy=((Xc0, Yc0), (Xcm, yZero)), fill=plusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) im_drawer.line( xy=((Xcm, yZero), (Xc, yZero - (Yc - yZero))), fill=minusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) else: im_drawer.line( xy=((Xc0, yZero - (Yc0 - yZero)), (Xcm, yZero)), fill=minusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) im_drawer.line( xy=((Xcm, yZero), (Xc, Yc)), fill=plusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) elif (Yc0 - yZero) * (Yc - yZero) > 0: if Yc < yZero: @@ -2523,14 +2513,12 @@ class DisplayMappingResults: xy=((Xc0, Yc0), (Xc, Yc)), fill=plusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) else: im_drawer.line( xy=((Xc0, yZero - (Yc0 - yZero)), (Xc, yZero - (Yc - yZero))), fill=minusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) else: minYc = min(Yc - yZero, Yc0 - yZero) @@ -2538,14 +2526,12 @@ class DisplayMappingResults: im_drawer.line( xy=((Xc0, Yc0), (Xc, Yc)), fill=plusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) else: im_drawer.line( xy=((Xc0, yZero - (Yc0 - yZero)), (Xc, yZero - (Yc - yZero))), fill=minusColor, width=lineWidth - # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) LRSCoordXY = [] @@ -2558,28 +2544,29 @@ class DisplayMappingResults: startPosX += newStartPosX oldStartPosX = newStartPosX - # ZS: This is because the chromosome value stored in qtlresult['chr'] can be (for example) either X or 20 depending upon the mapping method/scale used + # This is because the chromosome value stored in qtlresult['chr'] can be (for example) either X or 20 depending upon the mapping method/scale used this_chr = str(self.ChrList[self.selectedChr][0]) if self.plotScale != "physic": this_chr = str(self.ChrList[self.selectedChr][1] + 1) if self.selectedChr == -1 or str(qtlresult['chr']) == this_chr: if self.plotScale != "physic" and self.mapping_method == "reaper" and not self.manhattan_plot: - Xc = startPosX + (qtlresult['cM'] - startMb) * plotXScale + start_cm = self.genotype[self.selectedChr - 1][0].cM + Xc = startPosX + (qtlresult['cM'] - start_cm) * plotXScale if hasattr(self.genotype, "filler"): if self.genotype.filler: if self.selectedChr != -1: - start_cm = self.genotype[self.selectedChr - 1][0].cM Xc = startPosX + \ (qtlresult['Mb'] - start_cm) * plotXScale else: - start_cm = self.genotype[previous_chr_as_int][0].cM Xc = startPosX + ((qtlresult['Mb'] - start_cm - startMb) * plotXScale) * ( ((qtlresult['Mb'] - start_cm - startMb) * plotXScale) / ((qtlresult['Mb'] - start_cm - startMb + self.GraphInterval) * plotXScale)) else: if self.selectedChr != -1 and qtlresult['Mb'] > endMb: Xc = startPosX + endMb * plotXScale else: + if qtlresult['Mb'] - startMb < 0: + continue Xc = startPosX + (qtlresult['Mb'] - startMb) * plotXScale # updated by NL 06-18-2011: @@ -2646,9 +2633,8 @@ class DisplayMappingResults: AdditiveHeightThresh / additiveMax AdditiveCoordXY.append((Xc, Yc)) - if self.selectedChr != -1 and qtlresult['Mb'] > endMb: + if self.selectedChr != -1 and qtlresult['Mb'] > endMb and endMb != -1: break - m += 1 if self.manhattan_plot != True: diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 63e8c334..65896e06 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -12,13 +12,11 @@ import numpy as np from base.webqtlConfig import TMPDIR from base.trait import create_trait -from utility.tools import locate +from utility.tools import locate, GN3_LOCAL_URL import utility.logger logger = utility.logger.getLogger(__name__) -GN3_RQTL_URL = "http://localhost:8086/api/rqtl/compute" -GN3_TMP_PATH = "/export/local/home/zas1024/genenetwork3/tmp" def run_rqtl(trait_name, vals, samples, dataset, pair_scan, mapping_scale, model, method, num_perm, perm_strata_list, do_control, control_marker, manhattan_plot, cofactors): """Run R/qtl by making a request to the GN3 endpoint and reading in the output file(s)""" @@ -52,7 +50,7 @@ def run_rqtl(trait_name, vals, samples, dataset, pair_scan, mapping_scale, model if perm_strata_list: post_data["pstrata"] = True - rqtl_output = requests.post(GN3_RQTL_URL, data=post_data).json() + rqtl_output = requests.post(GN3_LOCAL_URL + "api/rqtl/compute", data=post_data).json() if num_perm > 0: return rqtl_output['perm_results'], rqtl_output['suggestive'], rqtl_output['significant'], rqtl_output['results'] else: diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index 640cf9cd..cf217a96 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -229,7 +229,7 @@ class RunMapping: self.perm_strata = get_perm_strata( self.this_trait, primary_samples, self.categorical_vars, self.samples) - self.score_type = "LOD" + self.score_type = "-logP" self.control_marker = start_vars['control_marker'] self.do_control = start_vars['do_control'] if 'mapmethod_rqtl' in start_vars: @@ -531,7 +531,8 @@ def export_mapping_results(dataset, trait, markers, results_path, mapping_method transform_text = "" output_file.write(transform_text + "\n") if dataset.type == "ProbeSet": - output_file.write("Gene Symbol: " + trait.symbol + "\n") + if trait.symbol: + output_file.write("Gene Symbol: " + trait.symbol + "\n") output_file.write("Location: " + str(trait.chr) + \ " @ " + str(trait.mb) + " Mb\n") if len(covariates) > 0: diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py new file mode 100644 index 00000000..dc738f88 --- /dev/null +++ b/wqflask/wqflask/metadata_edits.py @@ -0,0 +1,557 @@ +import datetime +import json +import os +import re + +from collections import namedtuple +from itertools import groupby +from typing import Dict + +import MySQLdb +import difflib +import redis + +from flask import Blueprint +from flask import Response +from flask import current_app +from flask import flash +from flask import g +from flask import redirect +from flask import render_template +from flask import request +from flask import url_for + +from wqflask.decorators import edit_access_required +from wqflask.decorators import edit_admins_access_required +from wqflask.decorators import login_required + +from gn3.authentication import AdminRole +from gn3.authentication import DataRole +from gn3.authentication import get_highest_user_access_role +from gn3.authentication import get_user_membership +from gn3.commands import run_cmd +from gn3.db import diff_from_dict +from gn3.db import fetchall +from gn3.db import fetchone +from gn3.db import insert +from gn3.db import update +from gn3.db.metadata_audit import MetadataAudit +from gn3.db.phenotypes import Phenotype +from gn3.db.phenotypes import Probeset +from gn3.db.phenotypes import Publication +from gn3.db.phenotypes import PublishXRef +from gn3.db.phenotypes import probeset_mapping +from gn3.db.traits import get_trait_csv_sample_data +from gn3.db.traits import update_sample_data +from gn3.db.traits import delete_sample_data +from gn3.db.traits import insert_sample_data + + +metadata_edit = Blueprint('metadata_edit', __name__) + + +def _get_diffs(diff_dir: str, + user_id: str, + redis_conn: redis.Redis, + gn_proxy_url: str): + def __get_file_metadata(file_name: str) -> Dict: + author, resource_id, time_stamp, *_ = file_name.split(".") + + return { + "resource_id": resource_id, + "file_name": file_name, + "author": json.loads(redis_conn.hget("users", + author)).get("full_name"), + "time_stamp": time_stamp, + "roles": get_highest_user_access_role( + resource_id=resource_id, + user_id=user_id, + gn_proxy_url=gn_proxy_url), + } + + approved, rejected, waiting = [], [], [] + if os.path.exists(diff_dir): + for name in os.listdir(diff_dir): + file_metadata = __get_file_metadata(file_name=name) + admin_status = file_metadata["roles"].get("admin") + append_p = (user_id in name or + admin_status > AdminRole.EDIT_ACCESS) + if name.endswith(".rejected") and append_p: + rejected.append(__get_file_metadata(file_name=name)) + elif name.endswith(".approved") and append_p: + approved.append(__get_file_metadata(file_name=name)) + elif append_p: # Normal file + waiting.append(__get_file_metadata(file_name=name)) + return { + "approved": approved, + "rejected": rejected, + "waiting": waiting, + } + + +def edit_phenotype(conn, name, dataset_id): + publish_xref = fetchone( + conn=conn, + table="PublishXRef", + where=PublishXRef(id_=name, + inbred_set_id=dataset_id)) + phenotype_ = fetchone( + conn=conn, + table="Phenotype", + where=Phenotype(id_=publish_xref.phenotype_id)) + publication_ = fetchone( + conn=conn, + table="Publication", + where=Publication(id_=publish_xref.publication_id)) + json_data = fetchall( + conn, + "metadata_audit", + where=MetadataAudit(dataset_id=publish_xref.id_)) + Edit = namedtuple("Edit", ["field", "old", "new", "diff"]) + Diff = namedtuple("Diff", ["author", "diff", "timestamp"]) + diff_data = [] + for data in json_data: + json_ = json.loads(data.json_data) + timestamp = json_.get("timestamp") + author = json_.get("author") + for key, value in json_.items(): + if isinstance(value, dict): + for field, data_ in value.items(): + diff_data.append( + Diff(author=author, + diff=Edit(field, + data_.get("old"), + data_.get("new"), + "\n".join(difflib.ndiff( + [data_.get("old")], + [data_.get("new")]))), + timestamp=timestamp)) + diff_data_ = None + if len(diff_data) > 0: + diff_data_ = groupby(diff_data, lambda x: x.timestamp) + return { + "diff": diff_data_, + "publish_xref": publish_xref, + "phenotype": phenotype_, + "publication": publication_, + } + + +def edit_probeset(conn, name): + probeset_ = fetchone(conn=conn, + table="ProbeSet", + columns=list(probeset_mapping.values()), + where=Probeset(name=name)) + json_data = fetchall( + conn, + "metadata_audit", + where=MetadataAudit(dataset_id=probeset_.id_)) + Edit = namedtuple("Edit", ["field", "old", "new", "diff"]) + Diff = namedtuple("Diff", ["author", "diff", "timestamp"]) + diff_data = [] + for data in json_data: + json_ = json.loads(data.json_data) + timestamp = json_.get("timestamp") + author = json_.get("author") + for key, value in json_.items(): + if isinstance(value, dict): + for field, data_ in value.items(): + diff_data.append( + Diff(author=author, + diff=Edit(field, + data_.get("old"), + data_.get("new"), + "\n".join(difflib.ndiff( + [data_.get("old")], + [data_.get("new")]))), + timestamp=timestamp)) + diff_data_ = None + if len(diff_data) > 0: + diff_data_ = groupby(diff_data, lambda x: x.timestamp) + return { + "diff": diff_data_, + "probeset": probeset_, + } + + +@metadata_edit.route("/<dataset_id>/traits/<name>") +@edit_access_required +@login_required +def display_phenotype_metadata(dataset_id: str, name: str): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + _d = edit_phenotype(conn=conn, name=name, dataset_id=dataset_id) + return render_template( + "edit_phenotype.html", + diff=_d.get("diff"), + publish_xref=_d.get("publish_xref"), + phenotype=_d.get("phenotype"), + publication=_d.get("publication"), + dataset_id=dataset_id, + resource_id=request.args.get("resource-id"), + version=os.environ.get("GN_VERSION"), + ) + + +@metadata_edit.route("/traits/<name>") +@edit_access_required +@login_required +def display_probeset_metadata(name: str): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + _d = edit_probeset(conn=conn, name=name) + return render_template( + "edit_probeset.html", + diff=_d.get("diff"), + probeset=_d.get("probeset"), + name=name, + resource_id=request.args.get("resource-id"), + version=os.environ.get("GN_VERSION"), + ) + + +@metadata_edit.route("/<dataset_id>/traits/<name>", methods=("POST",)) +@edit_access_required +@login_required +def update_phenotype(dataset_id: str, name: str): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + data_ = request.form.to_dict() + TMPDIR = current_app.config.get("TMPDIR") + author = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") + phenotype_id = str(data_.get('phenotype-id')) + if not (file_ := request.files.get("file")): + flash("No sample-data has been uploaded", "warning") + else: + if not os.path.exists(SAMPLE_DATADIR := os.path.join(TMPDIR, "sample-data")): + os.makedirs(SAMPLE_DATADIR) + if not os.path.exists(os.path.join(SAMPLE_DATADIR, + "diffs")): + os.makedirs(os.path.join(SAMPLE_DATADIR, + "diffs")) + if not os.path.exists(os.path.join(SAMPLE_DATADIR, + "updated")): + os.makedirs(os.path.join(SAMPLE_DATADIR, + "updated")) + current_time = str(datetime.datetime.now().isoformat()) + _file_name = (f"{author}.{request.args.get('resource-id')}." + f"{current_time}") + new_file_name = (os.path.join(TMPDIR, + f"sample-data/updated/{_file_name}.csv")) + uploaded_file_name = (os.path.join( + TMPDIR, "sample-data/updated/", + f"{_file_name}.csv.uploaded")) + file_.save(new_file_name) + with open(uploaded_file_name, "w") as f_: + f_.write(get_trait_csv_sample_data( + conn=conn, + trait_name=str(name), + phenotype_id=str(phenotype_id))) + r = run_cmd(cmd=("csvdiff " + f"'{uploaded_file_name}' '{new_file_name}' " + "--format json")) + + # Edge case where the csv file has not been edited! + if not any(json.loads(r.get("output")).values()): + flash(f"You have not modified the csv file you downloaded!", + "warning") + return redirect(f"/datasets/{dataset_id}/traits/{name}" + f"?resource-id={request.args.get('resource-id')}") + diff_output = (f"{TMPDIR}/sample-data/diffs/" + f"{_file_name}.json") + with open(diff_output, "w") as f: + dict_ = json.loads(r.get("output")) + dict_.update({ + "trait_name": str(name), + "phenotype_id": str(phenotype_id), + "author": author, + "timestamp": datetime.datetime.now().strftime( + "%Y-%m-%d %H:%M:%S") + }) + f.write(json.dumps(dict_)) + flash("Sample-data has been successfully uploaded", "success") + # Run updates: + phenotype_ = { + "pre_pub_description": data_.get("pre-pub-desc"), + "post_pub_description": data_.get("post-pub-desc"), + "original_description": data_.get("orig-desc"), + "units": data_.get("units"), + "pre_pub_abbreviation": data_.get("pre-pub-abbrev"), + "post_pub_abbreviation": data_.get("post-pub-abbrev"), + "lab_code": data_.get("labcode"), + "submitter": data_.get("submitter"), + "owner": data_.get("owner"), + "authorized_users": data_.get("authorized-users"), + } + updated_phenotypes = update( + conn, "Phenotype", + data=Phenotype(**phenotype_), + where=Phenotype(id_=data_.get("phenotype-id"))) + diff_data = {} + if updated_phenotypes: + diff_data.update({"Phenotype": diff_from_dict(old={ + k: data_.get(f"old_{k}") for k, v in phenotype_.items() + if v is not None}, new=phenotype_)}) + publication_ = { + "abstract": data_.get("abstract"), + "authors": data_.get("authors"), + "title": data_.get("title"), + "journal": data_.get("journal"), + "volume": data_.get("volume"), + "pages": data_.get("pages"), + "month": data_.get("month"), + "year": data_.get("year") + } + updated_publications = update( + conn, "Publication", + data=Publication(**publication_), + where=Publication(id_=data_.get("pubmed-id", + data_.get("old_id_")))) + if updated_publications: + diff_data.update({"Publication": diff_from_dict(old={ + k: data_.get(f"old_{k}") for k, v in publication_.items() + if v is not None}, new=publication_)}) + if diff_data: + diff_data.update({ + "phenotype_id": str(phenotype_id), + "dataset_id": name, + "resource_id": request.args.get('resource-id'), + "author": author, + "timestamp": (datetime + .datetime + .now() + .strftime("%Y-%m-%d %H:%M:%S")), + }) + insert(conn, + table="metadata_audit", + data=MetadataAudit(dataset_id=name, + editor=author, + json_data=json.dumps(diff_data))) + flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success") + return redirect(f"/datasets/{dataset_id}/traits/{name}" + f"?resource-id={request.args.get('resource-id')}") + + +@metadata_edit.route("/traits/<name>", methods=("POST",)) +@edit_access_required +@login_required +def update_probeset(name: str): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + data_ = request.form.to_dict() + probeset_ = { + "id_": data_.get("id"), + "symbol": data_.get("symbol"), + "description": data_.get("description"), + "probe_target_description": data_.get("probe_target_description"), + "chr_": data_.get("chr"), + "mb": data_.get("mb"), + "alias": data_.get("alias"), + "geneid": data_.get("geneid"), + "homologeneid": data_.get("homologeneid"), + "unigeneid": data_.get("unigeneid"), + "omim": data_.get("OMIM"), + "refseq_transcriptid": data_.get("refseq_transcriptid"), + "blatseq": data_.get("blatseq"), + "targetseq": data_.get("targetseq"), + "strand_probe": data_.get("Strand_Probe"), + "probe_set_target_region": data_.get("probe_set_target_region"), + "probe_set_specificity": data_.get("probe_set_specificity"), + "probe_set_blat_score": data_.get("probe_set_blat_score"), + "probe_set_blat_mb_start": data_.get("probe_set_blat_mb_start"), + "probe_set_blat_mb_end": data_.get("probe_set_blat_mb_end"), + "probe_set_strand": data_.get("probe_set_strand"), + "probe_set_note_by_rw": data_.get("probe_set_note_by_rw"), + "flag": data_.get("flag") + } + diff_data = {} + author = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") + if (updated_probeset := update( + conn, "ProbeSet", + data=Probeset(**probeset_), + where=Probeset(id_=data_.get("id")))): + diff_data.update({"Probeset": diff_from_dict(old={ + k: data_.get(f"old_{k}") for k, v in probeset_.items() + if v is not None}, new=probeset_)}) + if diff_data: + diff_data.update({"probeset_name": data_.get("probeset_name")}) + diff_data.update({"author": author}) + diff_data.update({"resource_id": request.args.get('resource-id')}) + diff_data.update({"timestamp": datetime.datetime.now().strftime( + "%Y-%m-%d %H:%M:%S")}) + insert(conn, + table="metadata_audit", + data=MetadataAudit(dataset_id=data_.get("id"), + editor=author, + json_data=json.dumps(diff_data))) + return redirect(f"/datasets/traits/{name}" + f"?resource-id={request.args.get('resource-id')}") + + +@metadata_edit.route("/<dataset_id>/traits/<phenotype_id>/csv") +@login_required +def get_sample_data_as_csv(dataset_id: str, phenotype_id: int): + return Response( + get_trait_csv_sample_data( + conn=MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")), + trait_name=str(dataset_id), + phenotype_id=str(phenotype_id)), + mimetype="text/csv", + headers={"Content-disposition": + "attachment; filename=myplot.csv"} + ) + + +@metadata_edit.route("/diffs") +@login_required +def list_diffs(): + files = _get_diffs( + diff_dir=f"{current_app.config.get('TMPDIR')}/sample-data/diffs", + user_id=((g.user_session.record.get(b"user_id") or + b"").decode("utf-8") + or g.user_session.record.get("user_id") or ""), + redis_conn=redis.from_url(current_app.config["REDIS_URL"], + decode_responses=True), + gn_proxy_url=current_app.config.get("GN2_PROXY")) + return render_template( + "display_files.html", + approved=sorted(files.get("approved"), + reverse=True, + key=lambda d: d.get("time_stamp")), + rejected=sorted(files.get("rejected"), + reverse=True, + key=lambda d: d.get("time_stamp")), + waiting=sorted(files.get("waiting"), + reverse=True, + key=lambda d: d.get("time_stamp"))) + + +@metadata_edit.route("/diffs/<name>") +def show_diff(name): + TMPDIR = current_app.config.get("TMPDIR") + with open(os.path.join(f"{TMPDIR}/sample-data/diffs", + name), 'r') as myfile: + content = myfile.read() + content = json.loads(content) + for data in content.get("Modifications"): + data["Diff"] = "\n".join(difflib.ndiff([data.get("Original")], + [data.get("Current")])) + return render_template( + "display_diffs.html", + diff=content + ) + + +@metadata_edit.route("<resource_id>/diffs/<file_name>/reject") +@edit_admins_access_required +@login_required +def reject_data(resource_id: str, file_name: str): + TMPDIR = current_app.config.get("TMPDIR") + os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), + os.path.join(f"{TMPDIR}/sample-data/diffs", + f"{file_name}.rejected")) + flash(f"{file_name} has been rejected!", "success") + return redirect(url_for('metadata_edit.list_diffs')) + + +@metadata_edit.route("<resource_id>/diffs/<file_name>/approve") +@edit_admins_access_required +@login_required +def approve_data(resource_id:str, file_name: str): + sample_data = {file_name: str} + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + TMPDIR = current_app.config.get("TMPDIR") + with open(os.path.join(f"{TMPDIR}/sample-data/diffs", + file_name), 'r') as myfile: + sample_data = json.load(myfile) + for modification in ( + modifications := [d for d in sample_data.get("Modifications")]): + if modification.get("Current"): + (strain_name, + value, se, count) = modification.get("Current").split(",") + update_sample_data( + conn=conn, + trait_name=sample_data.get("trait_name"), + strain_name=strain_name, + phenotype_id=int(sample_data.get("phenotype_id")), + value=value, + error=se, + count=count) + + n_deletions = 0 + for deletion in (deletions := [d for d in sample_data.get("Deletions")]): + strain_name, _, _, _ = deletion.split(",") + __deletions, _, _ = delete_sample_data( + conn=conn, + trait_name=sample_data.get("trait_name"), + strain_name=strain_name, + phenotype_id=int(sample_data.get("phenotype_id"))) + if __deletions: + n_deletions += 1 + # Remove any data that already exists from sample_data deletes + else: + sample_data.get("Deletions").remove(deletion) + + n_insertions = 0 + for insertion in ( + insertions := [d for d in sample_data.get("Additions")]): + (strain_name, + value, se, count) = insertion.split(",") + __insertions, _, _ = insert_sample_data( + conn=conn, + trait_name=sample_data.get("trait_name"), + strain_name=strain_name, + phenotype_id=int(sample_data.get("phenotype_id")), + value=value, + error=se, + count=count) + if __insertions: + n_insertions += 1 + # Remove any data that already exists from sample_data inserts + else: + sample_data.get("Additions").remove(insertion) + if any([sample_data.get("Additions"), + sample_data.get("Modifications"), + sample_data.get("Deletions")]): + insert(conn, + table="metadata_audit", + data=MetadataAudit( + dataset_id=sample_data.get("trait_name"), + editor=sample_data.get("author"), + json_data=json.dumps(sample_data))) + # Once data is approved, rename it! + os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), + os.path.join(f"{TMPDIR}/sample-data/diffs", + f"{file_name}.approved")) + message = "" + if n_deletions: + flash(f"# Deletions: {n_deletions}", "success") + if n_insertions: + flash("# Additions: {len(modifications)", "success") + if len(modifications): + flash("# Modifications: {len(modifications)}", "success") + else: # Edge case where you need to automatically reject the file + os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), + os.path.join(f"{TMPDIR}/sample-data/diffs", + f"{file_name}.rejected")) + flash(("Automatically rejecting this file since no " + "changes could be applied."), "warning") + + return redirect(url_for('metadata_edit.list_diffs')) + diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index c54dd0b3..c0717314 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -1,144 +1,169 @@ import json +import redis +import requests -from flask import (Flask, g, render_template, url_for, request, make_response, - redirect, flash) - -from wqflask import app - -from utility.authentication_tools import check_owner_or_admin -from utility.redis_tools import get_resource_info, get_group_info, get_groups_like_unique_column, get_user_id, get_user_by_unique_column, get_users_like_unique_column, add_access_mask, add_resource, change_resource_owner - - - -@app.route("/resources/manage", methods=('GET', 'POST')) -def manage_resource(): - params = request.form if request.form else request.args - if 'resource_id' in request.args: - resource_id = request.args['resource_id'] - admin_status = check_owner_or_admin(resource_id=resource_id) - - resource_info = get_resource_info(resource_id) - group_masks = resource_info['group_masks'] - group_masks_with_names = get_group_names(group_masks) - default_mask = resource_info['default_mask']['data'] - owner_id = resource_info['owner_id'] - - owner_display_name = None - if owner_id != "none": - try: # ZS: User IDs are sometimes stored in Redis as bytes and sometimes as strings, so this is just to avoid any errors for the time being - owner_id = str.encode(owner_id) - except: - pass - owner_info = get_user_by_unique_column("user_id", owner_id) - if 'name' in owner_info: - owner_display_name = owner_info['full_name'] - elif 'user_name' in owner_info: - owner_display_name = owner_info['user_name'] - elif 'email_address' in owner_info: - owner_display_name = owner_info['email_address'] - - return render_template("admin/manage_resource.html", owner_name=owner_display_name, resource_id=resource_id, resource_info=resource_info, default_mask=default_mask, group_masks=group_masks_with_names, admin_status=admin_status) - - -@app.route("/search_for_users", methods=('POST',)) -def search_for_user(): - params = request.form - user_list = [] - user_list += get_users_like_unique_column("full_name", params['user_name']) - user_list += get_users_like_unique_column( - "email_address", params['user_email']) - - return json.dumps(user_list) - - -@app.route("/search_for_groups", methods=('POST',)) -def search_for_groups(): - params = request.form - group_list = [] - group_list += get_groups_like_unique_column("id", params['group_id']) - group_list += get_groups_like_unique_column("name", params['group_name']) - - user_list = [] - user_list += get_users_like_unique_column("full_name", params['user_name']) - user_list += get_users_like_unique_column( - "email_address", params['user_email']) - for user in user_list: - group_list += get_groups_like_unique_column("admins", user['user_id']) - group_list += get_groups_like_unique_column("members", user['user_id']) - - return json.dumps(group_list) - - -@app.route("/resources/change_owner", methods=('POST',)) -def change_owner(): - resource_id = request.form['resource_id'] - if 'new_owner' in request.form: - admin_status = check_owner_or_admin(resource_id=resource_id) - if admin_status == "owner": - new_owner_id = request.form['new_owner'] - change_resource_owner(resource_id, new_owner_id) - flash("The resource's owner has beeen changed.", "alert-info") - return redirect(url_for("manage_resource", resource_id=resource_id)) - else: - flash("You lack the permissions to make this change.", "error") - return redirect(url_for("manage_resource", resource_id=resource_id)) - else: - return render_template("admin/change_resource_owner.html", resource_id=resource_id) - - -@app.route("/resources/change_default_privileges", methods=('POST',)) -def change_default_privileges(): - resource_id = request.form['resource_id'] - admin_status = check_owner_or_admin(resource_id=resource_id) - if admin_status == "owner" or admin_status == "edit-admins": - resource_info = get_resource_info(resource_id) - default_mask = resource_info['default_mask'] - if request.form['open_to_public'] == "True": - default_mask['data'] = 'view' - else: - default_mask['data'] = 'no-access' - resource_info['default_mask'] = default_mask - add_resource(resource_info) - flash("Your changes have been saved.", "alert-info") - return redirect(url_for("manage_resource", resource_id=resource_id)) - else: - return redirect(url_for("no_access_page")) - - -@app.route("/resources/add_group", methods=('POST',)) -def add_group_to_resource(): - resource_id = request.form['resource_id'] - admin_status = check_owner_or_admin(resource_id=resource_id) - if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access": - if 'selected_group' in request.form: - group_id = request.form['selected_group'] - resource_info = get_resource_info(resource_id) - default_privileges = resource_info['default_mask'] - return render_template("admin/set_group_privileges.html", resource_id=resource_id, group_id=group_id, default_privileges=default_privileges) - elif all(key in request.form for key in ('data_privilege', 'metadata_privilege', 'admin_privilege')): - group_id = request.form['group_id'] - group_name = get_group_info(group_id)['name'] - access_mask = { - 'data': request.form['data_privilege'], - 'metadata': request.form['metadata_privilege'], - 'admin': request.form['admin_privilege'] - } - add_access_mask(resource_id, group_id, access_mask) - flash("Privileges have been added for group {}.".format( - group_name), "alert-info") - return redirect(url_for("manage_resource", resource_id=resource_id)) - else: - return render_template("admin/search_for_groups.html", resource_id=resource_id) - else: - return redirect(url_for("no_access_page")) +from flask import Blueprint +from flask import current_app +from flask import flash +from flask import g +from flask import redirect +from flask import render_template +from flask import request +from flask import url_for + +from gn3.authentication import AdminRole +from gn3.authentication import DataRole +from gn3.authentication import get_user_membership +from gn3.authentication import get_highest_user_access_role + +from typing import Dict, Tuple +from urllib.parse import urljoin + + +from wqflask.decorators import edit_access_required +from wqflask.decorators import edit_admins_access_required +from wqflask.decorators import login_required -def get_group_names(group_masks): - group_masks_with_names = {} - for group_id, group_mask in list(group_masks.items()): - this_mask = group_mask - group_name = get_group_info(group_id)['name'] - this_mask['name'] = group_name - group_masks_with_names[group_id] = this_mask +resource_management = Blueprint('resource_management', __name__) - return group_masks_with_names + +def add_extra_resource_metadata(conn: redis.Redis, + resource_id: str, + resource: Dict) -> Dict: + """If resource['owner_id'] exists, add metadata about that user. Also, +if the resource contains group masks, add the group name into the +resource dict. Note that resource['owner_id'] and the group masks are +unique identifiers so they aren't human readable names. + + Args: + - conn: A redis connection with the responses decoded. + - resource_id: The unique identifier of the resource. + - resource: A dict containing details(metadata) about a + given resource. + + Returns: + An embellished dictionary with its resource id; the human + readable names of the group masks; and the owner id if it was set. + + """ + resource["resource_id"] = resource_id + + # Embellish the resource information with owner details if the + # owner is set + if (owner_id := resource.get("owner_id", "none").lower()) == "none": + resource["owner_id"] = None + resource["owner_details"] = None + else: + user_details = json.loads(conn.hget("users", owner_id)) + resource["owner_details"] = { + "email_address": user_details.get("email_address"), + "full_name": user_details.get("full_name"), + "organization": user_details.get("organization"), + } + + # Embellish the resources information with the group name if the + # group masks are present + if groups := resource.get('group_masks', {}): + for group_id in groups.keys(): + resource['group_masks'][group_id]["group_name"] = ( + json.loads(conn.hget("groups", group_id)).get('name')) + return resource + + +@resource_management.route("/resources/<resource_id>") +@login_required +def view_resource(resource_id: str): + user_id = (g.user_session.record.get(b"user_id", + b"").decode("utf-8") or + g.user_session.record.get("user_id", "")) + redis_conn = redis.from_url( + current_app.config["REDIS_URL"], + decode_responses=True) + # Abort early if the resource can't be found + if not (resource := redis_conn.hget("resources", resource_id)): + return f"Resource: {resource_id} Not Found!", 401 + + return render_template( + "admin/manage_resource.html", + resource_info=(add_extra_resource_metadata( + conn=redis_conn, + resource_id=resource_id, + resource=json.loads(resource))), + access_role=get_highest_user_access_role( + resource_id=resource_id, + user_id=user_id, + gn_proxy_url=current_app.config.get("GN2_PROXY"))) + + +@resource_management.route("/resources/<resource_id>/make-public", + methods=('POST',)) +@edit_access_required +@login_required +def update_resource_publicity(resource_id: str): + redis_conn = redis.from_url( + current_app.config["REDIS_URL"], + decode_responses=True) + resource_info = json.loads(redis_conn.hget("resources", resource_id)) + + if (is_open_to_public := request + .form + .to_dict() + .get("open_to_public")) == "True": + resource_info['default_mask'] = { + 'data': DataRole.VIEW.value, + 'admin': AdminRole.NOT_ADMIN.value, + 'metadata': DataRole.VIEW.value, + } + elif is_open_to_public == "False": + resource_info['default_mask'] = { + 'data': DataRole.NO_ACCESS.value, + 'admin': AdminRole.NOT_ADMIN.value, + 'metadata': DataRole.NO_ACCESS.value, + } + redis_conn.hset("resources", resource_id, json.dumps(resource_info)) + return redirect(url_for("resource_management.view_resource", + resource_id=resource_id)) + + +@resource_management.route("/resources/<resource_id>/change-owner") +@edit_admins_access_required +@login_required +def view_resource_owner(resource_id: str): + return render_template( + "admin/change_resource_owner.html", + resource_id=resource_id) + + +@resource_management.route("/resources/<resource_id>/change-owner", + methods=('POST',)) +@edit_admins_access_required +@login_required +def change_owner(resource_id: str): + if user_id := request.form.get("new_owner"): + redis_conn = redis.from_url( + current_app.config["REDIS_URL"], + decode_responses=True) + resource = json.loads(redis_conn.hget("resources", resource_id)) + resource["owner_id"] = user_id + redis_conn.hset("resources", resource_id, json.dumps(resource)) + flash("The resource's owner has been changed.", "alert-info") + return redirect(url_for("resource_management.view_resource", + resource_id=resource_id)) + + +@resource_management.route("<resource_id>/users/search", methods=('POST',)) +@edit_admins_access_required +@login_required +def search_user(resource_id: str): + results = {} + for user in (users := redis.from_url( + current_app.config["REDIS_URL"], + decode_responses=True).hgetall("users")): + user = json.loads(users[user]) + for q in (request.form.get("user_name"), + request.form.get("user_email")): + if q and (q in user.get("email_address") or + q in user.get("full_name")): + results[user.get("user_id", "")] = user + return json.dumps(tuple(results.values())) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 3cbda3dd..cf2905c9 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -4,6 +4,8 @@ from math import * import time import re import requests +from types import SimpleNamespace +import unicodedata from pprint import pformat as pf @@ -11,6 +13,7 @@ import json from base.data_set import create_dataset from base.trait import create_trait +from base.webqtlConfig import PUBMEDLINK_URL from wqflask import parser from wqflask import do_search from db import webqtlDatabaseFunction @@ -18,13 +21,13 @@ from db import webqtlDatabaseFunction from flask import Flask, g from utility import hmac, helper_functions +from utility.authentication_tools import check_resource_availability from utility.tools import GN2_BASE_URL from utility.type_checking import is_str from utility.logger import getLogger logger = getLogger(__name__) - class SearchResultPage: #maxReturn = 3000 @@ -40,9 +43,7 @@ class SearchResultPage: self.uc_id = uuid.uuid4() self.go_term = None - logger.debug("uc_id:", self.uc_id) # contains a unique id - logger.debug("kw is:", kw) # dict containing search terms if kw['search_terms_or']: self.and_or = "or" self.search_terms = kw['search_terms_or'] @@ -55,15 +56,17 @@ class SearchResultPage: rx = re.compile( r'.*\W(href|http|sql|select|update)\W.*', re.IGNORECASE) if rx.match(search): - logger.info("Regex failed search") + logger.debug("Regex failed search") self.search_term_exists = False return else: self.search_term_exists = True self.results = [] + max_result_count = 100000 # max number of results to display type = kw.get('type') if type == "Phenotypes": # split datatype on type field + max_result_count = 50000 dataset_type = "Publish" elif type == "Genotypes": dataset_type = "Geno" @@ -72,9 +75,8 @@ class SearchResultPage: assert(is_str(kw.get('dataset'))) self.dataset = create_dataset(kw['dataset'], dataset_type) - logger.debug("search_terms:", self.search_terms) - # ZS: I don't like using try/except, but it seems like the easiest way to account for all possible bad searches here + # I don't like using try/except, but it seems like the easiest way to account for all possible bad searches here try: self.search() except: @@ -82,7 +84,7 @@ class SearchResultPage: self.too_many_results = False if self.search_term_exists: - if len(self.results) > 50000: + if len(self.results) > max_result_count: self.trait_list = [] self.too_many_results = True else: @@ -97,88 +99,125 @@ class SearchResultPage: trait_list = [] json_trait_list = [] - species = webqtlDatabaseFunction.retrieve_species( - self.dataset.group.name) # result_set represents the results for each search term; a search of # "shh grin2b" would have two sets of results, one for each term - logger.debug("self.results is:", pf(self.results)) + + if self.dataset.type == "ProbeSet": + self.header_data_names = ['index', 'display_name', 'symbol', 'description', 'location', 'mean', 'lrs_score', 'lrs_location', 'additive'] + elif self.dataset.type == "Publish": + self.header_data_names = ['index', 'display_name', 'description', 'mean', 'authors', 'pubmed_text', 'lrs_score', 'lrs_location', 'additive'] + elif self.dataset.type == "Geno": + self.header_data_names = ['index', 'display_name', 'location'] for index, result in enumerate(self.results): if not result: continue - #### Excel file needs to be generated #### - trait_dict = {} - trait_id = result[0] - this_trait = create_trait( - dataset=self.dataset, name=trait_id, get_qtl_info=True, get_sample_info=False) - if this_trait: - trait_dict['index'] = index + 1 - trait_dict['name'] = this_trait.name - if this_trait.dataset.type == "Publish": - trait_dict['display_name'] = this_trait.display_name + trait_dict['index'] = index + 1 + + trait_dict['dataset'] = self.dataset.name + if self.dataset.type == "ProbeSet": + trait_dict['display_name'] = result[2] + trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait_dict['display_name'], trait_dict['dataset'])) + trait_dict['symbol'] = "N/A" if result[3] is None else result[3].strip() + description_text = "" + if result[4] is not None and str(result[4]) != "": + description_text = unicodedata.normalize("NFKD", result[4].decode('latin1')) + + target_string = result[5].decode('utf-8') if result[5] else "" + description_display = description_text if target_string is None or str(target_string) == "" else description_text + "; " + str(target_string).strip() + trait_dict['description'] = description_display + + trait_dict['location'] = "N/A" + if (result[6] is not None) and (result[6] != "") and (result[7] is not None) and (result[7] != 0): + trait_dict['location'] = f"Chr{result[6]}: {float(result[7]):.6f}" + + trait_dict['mean'] = "N/A" if result[8] is None or result[8] == "" else f"{result[8]:.3f}" + trait_dict['additive'] = "N/A" if result[12] is None or result[12] == "" else f"{result[12]:.3f}" + trait_dict['lod_score'] = "N/A" if result[9] is None or result[9] == "" else f"{float(result[9]) / 4.61:.1f}" + trait_dict['lrs_location'] = "N/A" if result[13] is None or result[13] == "" or result[14] is None else f"Chr{result[13]}: {float(result[14]):.6f}" + elif self.dataset.type == "Geno": + trait_dict['display_name'] = str(result[0]) + trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait_dict['display_name'], trait_dict['dataset'])) + trait_dict['location'] = "N/A" + if (result[4] != "NULL" and result[4] != "") and (result[5] != 0): + trait_dict['location'] = f"Chr{result[4]}: {float(result[5]):.6f}" + elif self.dataset.type == "Publish": + # Check permissions on a trait-by-trait basis for phenotype traits + trait_dict['name'] = trait_dict['display_name'] = str(result[0]) + trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait_dict['name'], trait_dict['dataset'])) + permissions = check_resource_availability(self.dataset, trait_dict['display_name']) + if "view" not in permissions['data']: + continue + + if result[10]: + trait_dict['display_name'] = str(result[10]) + "_" + str(result[0]) + trait_dict['description'] = "N/A" + trait_dict['pubmed_id'] = "N/A" + trait_dict['pubmed_link'] = "N/A" + trait_dict['pubmed_text'] = "N/A" + trait_dict['mean'] = "N/A" + trait_dict['additive'] = "N/A" + pre_pub_description = "N/A" if result[1] is None else result[1].strip() + post_pub_description = "N/A" if result[2] is None else result[2].strip() + if result[5] != "NULL" and result[5] != None: + trait_dict['pubmed_id'] = result[5] + trait_dict['pubmed_link'] = PUBMEDLINK_URL % trait_dict['pubmed_id'] + trait_dict['description'] = post_pub_description else: - trait_dict['display_name'] = this_trait.name - trait_dict['dataset'] = this_trait.dataset.name - trait_dict['hmac'] = hmac.data_hmac( - '{}:{}'.format(this_trait.name, this_trait.dataset.name)) - if this_trait.dataset.type == "ProbeSet": - trait_dict['symbol'] = this_trait.symbol if this_trait.symbol else "N/A" - trait_dict['description'] = "N/A" - if this_trait.description_display: - trait_dict['description'] = this_trait.description_display - trait_dict['location'] = this_trait.location_repr - trait_dict['mean'] = "N/A" - trait_dict['additive'] = "N/A" - if this_trait.mean != "" and this_trait.mean != None: - trait_dict['mean'] = f"{this_trait.mean:.3f}" - try: - trait_dict['lod_score'] = f"{float(this_trait.LRS_score_repr) / 4.61:.1f}" - except: - trait_dict['lod_score'] = "N/A" - trait_dict['lrs_location'] = this_trait.LRS_location_repr - if this_trait.additive != "": - trait_dict['additive'] = f"{this_trait.additive:.3f}" - elif this_trait.dataset.type == "Geno": - trait_dict['location'] = this_trait.location_repr - elif this_trait.dataset.type == "Publish": - trait_dict['description'] = "N/A" - if this_trait.description_display: - trait_dict['description'] = this_trait.description_display - trait_dict['authors'] = this_trait.authors - trait_dict['pubmed_id'] = "N/A" - if this_trait.pubmed_id: - trait_dict['pubmed_id'] = this_trait.pubmed_id - trait_dict['pubmed_link'] = this_trait.pubmed_link - trait_dict['pubmed_text'] = this_trait.pubmed_text - trait_dict['mean'] = "N/A" - if this_trait.mean != "" and this_trait.mean != None: - trait_dict['mean'] = f"{this_trait.mean:.3f}" + trait_dict['description'] = pre_pub_description + + if result[4].isdigit(): + trait_dict['pubmed_text'] = result[4] + + trait_dict['authors'] = result[3] + + if result[6] != "" and result[6] != None: + trait_dict['mean'] = f"{result[6]:.3f}" + + try: + trait_dict['lod_score'] = f"{float(result[7]) / 4.61:.1f}" + except: + trait_dict['lod_score'] = "N/A" + + try: + trait_dict['lrs_location'] = f"Chr{result[11]}: {float(result[12]):.6f}" + except: + trait_dict['lrs_location'] = "N/A" + + trait_dict['additive'] = "N/A" if not result[8] else f"{result[8]:.3f}" + + # Convert any bytes in dict to a normal utf-8 string + for key in trait_dict.keys(): + if isinstance(trait_dict[key], bytes): try: - trait_dict['lod_score'] = f"{float(this_trait.LRS_score_repr) / 4.61:.1f}" - except: - trait_dict['lod_score'] = "N/A" - trait_dict['lrs_location'] = this_trait.LRS_location_repr - trait_dict['additive'] = "N/A" - if this_trait.additive != "": - trait_dict['additive'] = f"{this_trait.additive:.3f}" - # Convert any bytes in dict to a normal utf-8 string - for key in trait_dict.keys(): - if isinstance(trait_dict[key], bytes): trait_dict[key] = trait_dict[key].decode('utf-8') - trait_list.append(trait_dict) + except UnicodeDecodeError: + trait_dict[key] = trait_dict[key].decode('latin-1') + + trait_list.append(trait_dict) + + if self.results: + self.max_widths = {} + for i, trait in enumerate(trait_list): + for key in trait.keys(): + if key == "authors": + authors_string = ",".join(str(trait[key]).split(",")[:6]) + ", et al." + self.max_widths[key] = max(len(authors_string), self.max_widths[key]) if key in self.max_widths else len(str(trait[key])) + else: + self.max_widths[key] = max(len(str(trait[key])), self.max_widths[key]) if key in self.max_widths else len(str(trait[key])) - self.trait_list = trait_list + self.wide_columns_exist = False + if self.dataset.type == "Publish": + if (self.max_widths['display_name'] > 25 or self.max_widths['description'] > 100 or self.max_widths['authors']> 80): + self.wide_columns_exist = True + if self.dataset.type == "ProbeSet": + if (self.max_widths['display_name'] > 25 or self.max_widths['symbol'] > 25 or self.max_widths['description'] > 100): + self.wide_columns_exist = True - if self.dataset.type == "ProbeSet": - self.header_data_names = ['index', 'display_name', 'symbol', 'description', - 'location', 'mean', 'lrs_score', 'lrs_location', 'additive'] - elif self.dataset.type == "Publish": - self.header_data_names = ['index', 'display_name', 'description', 'mean', - 'authors', 'pubmed_text', 'lrs_score', 'lrs_location', 'additive'] - elif self.dataset.type == "Geno": - self.header_data_names = ['index', 'display_name', 'location'] + + self.trait_list = trait_list def search(self): """ @@ -186,14 +225,12 @@ class SearchResultPage: """ self.search_terms = parser.parse(self.search_terms) - logger.debug("After parsing:", self.search_terms) combined_from_clause = "" combined_where_clause = "" # The same table can't be referenced twice in the from clause previous_from_clauses = [] - logger.debug("len(search_terms)>1") symbol_list = [] if self.dataset.type == "ProbeSet": for a_search in self.search_terms: diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 52d7d308..d9821d9c 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -20,15 +20,16 @@ from base import data_set from utility import helper_functions from utility.authentication_tools import check_owner_or_admin from utility.tools import locate_ignore_error +from utility.tools import GN_PROXY_URL from utility.redis_tools import get_redis_conn, get_resource_id -from utility.logger import getLogger +from gn3.authentication import AdminRole +from gn3.authentication import DataRole +from gn3.authentication import get_highest_user_access_role Redis = get_redis_conn() ONE_YEAR = 60 * 60 * 24 * 365 -logger = getLogger(__name__) - ############################################### # # Todo: Put in security to ensure that user has permission to access @@ -38,14 +39,18 @@ logger = getLogger(__name__) class ShowTrait: - def __init__(self, kw): + def __init__(self, user_id, kw): + self.admin_status = None if 'trait_id' in kw and kw['dataset'] != "Temp": self.temp_trait = False self.trait_id = kw['trait_id'] helper_functions.get_species_dataset_trait(self, kw) - self.resource_id = get_resource_id(self.dataset, self.trait_id) - self.admin_status = check_owner_or_admin( - resource_id=self.resource_id) + self.resource_id = get_resource_id(self.dataset, + self.trait_id) + self.admin_status = get_highest_user_access_role( + user_id=user_id, + resource_id=(self.resource_id or ""), + gn_proxy_url=GN_PROXY_URL) elif 'group' in kw: self.temp_trait = True self.trait_id = "Temp_" + kw['species'] + "_" + kw['group'] + \ @@ -62,9 +67,6 @@ class ShowTrait: self.this_trait = create_trait(dataset=self.dataset, name=self.trait_id, cellid=None) - - self.admin_status = check_owner_or_admin( - dataset=self.dataset, trait_id=self.trait_id) else: self.temp_trait = True self.trait_id = kw['trait_id'] @@ -75,10 +77,7 @@ class ShowTrait: self.this_trait = create_trait(dataset=self.dataset, name=self.trait_id, cellid=None) - self.trait_vals = Redis.get(self.trait_id).split() - self.admin_status = check_owner_or_admin( - dataset=self.dataset, trait_id=self.trait_id) # ZS: Get verify/rna-seq link URLs try: @@ -528,10 +527,6 @@ class ShowTrait: sample_group_type='primary', header="%s Only" % (self.dataset.group.name)) self.sample_groups = (primary_samples,) - print("\nttttttttttttttttttttttttttttttttttttttttttttt\n") - print(self.sample_groups) - print("\nttttttttttttttttttttttttttttttttttttttttttttt\n") - self.primary_sample_names = primary_sample_names self.dataset.group.allsamples = all_samples_ordered @@ -617,7 +612,6 @@ def get_nearest_marker(this_trait, this_db): GenoFreeze.Id = GenoXRef.GenoFreezeId AND GenoFreeze.Name = '{}' ORDER BY ABS( Geno.Mb - {}) LIMIT 1""".format(this_chr, this_db.group.name + "Geno", this_mb) - logger.sql(query) result = g.db.execute(query).fetchall() if result == []: diff --git a/wqflask/wqflask/static/gif/waitAnima2.gif b/wqflask/wqflask/static/gif/waitAnima2.gif Binary files differnew file mode 100644 index 00000000..50aff7f2 --- /dev/null +++ b/wqflask/wqflask/static/gif/waitAnima2.gif diff --git a/wqflask/wqflask/static/new/css/jupyter_notebooks.css b/wqflask/wqflask/static/new/css/jupyter_notebooks.css new file mode 100644 index 00000000..db972a17 --- /dev/null +++ b/wqflask/wqflask/static/new/css/jupyter_notebooks.css @@ -0,0 +1,16 @@ +.jupyter-links { + padding: 1.5em; +} + +.jupyter-links:nth-of-type(2n) { + background: #EEEEEE; +} + +.jupyter-links .main-link { + font-size: larger; + display: block; +} + +.jupyter-links .src-link { + font-size: smaller; +} diff --git a/wqflask/wqflask/static/new/css/show_trait.css b/wqflask/wqflask/static/new/css/show_trait.css index b0514e01..3780a8f1 100644 --- a/wqflask/wqflask/static/new/css/show_trait.css +++ b/wqflask/wqflask/static/new/css/show_trait.css @@ -67,7 +67,7 @@ table.dataTable.cell-border tbody tr td:first-child { } .showtrait-main-div { - min-width: 1100px; + min-width: 1400px; } table.dataTable tbody td.column_name-Checkbox { @@ -250,7 +250,6 @@ div.export-code-container { table.sample-table { float: left; - width:100%; } input.trait-value-input { diff --git a/wqflask/wqflask/static/new/css/trait_list.css b/wqflask/wqflask/static/new/css/trait_list.css index c7249721..ce3075d4 100644 --- a/wqflask/wqflask/static/new/css/trait_list.css +++ b/wqflask/wqflask/static/new/css/trait_list.css @@ -51,4 +51,3 @@ div.dts div.dataTables_scrollBody table { div.dts div.dataTables_paginate,div.dts div.dataTables_length{ display:none } - diff --git a/wqflask/wqflask/static/new/javascript/group_manager.js b/wqflask/wqflask/static/new/javascript/group_manager.js index 4c172cbf..cd56133a 100644 --- a/wqflask/wqflask/static/new/javascript/group_manager.js +++ b/wqflask/wqflask/static/new/javascript/group_manager.js @@ -16,23 +16,22 @@ $('#clear_members').click(function(){ function add_emails(user_type){ - var email_address = $('input[name=user_email]').val(); - var email_list_string = $('input[name=' + user_type + '_emails_to_add]').val().trim() - console.log(email_list_string) + let email_address = $('input[name=user_email]').val(); + let email_list_string = $('input[name=' + user_type + '_emails_to_add]').val().trim() if (email_list_string == ""){ - var email_set = new Set(); + let email_set = new Set(); } else { - var email_set = new Set(email_list_string.split(",")) + let email_set = new Set(email_list_string.split(",")) } email_set.add(email_address) $('input[name=' + user_type + '_emails_to_add]').val(Array.from(email_set).join(',')) - var emails_display_string = Array.from(email_set).join('\n') + let emails_display_string = Array.from(email_set).join('\n') $('.added_' + user_type + 's').val(emails_display_string) } function clear_emails(user_type){ $('input[name=' + user_type + '_emails_to_add]').val("") $('.added_' + user_type + 's').val("") -}
\ No newline at end of file +} diff --git a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js index 0a060cdc..e1026f8c 100644 --- a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js +++ b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js @@ -15,6 +15,8 @@ build_columns = function() { 'data': null, 'orderDataType': "dom-checkbox", 'searchable' : false, + 'targets': 0, + 'width': "25px", 'render': function(data, type, row, meta) { return '<input type="checkbox" name="searchResult" class="checkbox edit_sample_checkbox" value="">' } @@ -23,12 +25,16 @@ build_columns = function() { 'title': "ID", 'type': "natural", 'searchable' : false, + 'targets': 1, + 'width': "35px", 'data': "this_id" }, { 'title': "Sample", 'type': "natural", 'data': null, + 'targets': 2, + 'width': "60px", 'render': function(data, type, row, meta) { return '<span class="edit_sample_sample_name">' + data.name + '</span>' } @@ -38,6 +44,8 @@ build_columns = function() { 'orderDataType': "dom-input", 'type': "cust-txt", 'data': null, + 'targets': 3, + 'width': "60px", 'render': function(data, type, row, meta) { if (data.value == null) { return '<input type="text" data-value="x" data-qnorm="x" data-zscore="x" name="value:' + data.name + '" style="text-align: right;" class="trait_value_input edit_sample_value" value="x" size=' + js_data.max_digits[0] + '>' @@ -48,13 +56,17 @@ build_columns = function() { } ]; + attr_start = 4 if (js_data.se_exists) { + attr_start += 2 column_list.push( { 'bSortable': false, 'type': "natural", 'data': null, + 'targets': 4, 'searchable' : false, + 'width': "25px", 'render': function(data, type, row, meta) { return '±' } @@ -64,6 +76,8 @@ build_columns = function() { 'orderDataType': "dom-input", 'type': "cust-txt", 'data': null, + 'targets': 5, + 'width': "60px", 'render': function(data, type, row, meta) { if (data.variance == null) { return '<input type="text" data-value="x" data-qnorm="x" data-zscore="x" name="value:' + data.name + '" class="trait_value_input edit_sample_se" value="x" size=6>' @@ -73,24 +87,49 @@ build_columns = function() { } } ); - } - if (js_data.has_num_cases === true) { - column_list.push( - { - 'title': "<div style='text-align: right;'>N</div>", - 'orderDataType': "dom-input", - 'type': "cust-txt", - 'data': null, - 'render': function(data, type, row, meta) { - if (data.num_cases == null || data.num_cases == undefined) { - return '<input type="text" data-value="x" data-qnorm="x" data-zscore="x" name="value:' + data.name + '" class="trait_value_input edit_sample_num_cases" value="x" size=4 maxlength=4>' - } else { - return '<input type="text" data-value="' + data.num_cases + '" data-qnorm="x" data-zscore="x" name="value:' + data.name + '" class="trait_value_input edit_sample_num_cases" value="' + data.num_cases + '" size=4 maxlength=4>' + if (js_data.has_num_cases === true) { + attr_start += 1 + column_list.push( + { + 'title': "<div style='text-align: right;'>N</div>", + 'orderDataType': "dom-input", + 'type': "cust-txt", + 'data': null, + 'targets': 6, + 'width': "60px", + 'render': function(data, type, row, meta) { + if (data.num_cases == null || data.num_cases == undefined) { + return '<input type="text" data-value="x" data-qnorm="x" data-zscore="x" name="value:' + data.name + '" class="trait_value_input edit_sample_num_cases" value="x" size=4 maxlength=4>' + } else { + return '<input type="text" data-value="' + data.num_cases + '" data-qnorm="x" data-zscore="x" name="value:' + data.name + '" class="trait_value_input edit_sample_num_cases" value="' + data.num_cases + '" size=4 maxlength=4>' + } } } - } - ); + ); + } + } + else { + if (js_data.has_num_cases === true) { + attr_start += 1 + column_list.push( + { + 'title': "<div style='text-align: right;'>N</div>", + 'orderDataType': "dom-input", + 'type': "cust-txt", + 'data': null, + 'targets': 4, + 'width': "60px", + 'render': function(data, type, row, meta) { + if (data.num_cases == null || data.num_cases == undefined) { + return '<input type="text" data-value="x" data-qnorm="x" data-zscore="x" name="value:' + data.name + '" class="trait_value_input edit_sample_num_cases" value="x" size=4 maxlength=4>' + } else { + return '<input type="text" data-value="' + data.num_cases + '" data-qnorm="x" data-zscore="x" name="value:' + data.name + '" class="trait_value_input edit_sample_num_cases" value="' + data.num_cases + '" size=4 maxlength=4>' + } + } + } + ); + } } attr_keys = Object.keys(js_data.attributes).sort((a, b) => (js_data.attributes[a].id > js_data.attributes[b].id) ? 1 : -1) @@ -100,6 +139,7 @@ build_columns = function() { 'title': "<div title='" + js_data.attributes[attr_keys[i]].description + "' style='text-align: " + js_data.attributes[attr_keys[i]].alignment + "'>" + js_data.attributes[attr_keys[i]].name + "</div>", 'type': "natural", 'data': null, + 'targets': attr_start + i, 'render': function(data, type, row, meta) { attr_name = Object.keys(data.extra_attributes).sort((a, b) => (parseInt(a) > parseInt(b)) ? 1 : -1)[meta.col - data.first_attr_col] @@ -119,14 +159,27 @@ build_columns = function() { return column_list } -var primary_table = $('#samples_primary').DataTable( { - 'initComplete': function(settings, json) { - $('.edit_sample_value').change(function() { - edit_data_change(); - }); - }, +columnDefs = build_columns() + +loadDataTable(first_run=true, table_id="samples_primary", table_data=js_data['sample_lists'][0]) +if (js_data.sample_lists.length > 1){ + loadDataTable(first_run=true, table_id="samples_other", table_data=js_data['sample_lists'][1]) +} + +function loadDataTable(first_run=false, table_id, table_data){ + if (!first_run){ + setUserColumnsDefWidths(table_id); + } + + if (table_id == "samples_primary"){ + table_type = "Primary" + } else { + table_type = "Other" + } + + table_settings = { 'createdRow': function ( row, data, index ) { - $(row).attr('id', "Primary_" + data.this_id) + $(row).attr('id', table_type + "_" + data.this_id) $(row).addClass("value_se"); if (data.outlier) { $(row).addClass("outlier"); @@ -155,76 +208,79 @@ var primary_table = $('#samples_primary').DataTable( { $('td', row).eq(attribute_start_pos + i + 1).attr("style", "text-align: " + js_data.attributes[attr_keys[i]].alignment + "; padding-top: 2px; padding-bottom: 0px;") } }, - 'data': js_data['sample_lists'][0], - 'columns': build_columns(), - 'order': [[1, "asc"]], - 'sDom': "Ztr", - 'autoWidth': true, - 'orderClasses': true, + 'data': table_data, + 'columns': columnDefs, + "order": [[1, "asc" ]], + "sDom": "iti", + "destroy": true, + "autoWidth": false, + "bSortClasses": false, "scrollY": "100vh", - 'scroller': true, - 'scrollCollapse': true -} ); + "scrollCollapse": true, + "scroller": true, + "iDisplayLength": -1, + "initComplete": function (settings) { + //Add JQueryUI resizable functionality to each th in the ScrollHead table + $('#' + table_id + '_wrapper .dataTables_scrollHead thead th').resizable({ + handles: "e", + alsoResize: '#' + table_id + '_wrapper .dataTables_scrollHead table', //Not essential but makes the resizing smoother + resize: function( event, ui ) { + width_change = ui.size.width - ui.originalSize.width; + }, + stop: function () { + saveColumnSettings(table_id, the_table); + loadDataTable(first_run=false, table_id, table_data); + } + }); + } + } + + if (!first_run){ + $('#' + table_type.toLowerCase() + '_container').css("width", String($('#' + table_id).width() + width_change + 17) + "px"); //ZS : Change the container width by the change in width of the adjusted column, so the overall table size adjusts properly -primary_table.draw(); //ZS: This makes the table adjust its height properly on initial load + let checked_rows = get_checked_rows(table_id); + the_table = $('#' + table_id).DataTable(table_settings); + if (checked_rows.length > 0){ + recheck_rows(the_table, checked_rows); + } + } else { + the_table = $('#' + table_id).DataTable(table_settings); + the_table.draw(); + } -primary_table.on( 'order.dt search.dt draw.dt', function () { - primary_table.column(1, {search:'applied', order:'applied'}).nodes().each( function (cell, i) { + the_table.on( 'order.dt search.dt draw.dt', function () { + the_table.column(1, {search:'applied', order:'applied'}).nodes().each( function (cell, i) { cell.innerHTML = i+1; } ); -} ).draw(); + } ).draw(); -$('#primary_searchbox').on( 'keyup', function () { - primary_table.search($(this).val()).draw(); -} ); + if (first_run){ + $('#' + table_type.toLowerCase() + '_container').css("width", String($('#' + table_id).width() + 17) + "px"); + } -if (js_data.sample_lists.length > 1){ - var other_table = $('#samples_other').DataTable( { - 'initComplete': function(settings, json) { - $('.edit_sample_value').change(function() { - edit_data_change(); - }); - }, - 'createdRow': function ( row, data, index ) { - $(row).attr('id', "Primary_" + data.this_id) - $(row).addClass("value_se"); - if (data.outlier) { - $(row).addClass("outlier"); - $(row).attr("style", "background-color: orange;"); - } - $('td', row).eq(1).addClass("column_name-Index") - $('td', row).eq(2).addClass("column_name-Sample") - $('td', row).eq(3).addClass("column_name-Value") - if (js_data.se_exists) { - $('td', row).eq(5).addClass("column_name-SE") - if (js_data.has_num_cases === true) { - $('td', row).eq(6).addClass("column_name-num_cases") - } else { - if (js_data.has_num_cases === true) { - $('td', row).eq(4).addClass("column_name-num_cases") - } - } + $('#' + table_type.toLowerCase() + '_searchbox').on( 'keyup', function () { + the_table.search($(this).val()).draw(); + } ); + + $('.toggle-vis').on('click', function (e) { + e.preventDefault(); + + function toggle_column(column) { + //ZS: Toggle column visibility + column.visible( ! column.visible() ); + if (column.visible()){ + $(this).removeClass("active"); } else { - if (js_data.has_num_cases === true) { - $('td', row).eq(4).addClass("column_name-num_cases") - } + $(this).addClass("active"); } + } - for (i=0; i < attr_keys.length; i++) { - $('td', row).eq(attribute_start_pos + i + 1).addClass("column_name-" + js_data.attributes[attr_keys[i]].name) - $('td', row).eq(attribute_start_pos + i + 1).attr("style", "text-align: " + js_data.attributes[attr_keys[i]].alignment + "; padding-top: 2px; padding-bottom: 0px;") - } - }, - 'data': js_data['sample_lists'][1], - 'columns': build_columns(), - 'order': [[1, "asc"]], - 'sDom': "Ztr", - 'autoWidth': true, - 'orderClasses': true, - "scrollY": "100vh", - 'scroller': true, - 'scrollCollapse': true + // Get the column API object + var target_cols = $(this).attr('data-column').split(",") + for (let i = 0; i < target_cols.length; i++){ + var column = the_table.column( target_cols[i] ); + toggle_column(column); + } } ); - other_table.draw(); //ZS: This makes the table adjust its height properly on initial load } diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index f050d4ae..6b81b47e 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -1770,6 +1770,10 @@ $('#filter_by_value').click(function(){ edit_data_change(); }) +$('.edit_sample_value').change(function() { + edit_data_change(); +}); + $('#exclude_group').click(edit_data_change); $('#block_outliers').click(edit_data_change); $('#reset').click(edit_data_change); diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js index d3b44309..737a0b82 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js +++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js @@ -208,6 +208,7 @@ $(".gemma-tab, #gemma_compute").on("click", (function(_this) { var form_data, url; url = "/loading"; $('input[name=method]').val("gemma"); + $('input[name=mapping_scale]').val('physic'); $('input[name=selected_chr]').val($('#chr_gemma').val()); $('input[name=num_perm]').val(0); $('input[name=genofile]').val($('#genofile_gemma').val()); diff --git a/wqflask/wqflask/static/new/javascript/table_functions.js b/wqflask/wqflask/static/new/javascript/table_functions.js new file mode 100644 index 00000000..745563c2 --- /dev/null +++ b/wqflask/wqflask/static/new/javascript/table_functions.js @@ -0,0 +1,88 @@ +recheck_rows = function(the_table, checked_rows){ + //ZS: This is meant to recheck checkboxes after columns are resized + check_cells = the_table.column(0).nodes().to$(); + for (let i = 0; i < check_cells.length; i++) { + if (checked_rows.includes(i)){ + check_cells[i].childNodes[0].checked = true; + } + } + + check_rows = trait_table.rows().nodes(); + for (let i =0; i < check_rows.length; i++) { + if (checked_rows.includes(i)){ + check_rows[i].classList.add("selected") + } + } +} + +get_checked_rows = function(table_id){ + let checked_rows = [] + $("#" + table_id + " input").each(function(index){ + if ($(this).prop("checked") == true){ + checked_rows.push(index); + } + }); + + return checked_rows +} + +function setUserColumnsDefWidths(table_id) { + var userColumnDef; + + // Get the settings for this table from localStorage + var userColumnDefs = JSON.parse(localStorage.getItem(table_id)) || []; + + if (userColumnDefs.length === 0 ) return; + + columnDefs.forEach( function(columnDef) { + // Check if there is a width specified for this column + userColumnDef = userColumnDefs.find( function(column) { + return column.targets === columnDef.targets; + }); + + // If there is, set the width of this columnDef in px + if ( userColumnDef ) { + + columnDef.sWidth = userColumnDef.width + 'px'; + columnDef.width = userColumnDef.width + 'px'; + + $('.toggle-vis').each(function(){ + if ($(this).attr('data-column') == columnDef.targets){ + if ($(this).hasClass("active")){ + columnDef.bVisible = false + } else { + columnDef.bVisible = true + } + } + }) + } + }); +} + +function saveColumnSettings(table_id, trait_table) { + var userColumnDefs = JSON.parse(localStorage.getItem(table_id)) || []; + var width, header, existingSetting; + + trait_table.columns().every( function ( targets ) { + // Check if there is a setting for this column in localStorage + existingSetting = userColumnDefs.findIndex( function(column) { return column.targets === targets;}); + + // Get the width of this column + header = this.header(); + width = $(header).width(); + + if ( existingSetting !== -1 ) { + // Update the width + userColumnDefs[existingSetting].width = width; + } else { + // Add the width for this column + userColumnDefs.push({ + targets: targets, + width: width, + }); + } + }); + + // Save (or update) the settings in localStorage + localStorage.setItem(table_id, JSON.stringify(userColumnDefs)); +} diff --git a/wqflask/wqflask/templates/admin/change_resource_owner.html b/wqflask/wqflask/templates/admin/change_resource_owner.html index ae9409b0..7fd84387 100644 --- a/wqflask/wqflask/templates/admin/change_resource_owner.html +++ b/wqflask/wqflask/templates/admin/change_resource_owner.html @@ -10,8 +10,7 @@ <div class="page-header"> <h1>Search for user to assign ownership to:</h1> </div> - <form id="change_owner_form" action="/resources/change_owner" method="POST"> - <input type="hidden" name="resource_id" value="{{ resource_id }}"> + <form id="change_owner_form" action="/resource-management/resources/{{ resource_id }}/change-owner" method="POST"> <div style="min-width: 600px; max-width: 800px;"> <fieldset> <div class="form-horizontal" style="width: 900px;"> @@ -57,7 +56,7 @@ $('#find_users').click(function() { $.ajax({ method: "POST", - url: "/search_for_users", + url: "/resource-management/{{ resource_id }}/users/search", data: { user_name: $('input[name=user_name]').val(), user_email: $('input[name=user_email]').val() @@ -67,42 +66,41 @@ }) populate_users = function(json_user_list){ - var user_list = JSON.parse(json_user_list) - - var the_html = "" + let user_list = JSON.parse(json_user_list) + let searchResultsHtml = "" if (user_list.length > 0){ - the_html += "<table id='users_table' style='padding-top: 10px; width: 100%;' class='table-hover table-striped cell-border'>"; - the_html += "<thead><tr><th></th><th>Index</th><th>Name</th><th>E-mail Address</th><th>Organization</th></tr></thead>"; - the_html += "<tbody>"; + searchResultsHtml += "<table id='users_table' style='padding-top: 10px; width: 100%;' class='table-hover table-striped cell-border'>"; + searchResultsHtml += "<thead><tr><th></th><th>Index</th><th>Name</th><th>E-mail Address</th><th>Organization</th></tr></thead>"; + searchResultsHtml += "<tbody>"; for (_i = 0, _len = user_list.length; _i < _len; _i++) { this_user = user_list[_i] - the_html += "<tr>"; - the_html += "<td align='center' class='select_user'><input type='radio' name='new_owner' value='" + this_user.user_id + "'></td>"; - the_html += "<td>" + (_i + 1).toString() + "</td>" + searchResultsHtml += "<tr>"; + searchResultsHtml += "<td align='center' class='select_user'><input type='radio' name='new_owner' value='" + this_user.user_id + "'></td>"; + searchResultsHtml += "<td>" + (_i + 1).toString() + "</td>" if ("full_name" in this_user) { - the_html += "<td>" + this_user.full_name + "</td>"; + searchResultsHtml += "<td>" + this_user.full_name + "</td>"; } else { - the_html += "<td>N/A</td>" + searchResultsHtml += "<td>N/A</td>" } if ("email_address" in this_user) { - the_html += "<td>" + this_user.email_address + "</td>"; + searchResultsHtml += "<td>" + this_user.email_address + "</td>"; } else { - the_html += "<td>N/A</td>" + searchResultsHtml += "<td>N/A</td>" } if ("organization" in this_user) { - the_html += "<td>" + this_user.organization + "</td>"; + searchResultsHtml += "<td>" + this_user.organization + "</td>"; } else { - the_html += "<td>N/A</td>" + searchResultsHtml += "<td>N/A</td>" } - the_html += "</tr>" + searchResultsHtml += "</tr>" } - the_html += "</tbody>"; - the_html += "</table>"; + searchResultsHtml += "</tbody>"; + searchResultsHtml += "</table>"; } else { - the_html = "<span>No users were found matching the entered criteria.</span>" + searchResultsHtml = "<span>No users were found matching the entered criteria.</span>" } - $('#user_results').html(the_html) + $('#user_results').html(searchResultsHtml) if (user_list.length > 0){ $('#users_table').dataTable({ 'order': [[1, "asc" ]], diff --git a/wqflask/wqflask/templates/admin/create_group.html b/wqflask/wqflask/templates/admin/create_group.html index 21ef5653..b1d214ea 100644 --- a/wqflask/wqflask/templates/admin/create_group.html +++ b/wqflask/wqflask/templates/admin/create_group.html @@ -6,7 +6,8 @@ <div class="page-header"> <h1>Create Group</h1> </div> - <form action="/groups/create" method="POST"> + <form action="{{ url_for('group_management.create_new_group') }}" + method="POST"> <input type="hidden" name="admin_emails_to_add" value=""> <input type="hidden" name="member_emails_to_add" value=""> <fieldset> @@ -73,17 +74,11 @@ </form> </div> - - <!-- End of body --> - {% endblock %} {% block js %} <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.js') }}"></script> <script language="javascript" type="text/javascript" src="/static/new/javascript/group_manager.js"></script> <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='js_alt/underscore.min.js') }}"></script> - - <script type="text/javascript" charset="utf-8"> - </script> {% endblock %} diff --git a/wqflask/wqflask/templates/admin/group_manager.html b/wqflask/wqflask/templates/admin/group_manager.html index 692a7abc..eedfe138 100644 --- a/wqflask/wqflask/templates/admin/group_manager.html +++ b/wqflask/wqflask/templates/admin/group_manager.html @@ -12,8 +12,15 @@ <h1>Manage Groups</h1> {% if admin_groups|length != 0 or member_groups|length != 0 %} <div style="display: inline;"> - <button type="button" id="create_group" class="btn btn-primary" data-url="/groups/create">Create Group</button> - <button type="button" id="remove_groups" class="btn btn-primary" data-url="/groups/remove">Remove Selected Groups</button> + <a href="{{ url_for('group_management.view_create_group_page') }}" target="_blank"> + <button type="button" class="btn btn-primary"> + Create Group + </button> + </a> + <button type="button" id="remove_groups" class="btn btn-primary" + data-url="{{ url_for('group_management.delete_groups') }}"> + Remove Selected Groups + </button> </div> {% endif %} </div> @@ -23,7 +30,11 @@ {% if admin_groups|length == 0 and member_groups|length == 0 %} <h4>You currently aren't a member or admin of any groups.</h4> <br> - <button type="button" id="create_group" class="btn btn-primary" data-url="/groups/create">Create a new group</button> + <a href="{{ url_for('group_management.view_create_group_page') }}" target="_blank"> + <button type="button" class="btn btn-primary"> + Create Group + </button> + </a> {% else %} <div style="margin-top: 20px;"><h2>Admin Groups</h2></div> <hr> @@ -47,11 +58,12 @@ <tr> <td><input type="checkbox" name="group_id" value="{{ group.id }}"></td> <td align="right">{{ loop.index }}</td> - <td><a href="/groups/view?id={{ group.id }}">{{ group.name }}</a></td> + {% set group_url = url_for('group_management.view_group', group_id=group.uuid) %} + <td><a href="{{ group_url }}">{{ group.name }}</a></td> <td align="right">{{ group.admins|length + group.members|length }}</td> <td>{{ group.created_timestamp }}</td> <td>{{ group.changed_timestamp }}</td> - <td>{{ group.id }}</td> + <td>{{ group.uuid }}</td> </tr> {% endfor %} </tbody> @@ -81,7 +93,8 @@ <tr> <td><input type="checkbox" name="read" value="{{ group.id }}"></td> <td>{{ loop.index }}</td> - <td><a href="/groups/view?id={{ group.id }}">{{ group.name }}</a></td> + {% set group_url = url_for('group_management.view_group', group_id=group.uuid) %} + <td><a href="{{ group_url }}">{{ group.name }}</a></td> <td>{{ group.admins|length + group.members|length }}</td> <td>{{ group.created_timestamp }}</td> <td>{{ group.changed_timestamp }}</td> @@ -119,11 +132,6 @@ return $("#groups_form").submit(); }; - $("#create_group").on("click", function() { - url = $(this).data("url") - return submit_special(url) - }); - $("#remove_groups").on("click", function() { url = $(this).data("url") groups = [] diff --git a/wqflask/wqflask/templates/admin/manage_resource.html b/wqflask/wqflask/templates/admin/manage_resource.html index 33a37594..64d4b6eb 100644 --- a/wqflask/wqflask/templates/admin/manage_resource.html +++ b/wqflask/wqflask/templates/admin/manage_resource.html @@ -1,76 +1,95 @@ {% extends "base.html" %} {% block title %}Resource Manager{% endblock %} -{% block css %} - <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTables/css/jquery.dataTables.css') }}" /> - <link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" /> -{% endblock %} {% block content %} <!-- Start of body --> - <div class="container"> - {{ flash_me() }} - <div class="page-header" style="display: inline-block;"> - <h1>Resource Manager</h1> - <h3>{% if owner_name is not none %}Current Owner: {{ owner_name }}{% endif %} {% if admin_status == "owner" %}<button id="change_owner" class="btn btn-danger" data-url="/resources/change_owner" style="margin-left: 20px;">Change Owner</button>{% endif %}</h3> - </div> - <form id="manage_resource" action="/resources/manage" method="POST"> - <input type="hidden" name="resource_id" value="{{ resource_id }}"> - <div style="min-width: 600px; max-width: 800px;"> +<div class="container"> + <section> + {{ flash_me() }} + {% set DATA_ACCESS = access_role.get('data') %} + {% set METADATA_ACCESS = access_role.get('metadata') %} + {% set ADMIN_STATUS = access_role.get('admin') %} + {% set ADMIN_STATUS = access_role.get('admin') %} + <h1>Resource Manager</h1> + {% if resource_info.get('owner_id') %} + {% set user_details = resource_info.get('owner_details') %} + <h3> + Current Owner: {{ user_details.get('full_name') }} + </h3> + {% if user_details.get('organization') %} + <h3> + Organization: {{ user_details.get('organization')}} + </h3> + {% endif %} + {% if DATA_ACCESS > DataRole.VIEW and ADMIN_STATUS > AdminRole.NOT_ADMIN %} + <a class="btn btn-danger" target="_blank" + href="/resource-management/resources/{{ resource_info.get('resource_id') }}/change-owner"> + Change Owner + </a> + {% endif %} + {% endif %} + </section> + + <section class="container" style="margin-top: 2em;"> + <form class="container-fluid" action="/resource-management/resources/{{ resource_info.get('resource_id') }}/make-public" method="POST"> + <input type="hidden" name="resource_id" value="{{ resource_info.get('resource_id') }}"> + <div> <fieldset> <div class="form-horizontal" style="width: 900px; margin-bottom: 50px;"> <div class="form-group" style="padding-left: 20px;"> <label for="group_name" class="col-xs-3" style="float: left; font-size: 18px;">Resource Name:</label> <div class="controls input-append col-xs-9" style="display: flex; padding-left: 20px; float: left;"> - {{ resource_info.name }} + {{ resource_info.get('name') }} </div> </div> - {% if admin_status == "owner" %} + {% if DATA_ACCESS > DataRole.VIEW and ADMIN_STATUS > AdminRole.NOT_ADMIN %} + {% set is_open_to_public = DataRole(resource_info.get('default_mask').get('data')) > DataRole.NO_ACCESS %} <div class="form-group" style="padding-left: 20px;"> <label for="user_email" class="col-xs-3" style="float: left; font-size: 18px;">Open to Public:</label> <div class="controls input-append col-xs-9" style="display: flex; padding-left: 20px; float: left;"> <label class="radio-inline"> - <input type="radio" name="open_to_public" value="True" {% if default_mask != 'no-access' %}checked{% endif %}> + <input type="radio" name="open_to_public" value="True" {{ 'checked' if is_open_to_public }}> Yes </label> <label class="radio-inline"> - <input type="radio" name="open_to_public" value="False" {% if default_mask == 'no-access' %}checked{% endif %}> + <input type="radio" name="open_to_public" value="False" {{ 'checked' if not is_open_to_public }}> No - </label> + </label> </div> </div> <div class="form-group" style="padding-left: 20px;"> <label class="col-xs-3" style="float: left; font-size: 18px;"></label> <div class="controls input-append col-xs-9" style="display: flex; padding-left: 20px; float: left;"> - <button id="save_changes" class="btn btn-primary" data-url="/resources/change_default_privileges">Save Changes</button> + <button id="save_changes" class="btn btn-primary" data-url="/resource-management/resources/change_default_privileges">Save Changes</button> </div> </div> {% endif %} </div> </fieldset> </div> - {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} + {% if ADMIN_STATUS > AdminRole.NOT_ADMIN %} <div style="min-width: 600px; max-width: 800px;"> <hr> <button id="add_group_to_resource" class="btn btn-primary" style="margin-bottom: 30px;" data-url="/resources/add_group">Add Group</button> <br> - {% if group_masks|length > 0 %} + {% if resource_info.get('group_masks', [])|length > 0 %} <h2>Current Group Permissions</h2> <hr> - <table id="groups_table" class="table-hover table-striped cell-border"> + <table id="groups_table" class="table table-hover table-striped cell-border"> <thead> <tr> + <th>Id</th> <th>Name</th> <th>Data</th> <th>Metadata</th> - <th>Admin</th> </tr> </thead> <tbody> - {% for key, value in group_masks.items() %} + {% for key, value in resource_info.get('group_masks').items() %} <tr> - <td>{{ value.name }}</td> + <td>{{ key }}</td> + <td>{{ value.group_name}}</td> <td>{{ value.data }}</td> <td>{{ value.metadata }}</td> - <td>{{ value.admin }}</td> </tr> {% endfor %} </tbody> @@ -81,28 +100,25 @@ </div> {% endif %} </form> - </div> - - - -<!-- End of body --> + </section> -{% endblock %} + <!-- End of body --> -{% block js %} + {% endblock %} + {% block js %} <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.dataTables.min.js') }}"></script> <script type="text/javascript" charset="utf-8"> - $('#add_group_to_resource, #save_changes, #change_owner').click(function(){ - url = $(this).data("url"); - $('#manage_resource').attr("action", url) - $('#manage_resource').submit() - }) + $('#add_group_to_resource, #save_changes, #change_owner').click(function(){ + url = $(this).data("url"); + $('#manage_resource').attr("action", url) + $('#manage_resource').submit() + }) - {% if group_masks|length > 0 %} - $('#groups_table').dataTable({ - 'sDom': 'tr', - }); - {% endif %} + {% if group_masks|length > 0 %} + $('#groups_table').dataTable({ + 'sDom': 'tr', + }); + {% endif %} </script> -{% endblock %} + {% endblock %} diff --git a/wqflask/wqflask/templates/admin/view_group.html b/wqflask/wqflask/templates/admin/view_group.html index 26692fe8..c88ce0e7 100644 --- a/wqflask/wqflask/templates/admin/view_group.html +++ b/wqflask/wqflask/templates/admin/view_group.html @@ -1,26 +1,30 @@ {% extends "base.html" %} {% block title %}View and Edit Group{% endblock %} {% block css %} - <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTables/css/jquery.dataTables.css') }}" /> - <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTablesExtensions/buttonStyles/css/buttons.dataTables.min.css') }}" /> - <link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" /> +<link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTables/css/jquery.dataTables.css') }}" /> +<link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTablesExtensions/buttonStyles/css/buttons.dataTables.min.css') }}" /> +<link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" /> {% endblock %} {% block content %} <!-- Start of body --> - <div class="container"> +{% set GROUP_URL = url_for('group_management.view_group', group_id=group_info.guid) %} +{% set UPDATE_GROUP_URL = url_for('group_management.update_group', group_id=group_info.guid) %} +<div class="container"> <div class="page-header"> <h1> - <span id="group_name">{{ group_info.name }}</span> - <input type="text" name="new_group_name" style="font-size: 20px; display: none; width: 500px;" class="form-control" placeholder="{{ group_info.name }}"> + <span id="group_name">Name: {{ group_info.name }}</span> + <input type="text" name="new_group_name" style="font-size: 20px; display: none; width: 500px;" class="form-control" placeholder="{{ group_info.name }}"> + {% if is_admin %} <button class="btn btn-default" style="display: inline;" id="change_group_name">Change Group Name</button> + {% endif %} </h1> - {% if user_is_admin == true %} + {% if is_admin %} <div style="display: inline;"> <button type="button" id="remove_users" class="btn btn-danger" data-url="/groups/remove_users">Remove Selected Users from Group</button> </div> {% endif %} </div> - <form id="group_form" action="/groups/view" method="POST"> + <form id="group_form" action="{{ UPDATE_GROUP_URL }}" method="POST"> <input type="hidden" name="group_id" value="{{ group_info.id }}"> <input type="hidden" name="selected_admin_ids" value=""> <input type="hidden" name="selected_member_ids" value=""> @@ -37,6 +41,9 @@ <th>Name</th> <th>Email Address</th> <th>Organization</th> + {% if is_admin %} + <th>UID</th> + {% endif %} </tr> </thead> <tbody> @@ -47,17 +54,20 @@ <td>{% if 'full_name' in admin %}{{ admin.full_name }}{% elif 'name' in admin %}{{ admin.name }}{% else %}N/A{% endif %}</td> <td>{% if 'email_address' in admin %}{{ admin.email_address }}{% else %}N/A{% endif %}</td> <td>{% if 'organization' in admin %}{{ admin.organization }}{% else %}N/A{% endif %}</td> + {% if is_admin %} + <td>{{admin.user_id}}</td> + {% endif %} </tr> {% endfor %} </tbody> </table> - {% if user_is_admin == true %} + {% if is_admin %} <div style="margin-top: 20px;"> <span>E-mail of user to add to admins (multiple e-mails can be added separated by commas):</span> <input type="text" size="60" name="admin_emails_to_add" placeholder="Enter E-mail(s)" value=""> </div> <div style="margin-bottom: 30px; margin-top: 20px;"> - <button type="button" id="add_admins" class="btn btn-primary" data-usertype="admin" data-url="/groups/add_admins">Add Admin(s)</button> + <button type="button" id="add_admins" class="btn btn-primary" data-usertype="admin" data-url="{{ UPDATE_GROUP_URL }}">Add Admin(s)</button> </div> {% endif %} </div> @@ -74,38 +84,50 @@ <th>Name</th> <th>Email Address</th> <th>Organization</th> + {% if is_admin %} + <th>UID</th> + {% endif %} </tr> </thead> <tbody> {% for member in members %} <tr> - <td style="text-align: center; padding: 0px 10px 2px 10px;"><input type="checkbox" name="member_id" value="{{ member.user_id }}"></td> + + <td style="text-align: center; padding: 0px 10px 2px 10px;"> + {% if is_admin %} + <input type="checkbox" name="member_id" value="{{ member.user_id }}"> + {% endif %} + </td> <td align="right">{{ loop.index }}</td> <td>{% if 'full_name' in member %}{{ member.full_name }}{% elif 'name' in admin %}{{ admin.name }}{% else %}N/A{% endif %}</td> <td>{% if 'email_address' in member %}{{ member.email_address }}{% else %}N/A{% endif %}</td> <td>{% if 'organization' in member %}{{ member.organization }}{% else %}N/A{% endif %}</td> + {% if is_admin %} + <td>{{ member }}</td> + {% endif %} + </tr> {% endfor %} </tbody> </table> - {% if user_is_admin == true %} + {% if is_admin %} <div style="margin-top: 20px;"> <span>E-mail of user to add to members (multiple e-mails can be added separated by commas):</span> <input type="text" size="60" name="member_emails_to_add" placeholder="Enter E-mail(s)" value=""> </div> <div style="margin-bottom: 30px; margin-top: 20px;"> - <button type="button" id="add_members" class="btn btn-primary" data-usertype="member" data-url="/groups/add_members">Add Member(s)</button> + <button type="button" id="add_members" class="btn btn-primary" data-usertype="member" data-url="{{ GROUP_URL }}">Add Member(s)</button> </div> {% endif %} {% else %} There are currently no members in this group. - {% if user_is_admin == true %} + {% if is_admin %} <div style="margin-top: 20px;"> <span>E-mail of user to add to members (multiple e-mails can be added separated by commas):</span> <input type="text" size="60" name="member_emails_to_add" placeholder="Enter E-mail(s)" value=""> </div> <div style="margin-bottom: 30px; margin-top: 20px;"> - <button type="button" id="add_members" class="btn btn-primary" data-usertype="member" data-url="/groups/add_members">Add Member(s)</button> + <button type="button" id="add_members" class="btn btn-primary" data-usertype="member" data-url="{{ GROUP_URL }}">Add Member(s)</button> </div> {% endif %} {% endif %} @@ -219,6 +241,7 @@ $("#add_admins, #add_members").on("click", function() { url = $(this).data("url"); + console.log(url) return submit_special(url) }); @@ -230,7 +253,7 @@ new_name = $('input[name=new_group_name]').val() $.ajax({ type: "POST", - url: "/groups/change_name", + url: "{{ GROUP_URL }} ", data: { group_id: $('input[name=group_id]').val(), new_name: new_name diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index 14e6bc88..ad2e3744 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -68,7 +68,7 @@ <a href="/help" class="dropdow-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Help <span class="caret"></a> <ul class="dropdown-menu"> <li><a href="{{ url_for('references_blueprint.references') }}">References</a></li> - <li><a href="/tutorials">Tutorials/Primers</a></li> + <li><a href="/tutorials">Webinars, Tutorials/Primers</a></li> <li><a href="{{ url_for('blogs_blueprint.blogs_list') }}">Blogs</a></li> <li><a href="{{ url_for('glossary_blueprint.glossary') }}">Glossary of Term</a></li> <li><a href="http://gn1.genenetwork.org/faq.html">FAQ</a></li> @@ -87,7 +87,7 @@ <li><a href="https://systems-genetics.org/">Systems Genetics PheWAS</a></li> <li><a href="http://ucscbrowser.genenetwork.org/">Genome Browser</a></li> <li><a href="http://power.genenetwork.org">BXD Power Calculator</a></li> - <li><a href="http://notebook.genenetwork.org/">Jupyter Notebook Launcher</a></li> + <li><a href="{{url_for('jupyter_notebooks.launcher')}}">Jupyter Notebooks</a></li> <li><a href="http://datafiles.genenetwork.org">Interplanetary File System</a></li> </ul> </li> @@ -159,17 +159,16 @@ </a>; June 15, 2001 as <a href="https://www.ncbi.nlm.nih.gov/pubmed/15043217">WebQTL</a>; and Jan 5, 2005 as GeneNetwork. </p> <p> - This site is currently operated by + This site is currently operated by <a href="mailto:rwilliams@uthsc.edu">Rob Williams</a>, - <a href="http://thebird.nl/">Pjotr Prins</a>, + <a href="https://thebird.nl/">Pjotr Prins</a>, <a href="http://www.senresearch.org">Saunak Sen</a>, <a href="mailto:zachary.a.sloan@gmail.com">Zachary Sloan</a>, <a href="mailto:acenteno@uthsc.edu">Arthur Centeno</a>, - <a href="mailto:cfische7@uthsc.edu">Christian Fischer</a> and <a href="mailto:bonfacemunyoki@gmail.com">Bonface Munyoki</a>. </p> - <p>Design and code by Pjotr Prins, Zach Sloan, Arthur Centeno, Christan Fischer, Bonface Munyoki, Danny Arends, Sam Ockman, Lei Yan, Xiaodong Zhou, Christian Fernandez, - Ning Liu, Rudi Alberts, Elissa Chesler, Sujoy Roy, Evan G. Williams, Alexander G. Williams, Kenneth Manly, Jintao Wang, Robert W. Williams, and + <p>Design and code by Pjotr Prins, Zach Sloan, Arthur Centeno, Christan Fischer, Bonface Munyoki, Danny Arends, Arun Isaac, Alex Mwangi, Fred Muriithi, Sam Ockman, Lei Yan, Xiaodong Zhou, Christian Fernandez, + Ning Liu, Rudi Alberts, Elissa Chesler, Sujoy Roy, Evan G. Williams, Alexander G. Williams, Kenneth Manly, Jintao Wang, Robert W. Williams, and <!--<a href="http://genenetwork.org/credit.html">colleagues</a>.</p>--> <a href="/credits">colleagues</a>.</p> <br /> @@ -182,7 +181,12 @@ </li> <li> <a href="https://www.nigms.nih.gov/">NIGMS</a> - Systems Genetics and Precision Medicine Project (R01 GM123489, 2017-2021) + Systems Genetics and Precision Medicine Project (R01 GM123489, 2017-2026) + </li> + <li> + <a href="https://www.nsf.gov/awardsearch/showAward?AWD_ID=2118709">NSF</a> + Panorama: Integrated Rack-Scale Acceleration for Computational Pangenomics + (PPoSS 2118709, 2021-2016) </li> <li> <a href="https://www.drugabuse.gov/">NIDA</a> @@ -223,6 +227,7 @@ <a href="http://www.neuinfo.org" target="_blank"> <img src="/static/new/images/Nif.png" alt="Registered with Nif" border="0"> </a> + <script type="text/javascript" src="//rf.revolvermaps.com/0/0/8.js?i=526mdlpknyd&m=0&c=ff0000&cr1=ffffff&f=arial&l=33" async="async"></script> </div> </div> </footer> @@ -256,7 +261,8 @@ }) </script> <script src="{{ url_for('js', filename='jquery-cookie/jquery.cookie.js') }}" type="text/javascript"></script> - <script src="{{ url_for('js', filename='jquery-ui/jquery-ui.min.js') }}" type="text/javascript"></script> + <script src="https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.js"></script> + <!-- <script src="{{ url_for('js', filename='jquery-ui/jquery-ui.min.js') }}" type="text/javascript"></script> --> <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='colorbox/jquery.colorbox-min.js') }}"></script> diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index a3090bcf..0ded66a6 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -36,6 +36,28 @@ <div> <br /> + <form id="heatmaps_form"> + <fieldset> + <legend>Heatmap Orientation</legend> + <label for="heatmap-orient-vertical">Vertical</label> + <input id="heatmap-orient-vertical" + type="radio" + name="vertical" + value="true" /> + <label for="heatmap-orient-horizontal">Horizontal</label> + <input id="heatmap-orient-horizontal" + type="radio" + name="vertical" + value="false" /> + </fieldset> + <button id="clustered-heatmap" + class="btn btn-primary" + data-url="{{heatmap_data_url}}" + title="Generate heatmap from this collection"> + Generate Heatmap + </button> + </form> + <div class="collection-table-options"> <form id="export_form" method="POST" action="/export_traits_csv"> <button class="btn btn-default" id="select_all" type="button"><span class="glyphicon glyphicon-ok"></span> Select All</button> @@ -52,6 +74,8 @@ <button id="delete" class="btn btn-danger submit_special" data-url="/collections/delete" type="button" title="Delete this collection" > Delete Collection</button> </form> </div> + <div id="clustered-heatmap-image-area"> + </div> <div style="margin-top: 10px; margin-bottom: 5px;"> <b>Show/Hide Columns:</b> </div> @@ -139,6 +163,8 @@ <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/buttons/js/buttons.colVis.min.js') }}"></script> <script type="text/javascript" src="/static/new/javascript/search_results.js"></script> + <script type="text/javascript" src="{{ url_for('js', filename='plotly/plotly.min.js') }}"></script> + <script language="javascript" type="text/javascript"> $(document).ready( function () { @@ -247,6 +273,81 @@ $("#make_default").on("click", function(){ make_default(); }); + + $("#heatmaps_form").submit(function(e) { + e.preventDefault(); + }); + + function clear_heatmap_area() { + area = document.getElementById("clustered-heatmap-image-area"); + area.querySelectorAll("*").forEach(function(child) { + child.remove(); + }); + } + + function generate_progress_indicator() { + count = 0; + default_message = "Computing" + return function() { + message = default_message; + if(count >= 10) { + count = 0; + } + for(i = 0; i < count; i++) { + message = message + " ."; + } + clear_heatmap_area(); + $("#clustered-heatmap-image-area").append( + '<div class="alert alert-info"' + + ' style="font-weigh: bold; font-size: 150%;">' + + message + '</div>'); + count = count + 1; + }; + } + + function display_clustered_heatmap(heatmap_data) { + clear_heatmap_area(); + image_area = document.getElementById("clustered-heatmap-image-area") + Plotly.newPlot(image_area, heatmap_data) + } + + function process_clustered_heatmap_error(xhr, status, error) { + clear_heatmap_area() + $("#clustered-heatmap-image-area").append( + $( + '<div class="alert alert-danger">ERROR: ' + + xhr.responseJSON.message + + '</div>')); + } + + $("#clustered-heatmap").on("click", function() { + clear_heatmap_area(); + intv = window.setInterval(generate_progress_indicator(), 300); + vert_element = document.getElementById("heatmap-orient-vertical"); + vert_true = vert_element == null ? false : vert_element.checked; + heatmap_url = $(this).attr("data-url") + traits = $(".trait_checkbox:checked").map(function() { + return this.value + }).get(); + $.ajax({ + type: "POST", + url: heatmap_url, + contentType: "application/json", + data: JSON.stringify({ + "traits_names": traits, + "vertical": vert_true + }), + dataType: "JSON", + success: function(data, status, xhr) { + window.clearInterval(intv); + display_clustered_heatmap(data); + }, + error: function(xhr, status, error) { + window.clearInterval(intv); + process_clustered_heatmap_error(xhr, status, error); + } + }); + }); }); </script> diff --git a/wqflask/wqflask/templates/correlation_matrix.html b/wqflask/wqflask/templates/correlation_matrix.html index 8275f1dd..3da6981c 100644 --- a/wqflask/wqflask/templates/correlation_matrix.html +++ b/wqflask/wqflask/templates/correlation_matrix.html @@ -73,8 +73,13 @@ <br> {% if pca_works == "True" %} <h2>PCA Traits</h2> -<div style="margin-bottom: 20px; overflow:hidden;"> +<div style="margin-bottom: 20px; overflow:hidden; width: 450px;"> <table id="pca_table" class="table-hover table-striped cell-border" id='trait_table' style="float: left;"> + <colgroup> + <col span="1" style="width: 30px;"> + <col span="1" style="width: 50px;"> + <col span="1"> + </colgroup> <thead> <tr> <th></th> diff --git a/wqflask/wqflask/templates/display_diffs.html b/wqflask/wqflask/templates/display_diffs.html new file mode 100644 index 00000000..e787e468 --- /dev/null +++ b/wqflask/wqflask/templates/display_diffs.html @@ -0,0 +1,95 @@ +{% extends "base.html" %} +{% block title %}Trait Submission{% endblock %} + +{% block css %} +<link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTables/css/jquery.dataTables.css') }}" /> +{% endblock %} + +{% block content %} +<!-- Start of body --> +<div class="container"> + {% set additions = diff.get("Additions") %} + {% set modifications = diff.get("Modifications") %} + {% set deletions = diff.get("Deletions") %} + + {% if additions %} + <h2>Additions Data:</h2> + <div class="row"> + <div class="col-md-8"> + <table class="table-responsive table-hover table-striped cell-border" id="table-additions"> + <thead> + <th scope="col">Added Data</</th> + </thead> + <tbody> + {% for data in additions %} + <tr> + <td>{{ data }}</td> + </tr> + {% endfor %} + </tbody> + </table> + </div> + </div> + {% endif %} + + {% if modifications %} + <h2>Modified Data:</h2> + + <div class="row"> + <div class="col-md-8"> + <table class="table-responsive table-hover table-striped cell-border" id="table-modifications"> + <thead> + <th scope="col">Original</</th> + <th scope="col">Current</</th> + <th scope="col">Diff</</th> + </thead> + <tbody> + {% for data in modifications %} + <tr> + <td>{{ data.get("Original") }}</td> + <td>{{ data.get("Current") }}</td> + <td><pre>{{data.get("Diff")}}</pre></td> + </tr> + {% endfor %} + </tbody> + </table> + </div> + </div> + {% endif %} + + {% if deletions %} + <h2>Deleted Data:</h2> + <div class="row"> + <div class="col-md-8"> + <table class="table-responsive table-hover table-striped cell-border" id="table-deletions"> + <thead> + <th scope="col">Deleted Data</</th> + </thead> + <tbody> + {% for data in deletions %} + <tr> + <td>{{ data }}</td> + </tr> + {% endfor %} + </tbody> + </table> + </div> + </div> + {% endif %} + +</div> +{%endblock%} + +{% block js %} +<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.js') }}"></script> +<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.dataTables.min.js') }}"></script> +<script language="javascript" type="text/javascript"> + gn_server_url = "{{ gn_server_url }}"; + + $(document).ready( function() { + $('#table-additions').dataTable(); + $('#table-modifications').dataTable(); + $('#table-deletions').dataTable(); + }); +</script> +{% endblock %} diff --git a/wqflask/wqflask/templates/display_files.html b/wqflask/wqflask/templates/display_files.html new file mode 100644 index 00000000..5fad5d14 --- /dev/null +++ b/wqflask/wqflask/templates/display_files.html @@ -0,0 +1,127 @@ +{% extends "base.html" %} +{% block title %}Trait Submission{% endblock %} + +{% block css %} +<link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTables/css/jquery.dataTables.css') }}" /> +{% endblock %} + +{% block content %} +<!-- Start of body --> +{% with messages = get_flashed_messages(with_categories=true) %} +{% if messages %} +{% for category, message in messages %} +<div class="container-fluid bg-{{ category }}"> + <p>{{ message }}</p> +</div> +{% endfor %} +{% endif %} +{% endwith %} + +<div class="container"> + {% if waiting %} + <h2>Files for approval:</h2> + <div class="row"> + <div class="col-md-7"> + <table class="table table-hover table-striped cell-border"> + <thead> + <th scope="col">Resource Id</</th> + <th scope="col">Author</th> + <th scope="col">TimeStamp</th> + <th scope="col"></th> + <th scope="col"></th> + </thead> + <tbody> + {% for data in waiting %} + <tr> + {% set file_url = url_for('metadata_edit.show_diff', name=data.get('file_name')) %} + <td><a href="{{ file_url }}" target="_blank">{{ data.get("resource_id") }}</a></td> + <td>{{ data.get("author")}}</td> + <td>{{ data.get("time_stamp")}}</td> + {% if data.get("roles").get("admin") > AdminRole.EDIT_ACCESS %} + {% set reject_url = url_for('metadata_edit.reject_data', resource_id=data.get('resource_id'), file_name=data.get('file_name')) %} + {% set approve_url = url_for('metadata_edit.approve_data', resource_id=data.get('resource_id'), file_name=data.get('file_name')) %} + <td> + <button type="button" + class="btn btn-secondary btn-sm"> + <a href="{{ reject_url }}">Reject</a> + </button> + </td> + <td> + <button type="button" + class="btn btn-warning btn-sm"> + <a href="{{ approve_url }}">Approve</a> + </button> + </td> + {% endif %} + </tr> + {% endfor %} + </tbody> + </table> + </div> + </div> + {% endif %} + + {% if approved %} + <h2>Approved Data:</h2> + <div class="row"> + <div class="col-md-8"> + <table class="table-responsive table-hover table-striped cell-border" id="table-approved"> + <thead> + <th scope="col">Resource Id</</th> + <th scope="col">Author</th> + <th scope="col">TimeStamp</th> + </thead> + <tbody> + {% for data in approved %} + <tr> + {% set file_url = url_for('metadata_edit.show_diff', name=data.get('file_name')) %} + <td><a href="{{ file_url }}" target="_blank">{{ data.get("resource_id") }}</a></td> + <td>{{ data.get("author")}}</td> + <td>{{ data.get("time_stamp")}}</td> + </tr> + {% endfor %} + </tbody> + </table> + </div> + </div> + {% endif %} + + {% if rejected %} + <h2>Rejected Files:</h2> + <div class="row"> + <div class="col-md-8"> + <table class="table-responsive table-hover table-striped cell-border" id="table-rejected"> + <thead> + <th scope="col">Resource Id</</th> + <th scope="col">Author</th> + <th scope="col">TimeStamp</th> + </thead> + <tbody> + {% for data in rejected %} + <tr> + {% set file_url = url_for('metadata_edit.show_diff', name=data.get('file_name')) %} + <td><a href="{{ file_url }}" target="_blank">{{ data.get("resource_id") }}</a></td> + <td>{{ data.get("author")}}</td> + <td>{{ data.get("time_stamp")}}</td> + </tr> + {% endfor %} + </tbody> + </table> + </div> + </div> + {% endif %} +</div> +{%endblock%} + +{% block js %} +<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.js') }}"></script> +<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.dataTables.min.js') }}"></script> +<script language="javascript" type="text/javascript"> + gn_server_url = "{{ gn_server_url }}"; + + $(document).ready( function() { + $('#table-approved').dataTable(); + $('#table-rejected').dataTable(); + }); +</script> +{% endblock %} diff --git a/wqflask/wqflask/templates/display_files_admin.html b/wqflask/wqflask/templates/display_files_admin.html deleted file mode 100644 index 4b4babc4..00000000 --- a/wqflask/wqflask/templates/display_files_admin.html +++ /dev/null @@ -1,32 +0,0 @@ -{% extends "base.html" %} -{% block title %}Trait Submission{% endblock %} -{% block content %} -<!-- Start of body --> -{% with messages = get_flashed_messages(with_categories=true) %} -{% if messages %} -{% for category, message in messages %} -<div class="container-fluid bg-{{ category }}"> - <p>{{ message }}</p> -</div> -{% endfor %} -{% endif %} -{% endwith %} -Show files for approval - -<div> - <ul> - {% for file in files %} - <li><a href="/display-file/{{ file }}" target="_blank">{{ file }}</a><br/> - <button><a href="/data-samples/approve/{{ file }}">Approve</a></button> - <button><a href="/data-samples/reject/{{ file }}">Reject</a></button></li> - {% endfor %} - </ul> -</div> -{%endblock%} - -{% block js %} -<script> - gn_server_url = "{{ gn_server_url }}"; - -</script> -{% endblock %} diff --git a/wqflask/wqflask/templates/display_files_user.html b/wqflask/wqflask/templates/display_files_user.html deleted file mode 100644 index b6bab709..00000000 --- a/wqflask/wqflask/templates/display_files_user.html +++ /dev/null @@ -1,31 +0,0 @@ -{% extends "base.html" %} -{% block title %}Trait Submission{% endblock %} -{% block content %} -<!-- Start of body --> -{% with messages = get_flashed_messages(with_categories=true) %} -{% if messages %} -{% for category, message in messages %} -<div class="container-fluid bg-{{ category }}"> - <p>{{ message }}</p> -</div> -{% endfor %} -{% endif %} -{% endwith %} -Show files for approval - -<div> - <ul> - {% for file in files %} - <li><a href="/display-file/{{ file }}" target="_blank">{{ file }}</a><br/> - <button><a href="/data-samples/reject/{{ file }}">Reject</a></button></li> - {% endfor %} - </ul> -</div> -{%endblock%} - -{% block js %} -<script> - gn_server_url = "{{ gn_server_url }}"; - -</script> -{% endblock %} diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 7a841793..0daea51d 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -62,8 +62,7 @@ </div> {% endif %} - -<form id="edit-form" class="form-horizontal" method="post" action="/trait/update" enctype=multipart/form-data> +<form id="edit-form" class="form-horizontal" method="post" action="/datasets/{{dataset_id}}/traits/{{ publish_xref.id_ }}?resource-id={{ resource_id }}" enctype='multipart/form-data'> <h2 class="text-center">Trait Information:</h2> <div class="form-group"> <label for="pubmed-id" class="col-sm-2 control-label">Pubmed ID:</label> @@ -218,7 +217,7 @@ </div> </div> <div style="margin-left: 13%;"> - <a href="/trait/{{ publish_xref.id_ }}/sampledata/{{ publish_xref.phenotype_id }}" class="btn btn-link btn-sm"> + <a href="/datasets/{{ publish_xref.id_ }}/traits/{{ publish_xref.phenotype_id }}/csv?resource-id={{ resource_id }}" class="btn btn-link btn-sm"> Sample Data(CSV Download) </a> </div> @@ -226,7 +225,6 @@ <input type = "file" class="col-sm-4 control-label" name = "file" /> </div> <div class="controls center-block" style="width: max-content;"> - <input name="dataset-name" class="changed" type="hidden" value="{{ publish_xref.id_ }}"/> <input name="inbred-set-id" class="changed" type="hidden" value="{{ publish_xref.inbred_set_id }}"/> <input name="phenotype-id" class="changed" type="hidden" value="{{ publish_xref.phenotype_id }}"/> <input name="comments" class="changed" type="hidden" value="{{ publish_xref.comments }}"/> diff --git a/wqflask/wqflask/templates/edit_probeset.html b/wqflask/wqflask/templates/edit_probeset.html index 85d49561..ab91b701 100644 --- a/wqflask/wqflask/templates/edit_probeset.html +++ b/wqflask/wqflask/templates/edit_probeset.html @@ -9,52 +9,52 @@ Submit Trait | Reset <div class="container"> <details class="col-sm-12 col-md-10 col-lg-12"> - <summary> - <h2>Update History</h2> - </summary> - <table class="table"> - <tbody> - <tr> - <th>Timestamp</th> - <th>Editor</th> - <th>Field</th> - <th>Diff</th> - </tr> - {% set ns = namespace(display_cell=True) %} + <summary> + <h2>Update History</h2> + </summary> + <table class="table"> + <tbody> + <tr> + <th>Timestamp</th> + <th>Editor</th> + <th>Field</th> + <th>Diff</th> + </tr> + {% set ns = namespace(display_cell=True) %} - {% for timestamp, group in diff %} - {% set ns.display_cell = True %} - {% for i in group %} - <tr> - {% if ns.display_cell and i.timestamp == timestamp %} + {% for timestamp, group in diff %} + {% set ns.display_cell = True %} + {% for i in group %} + <tr> + {% if ns.display_cell and i.timestamp == timestamp %} - {% set author = i.author %} - {% set timestamp_ = i.timestamp %} + {% set author = i.author %} + {% set timestamp_ = i.timestamp %} - {% else %} + {% else %} - {% set author = "" %} - {% set timestamp_ = "" %} + {% set author = "" %} + {% set timestamp_ = "" %} - {% endif %} - <td>{{ timestamp_ }}</td> - <td>{{ author }}</td> - <td>{{ i.diff.field }}</td> - <td><pre>{{ i.diff.diff }}</pre></td> - {% set ns.display_cell = False %} - </tr> - {% endfor %} - {% endfor %} - </tbody> - </table> + {% endif %} + <td>{{ timestamp_ }}</td> + <td>{{ author }}</td> + <td>{{ i.diff.field }}</td> + <td><pre>{{ i.diff.diff }}</pre></td> + {% set ns.display_cell = False %} + </tr> + {% endfor %} + {% endfor %} + </tbody> + </table> </details> </div> {% endif %} -<form id="edit-form" class="form-horizontal" method="post" action="/probeset/update"> - <h2 class="text-center">Probeset Information:</h2> +<form id="edit-form" class="form-horizontal" method="post" action="/datasets/traits/{{ name }}?resource-id={{ resource_id }}"> + <h2 class="text-center">Probeset Information:</h2> <div class="form-group"> <label for="symbol" class="col-sm-2 control-label">Symbol:</label> <div class="col-sm-4"> diff --git a/wqflask/wqflask/templates/gsearch_gene.html b/wqflask/wqflask/templates/gsearch_gene.html index 5549ac8a..69281ec5 100644 --- a/wqflask/wqflask/templates/gsearch_gene.html +++ b/wqflask/wqflask/templates/gsearch_gene.html @@ -2,6 +2,7 @@ {% block title %}Search Results{% endblock %} {% block css %} <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTables/css/jquery.dataTables.css') }}" /> + <link href="https://code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css" rel="stylesheet" type="text/css" /> <link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" /> {% endblock %} {% block content %} @@ -31,7 +32,7 @@ </form> <br /> <br /> - <div style="min-width: 2000px; width: 100%;"> + <div id="table_container" style="width: 2000px;"> <table id="trait_table" class="table-hover table-striped cell-border" style="float: left;"> <tbody> <td colspan="100%" align="center"><br><b><font size="15">Loading...</font></b><br></td> @@ -54,6 +55,7 @@ <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/colReorder/js/dataTables.colReorder.js') }}"></script> <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/colResize/dataTables.colResize.js') }}"></script> <script language="javascript" type="text/javascript" src="/static/new/javascript/search_results.js"></script> + <script language="javascript" type="text/javascript" src="/static/new/javascript/table_functions.js"></script> <script type='text/javascript'> var getParams = function(url) { @@ -73,51 +75,152 @@ <script type="text/javascript" charset="utf-8"> $(document).ready( function () { - - $('#trait_table tr').click(function(event) { - if (event.target.type !== 'checkbox') { - $(':checkbox', this).trigger('click'); - } - }); + var tableId = "trait_table"; - function change_buttons() { - buttons = ["#add", "#remove"]; - num_checked = $('.trait_checkbox:checked').length; - if (num_checked === 0) { - for (_i = 0, _len = buttons.length; _i < _len; _i++) { - button = buttons[_i]; - $(button).prop("disabled", true); - } - } else { - for (_j = 0, _len2 = buttons.length; _j < _len2; _j++) { - button = buttons[_j]; - $(button).prop("disabled", false); - } + var width_change = 0; //ZS: For storing the change in width so overall table width can be adjusted by that amount + + columnDefs = [ + { + 'orderDataType': "dom-checkbox", + 'width': "5px", + 'data': null, + 'targets': 0, + 'render': function(data, type, row, meta) { + return '<input type="checkbox" name="searchResult" class="trait_checkbox checkbox" value="' + data.hmac + '">' } - //}); - if ($(this).is(":checked")) { - if (!$(this).closest('tr').hasClass('selected')) { - $(this).closest('tr').addClass('selected') - } + }, + { + 'title': "Index", + 'type': "natural", + 'width': "30px", + 'targets': 1, + 'data': "index" + }, + { + 'title': "Record", + 'type': "natural", + 'orderDataType': "dom-inner-text", + 'width': "60px", + 'data': null, + 'targets': 2, + 'render': function(data, type, row, meta) { + return '<a target="_blank" href="/show_trait?trait_id=' + data.name + '&dataset=' + data.dataset + '">' + data.name + '</a>' } - else { - if ($(this).closest('tr').hasClass('selected')) { - $(this).closest('tr').removeClass('selected') - } + }, + { + 'title': "Species", + 'type': "natural", + 'width': "60px", + 'targets': 3, + 'data': "species" + }, + { + 'title': "Group", + 'type': "natural", + 'width': "150px", + 'targets': 4, + 'data': "group" + }, + { + 'title': "Tissue", + 'type': "natural", + 'width': "150px", + 'targets': 5, + 'data': "tissue" + }, + { + 'title': "Dataset", + 'type': "natural", + 'targets': 6, + 'width': "320px", + 'data': "dataset_fullname" + }, + { + 'title': "Symbol", + 'type': "natural", + 'width': "60px", + 'targets': 7, + 'data': "symbol" + }, + { + 'title': "Description", + 'type': "natural", + 'data': null, + 'width': "120px", + 'targets': 8, + 'render': function(data, type, row, meta) { + try { + return decodeURIComponent(escape(data.description)) + } catch(err) { + return escape(data.description) + } } - } + }, + { + 'title': "Location", + 'type': "natural-minus-na", + 'width': "125px", + 'targets': 9, + 'data': "location_repr" + }, + { + 'title': "Mean", + 'type': "natural-minus-na", + 'orderSequence': [ "desc", "asc"], + 'width': "30px", + 'targets': 10, + 'data': "mean" + }, + { + 'title': "Max<br>LRS<a href=\"{{ url_for('glossary_blueprint.glossary') }}#LRS\" target=\"_blank\" style=\"color: white;\"><sup>?</sup></a>", + 'type': "natural-minus-na", + 'width': "60px", + 'targets': 11, + 'data': "LRS_score_repr", + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "Max LRS Location", + 'type': "natural-minus-na", + 'width': "125px", + 'targets': 12, + 'data': "max_lrs_text" + }, + { + 'title': "Additive<br>Effect<a href=\"{{ url_for('glossary_blueprint.glossary') }}#A\" target=\"_blank\" style=\"color: white;\"><sup>?</sup></a>", + 'type': "natural-minus-na", + 'width': "50px", + 'targets': 13, + 'data': "additive", + 'orderSequence': [ "desc", "asc"] + } + ] + + loadDataTable(true); - var the_table = $('#trait_table').DataTable( { + function loadDataTable(first_run=false){ + + if (!first_run){ + setUserColumnsDefWidths(tableId); + } + + table_settings = { 'drawCallback': function( settings ) { - $('#trait_table tr').click(function(event) { - if (event.target.type !== 'checkbox' && event.target.tagName.toLowerCase() !== 'a') { - $(':checkbox', this).trigger('click'); - } - }); - $('.trait_checkbox:checkbox').on("change", change_buttons); + $('#' + tableId + ' tr').off().on("click", function(event) { + if (event.target.type !== 'checkbox' && event.target.tagName.toLowerCase() !== 'a') { + var obj =$(this).find('input'); + obj.prop('checked', !obj.is(':checked')); + } + if ($(this).hasClass("selected") && event.target.tagName.toLowerCase() !== 'a'){ + $(this).removeClass("selected") + } else if (event.target.tagName.toLowerCase() !== 'a') { + $(this).addClass("selected") + } + change_buttons() + }); }, 'createdRow': function ( row, data, index ) { - $('td', row).eq(0).attr("style", "text-align: center; padding: 0px 10px 2px 13px;"); + $('td', row).eq(0).attr("style", "text-align: center; padding: 0px 10px 2px 10px;"); $('td', row).eq(1).attr("align", "right"); $('td', row).eq(4).attr('title', $('td', row).eq(4).text()); if ($('td', row).eq(4).text().length > 30) { @@ -155,144 +258,60 @@ $('td', row).eq(13).attr('data-export', $('td', row).eq(13).text()); }, 'data': trait_list, - 'columns': [ - { - 'orderDataType': "dom-checkbox", - 'width': "10px", - 'data': null, - 'render': function(data, type, row, meta) { - return '<input type="checkbox" name="searchResult" class="trait_checkbox checkbox" value="' + data.hmac + '">' - } - }, - { - 'title': "Index", - 'type': "natural", - 'width': "30px", - 'data': "index" - }, - { - 'title': "Record", - 'type': "natural", - 'orderDataType': "dom-inner-text", - 'width': "60px", - 'data': null, - 'render': function(data, type, row, meta) { - return '<a target="_blank" href="/show_trait?trait_id=' + data.name + '&dataset=' + data.dataset + '">' + data.name + '</a>' - } - }, - { - 'title': "Species", - 'type': "natural", - 'width': "60px", - 'data': "species" - }, - { - 'title': "Group", - 'type': "natural", - 'width': "150px", - 'data': "group" - }, - { - 'title': "Tissue", - 'type': "natural", - 'width': "150px", - 'data': "tissue" - }, - { - 'title': "Dataset", - 'type': "natural", - 'data': "dataset_fullname" - }, - { - 'title': "Symbol", - 'type': "natural", - 'width': "60px", - 'data': "symbol" - }, - { - 'title': "Description", - 'type': "natural", - 'data': null, - 'render': function(data, type, row, meta) { - try { - return decodeURIComponent(escape(data.description)) - } catch(err) { - return escape(data.description) - } - } - }, - { - 'title': "Location", - 'type': "natural-minus-na", - 'width': "125px", - 'data': "location_repr" - }, - { - 'title': "Mean", - 'type': "natural-minus-na", - 'orderSequence': [ "desc", "asc"], - 'width': "30px", - 'data': "mean" - }, - { - 'title': "Max<br>LRS<a href=\"{{ url_for('glossary_blueprint.glossary') }}#LRS\" target=\"_blank\" style=\"color: white;\"><sup>?</sup></a>", - 'type': "natural-minus-na", - 'width': "60px", - 'data': "LRS_score_repr", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "Max LRS Location", - 'type': "natural-minus-na", - 'width': "125px", - 'data': "max_lrs_text" - }, - { - 'title': "Additive<br>Effect<a href=\"{{ url_for('glossary_blueprint.glossary') }}#A\" target=\"_blank\" style=\"color: white;\"><sup>?</sup></a>", - 'type': "natural-minus-na", - 'width': "50px", - 'data': "additive", - 'orderSequence': [ "desc", "asc"] - } - ], + 'columns': columnDefs, "order": [[1, "asc" ]], 'sDom': "iti", - "autoWidth": true, + "destroy": true, + "deferRender": true, "bSortClasses": false, - 'processing': true, {% if trait_count > 20 %} "scrollY": "100vh", "scroller": true, - "scrollCollapse": true + "scrollCollapse": true, {% else %} - "iDisplayLength": -1 + "iDisplayLength": -1, {% endif %} - } ); + "initComplete": function (settings) { + //Add JQueryUI resizable functionality to each th in the ScrollHead table + $('#' + tableId + '_wrapper .dataTables_scrollHead thead th').resizable({ + handles: "e", + alsoResize: '#' + tableId + '_wrapper .dataTables_scrollHead table', //Not essential but makes the resizing smoother + resize: function( event, ui ) { + width_change = ui.size.width - ui.originalSize.width; + }, + stop: function () { + saveColumnSettings(tableId, trait_table); + loadDataTable(); + } + }); + } + } - $('#trait_table').append( - '<tfoot>' + - '<tr>' + - '<th></th>' + - '<th>Index</th>' + - '<th>Record ID</th>' + - '<th>Species</th> ' + - '<th>Group</th>' + - '<th>Tissue</th>' + - '<th>Dataset</th>' + - '<th>Symbol</th>' + - '<th>Description</th>' + - '<th>Location</th>' + - '<th>Mean</th>' + - '<th>Max LRS <a href="{{ url_for('glossary_blueprint.glossary') }}#LRS" target="_blank" style="color: white;"><sup>?</sup></a></th>' + - '<th>Max LRS Location</th>' + - '<th>Additive Effect <a href="{{ url_for('glossary_blueprint.glossary') }}#A" target="_blank" style="color: white;"><sup>?</sup></a></th>' + - '</tr>' + - '</tfoot>' - ); + if (!first_run){ + table_settings['autoWidth'] = false; + $('#table_container').css("width", String($('#trait_table').width() + width_change {% if trait_list|length > 20 %}+ 17{% endif %}) + "px"); //ZS : Change the container width by the change in width of the adjusted column, so the overall table size adjusts properly + } + + let checked_rows = get_checked_rows(tableId); + trait_table = $('#' + tableId).DataTable(table_settings); + if (checked_rows.length > 0){ + recheck_rows(trait_table, checked_rows); + } + + if (first_run){ + {% if trait_list|length > 20 %} + $('#table_container').css("width", String($('#trait_table').width() + 17) + "px"); + {% endif %} + trait_table.draw(); + } + } + + $('#redraw').click(function() { + var table = $('#' + tableId).DataTable(); + table.colReorder.reset() + }); - the_table.draw(); }); </script> - <script language="javascript" type="text/javascript" src="/static/new/javascript/search_results.js"></script> {% endblock %} diff --git a/wqflask/wqflask/templates/gsearch_pheno.html b/wqflask/wqflask/templates/gsearch_pheno.html index 89316cbc..7abdb222 100644 --- a/wqflask/wqflask/templates/gsearch_pheno.html +++ b/wqflask/wqflask/templates/gsearch_pheno.html @@ -2,6 +2,7 @@ {% block title %}Search Results{% endblock %} {% block css %} <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTables/css/jquery.dataTables.css') }}" /> + <link href="https://code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css" rel="stylesheet" type="text/css" /> <link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" /> {% endblock %} {% block content %} @@ -21,8 +22,8 @@ <button class="btn btn-default" id="deselect_all"><span class="glyphicon glyphicon-remove"></span> Deselect All</button> <button class="btn btn-default" id="invert"><span class="glyphicon glyphicon-resize-vertical"></span> Invert</button> <button class="btn btn-success" id="add" disabled ><span class="glyphicon glyphicon-plus-sign"></span> Add</button> - <input type="text" id="searchbox" class="form-control" style="width: 200px; display: inline;" placeholder="Search This Table For ..."> - <input type="text" id="select_top" class="form-control" style="width: 200px; display: inline;" placeholder="Select Top ..."> + <input type="text" id="searchbox" class="form-control" style="width: 180px; display: inline;" placeholder="Search This Table For ..."> + <input type="text" id="select_top" class="form-control" style="width: 120px; display: inline;" placeholder="Select Top ..."> <form id="export_form" method="POST" action="/export_traits_csv" style="display: inline;"> <input type="hidden" name="headers" id="headers" value="{% for field in header_fields %}{{ field }},{% endfor %}"> <input type="hidden" name="database_name" id="database_name" value="None"> @@ -31,7 +32,7 @@ </form> <br /> <br /> - <div style="min-width: 2000px; width: 100%;"> + <div id="table_container" style="width: 2000px;"> <table id="trait_table" class="table-hover table-striped cell-border" style="float: left;"> <tbody> <td colspan="100%" align="center"><br><b><font size="15">Loading...</font></b><br></td> @@ -54,6 +55,7 @@ <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/colReorder/js/dataTables.colReorder.js') }}"></script> <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/colResize/dataTables.colResize.js') }}"></script> <script language="javascript" type="text/javascript" src="/static/new/javascript/search_results.js"></script> + <script language="javascript" type="text/javascript" src="/static/new/javascript/table_functions.js"></script> <script type='text/javascript'> var getParams = function(url) { @@ -73,218 +75,233 @@ <script type="text/javascript" charset="utf-8"> $(document).ready( function () { + var tableId = "trait_table"; - $('#trait_table tr').click(function(event) { - if (event.target.type !== 'checkbox') { - $(':checkbox', this).trigger('click'); - } - }); + var width_change = 0; //ZS: For storing the change in width so overall table width can be adjusted by that amount - function change_buttons() { - buttons = ["#add", "#remove"]; - num_checked = $('.trait_checkbox:checked').length; - if (num_checked === 0) { - for (_i = 0, _len = buttons.length; _i < _len; _i++) { - button = buttons[_i]; - $(button).prop("disabled", true); - } + columnDefs = [ + { + 'data': null, + 'orderDataType': "dom-checkbox", + 'width': "10px", + 'targets': 0, + 'render': function(data, type, row, meta) { + return '<input type="checkbox" name="searchResult" class="trait_checkbox checkbox" value="' + data.hmac + '">' + } + }, + { + 'title': "Index", + 'type': "natural", + 'width': "30px", + 'targets': 1, + 'data': "index" + }, + { + 'title': "Species", + 'type': "natural", + 'width': "60px", + 'targets': 2, + 'data': "species" + }, + { + 'title': "Group", + 'type': "natural", + 'width': "100px", + 'targets': 3, + 'data': "group" + }, + { + 'title': "Record", + 'type': "natural", + 'data': null, + 'width': "60px", + 'targets': 4, + 'orderDataType': "dom-inner-text", + 'render': function(data, type, row, meta) { + return '<a target="_blank" href="/show_trait?trait_id=' + data.name + '&dataset=' + data.dataset + '">' + data.display_name + '</a>' + } + }, + { + 'title': "Description", + 'type': "natural", + 'width': "500px", + 'targets': 5, + 'data': null, + 'render': function(data, type, row, meta) { + try { + return decodeURIComponent(escape(data.description)) + } catch(err) { + return data.description + } + } + }, + { + 'title': "Mean", + 'type': "natural-minus-na", + 'width': "30px", + 'targets': 6, + 'data': "mean" + }, + { + 'title': "Authors", + 'type': "natural", + 'width': "300px", + 'targets': 7, + 'data': null, + 'render': function(data, type, row, meta) { + author_list = data.authors.split(",") + if (author_list.length >= 6) { + author_string = author_list.slice(0, 6).join(",") + ", et al." + } else{ + author_string = data.authors + } + return author_string + } + }, + { + 'title': "Year", + 'type': "natural-minus-na", + 'data': null, + 'orderDataType': "dom-inner-text", + 'width': "25px", + 'targets': 8, + 'render': function(data, type, row, meta) { + if (data.pubmed_id != "N/A"){ + return '<a href="' + data.pubmed_link + '">' + data.pubmed_text + '</a>' } else { - for (_j = 0, _len2 = buttons.length; _j < _len2; _j++) { - button = buttons[_j]; - $(button).prop("disabled", false); - } + return data.pubmed_text } - //}); - if ($(this).is(":checked")) { - if (!$(this).closest('tr').hasClass('selected')) { - $(this).closest('tr').addClass('selected') - } + }, + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "Max LRS<a href=\"{{ url_for('glossary_blueprint.glossary') }}#LRS\" target=\"_blank\" style=\"color: white;\"><sup>?</sup></a>", + 'type': "natural-minus-na", + 'data': "LRS_score_repr", + 'width': "60px", + 'targets': 9, + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "Max LRS Location", + 'type': "natural-minus-na", + 'width': "125px", + 'targets': 10, + 'data': "max_lrs_text" + }, + { + 'title': "Additive Effect<a href=\"{{ url_for('glossary_blueprint.glossary') }}#A\" target=\"_blank\" style=\"color: white;\"><sup>?</sup></a>", + 'type': "natural-minus-na", + 'data': "additive", + 'width': "60px", + 'targets': 11, + 'orderSequence': [ "desc", "asc"] + } + ] + + loadDataTable(true); + + function loadDataTable(first_run=false){ + + if (!first_run){ + setUserColumnsDefWidths(tableId); + } + + table_settings = { + 'drawCallback': function( settings ) { + $('#' + tableId + ' tr').off().on("click", function(event) { + if (event.target.type !== 'checkbox' && event.target.tagName.toLowerCase() !== 'a') { + var obj =$(this).find('input'); + obj.prop('checked', !obj.is(':checked')); + } + if ($(this).hasClass("selected") && event.target.tagName.toLowerCase() !== 'a'){ + $(this).removeClass("selected") + } else if (event.target.tagName.toLowerCase() !== 'a') { + $(this).addClass("selected") + } + change_buttons() + }); + }, + "createdRow": function ( row, data, index ) { + $('td', row).eq(0).attr("style", "text-align: center; padding: 4px 10px 2px 10px;"); + $('td', row).eq(1).attr("align", "right"); + $('td', row).eq(5).attr('title', $('td', row).eq(5).text()); + if ($('td', row).eq(5).text().length > 150) { + $('td', row).eq(5).text($('td', row).eq(5).text().substring(0, 150)); + $('td', row).eq(5).text($('td', row).eq(5).text() + '...') } - else { - if ($(this).closest('tr').hasClass('selected')) { - $(this).closest('tr').removeClass('selected') - } + $('td', row).eq(6).attr('title', $('td', row).eq(6).text()); + if ($('td', row).eq(6).text().length > 150) { + $('td', row).eq(6).text($('td', row).eq(6).text().substring(0, 150)); + $('td', row).eq(6).text($('td', row).eq(6).text() + '...') } + $('td', row).eq(6).attr("align", "right"); + $('td', row).slice(8,11).attr("align", "right"); + $('td', row).eq(1).attr('data-export', $('td', row).eq(1).text()); + $('td', row).eq(2).attr('data-export', $('td', row).eq(2).text()); + $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); + $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text()); + $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text()); + $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text()); + $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text()); + $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text()); + $('td', row).eq(9).attr('data-export', $('td', row).eq(9).text()); + $('td', row).eq(10).attr('data-export', $('td', row).eq(10).text()); + }, + 'data': trait_list, + 'columns': columnDefs, + "order": [[1, "asc" ]], + 'sDom': "iti", + "destroy": true, + "deferRender": true, + "bSortClasses": false, + {% if trait_count > 20 %} + "scrollY": "100vh", + "scroller": true, + "scrollCollapse": true, + {% else %} + "iDisplayLength": -1, + {% endif %} + "initComplete": function (settings) { + //Add JQueryUI resizable functionality to each th in the ScrollHead table + $('#' + tableId + '_wrapper .dataTables_scrollHead thead th').resizable({ + handles: "e", + alsoResize: '#' + tableId + '_wrapper .dataTables_scrollHead table', //Not essential but makes the resizing smoother + resize: function( event, ui ) { + width_change = ui.size.width - ui.originalSize.width; + }, + stop: function () { + saveColumnSettings(tableId, trait_table); + loadDataTable(); + } + }); + } } - var the_table = $('#trait_table').DataTable( { - 'drawCallback': function( settings ) { - $('#trait_table tr').click(function(event) { - if (event.target.type !== 'checkbox' && event.target.tagName.toLowerCase() !== 'a') { - $(':checkbox', this).trigger('click'); - } - }); - $('.trait_checkbox:checkbox').on("change", change_buttons); - }, - "createdRow": function ( row, data, index ) { - $('td', row).eq(0).attr("style", "text-align: center; padding: 4px 10px 2px 10px;"); - $('td', row).eq(1).attr("align", "right"); - $('td', row).eq(5).attr('title', $('td', row).eq(5).text()); - if ($('td', row).eq(5).text().length > 150) { - $('td', row).eq(5).text($('td', row).eq(5).text().substring(0, 150)); - $('td', row).eq(5).text($('td', row).eq(5).text() + '...') - } - $('td', row).eq(6).attr('title', $('td', row).eq(6).text()); - if ($('td', row).eq(6).text().length > 150) { - $('td', row).eq(6).text($('td', row).eq(6).text().substring(0, 150)); - $('td', row).eq(6).text($('td', row).eq(6).text() + '...') - } - $('td', row).eq(6).attr("align", "right"); - $('td', row).slice(8,11).attr("align", "right"); - $('td', row).eq(1).attr('data-export', $('td', row).eq(1).text()); - $('td', row).eq(2).attr('data-export', $('td', row).eq(2).text()); - $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); - $('td', row).eq(4).attr('data-export', $('td', row).eq(4).text()); - $('td', row).eq(5).attr('data-export', $('td', row).eq(5).text()); - $('td', row).eq(6).attr('data-export', $('td', row).eq(6).text()); - $('td', row).eq(7).attr('data-export', $('td', row).eq(7).text()); - $('td', row).eq(8).attr('data-export', $('td', row).eq(8).text()); - $('td', row).eq(9).attr('data-export', $('td', row).eq(9).text()); - $('td', row).eq(10).attr('data-export', $('td', row).eq(10).text()); - }, - 'data': trait_list, - 'columns': [ - { - 'data': null, - 'orderDataType': "dom-checkbox", - 'width': "10px", - 'render': function(data, type, row, meta) { - return '<input type="checkbox" name="searchResult" class="trait_checkbox checkbox" value="' + data.hmac + '">' - } - }, - { - 'title': "Index", - 'type': "natural", - 'width': "30px", - 'data': "index" - }, - { - 'title': "Species", - 'type': "natural", - 'width': "60px", - 'data': "species" - }, - { - 'title': "Group", - 'type': "natural", - 'width': "100px", - 'data': "group" - }, - { - 'title': "Record", - 'type': "natural", - 'data': null, - 'width': "60px", - 'orderDataType': "dom-inner-text", - 'render': function(data, type, row, meta) { - return '<a target="_blank" href="/show_trait?trait_id=' + data.name + '&dataset=' + data.dataset + '">' + data.display_name + '</a>' - } - }, - { - 'title': "Description", - 'type': "natural", - 'width': "500px", - 'data': null, - 'render': function(data, type, row, meta) { - try { - return decodeURIComponent(escape(data.description)) - } catch(err) { - return data.description - } - } - }, - { - 'title': "Mean", - 'type': "natural-minus-na", - 'width': "30px", - 'data': "mean" - }, - { - 'title': "Authors", - 'type': "natural", - 'width': "300px", - 'data': null, - 'render': function(data, type, row, meta) { - author_list = data.authors.split(",") - if (author_list.length >= 6) { - author_string = author_list.slice(0, 6).join(",") + ", et al." - } else{ - author_string = data.authors - } - return author_string - } - }, - { - 'title': "Year", - 'type': "natural-minus-na", - 'data': null, - 'orderDataType': "dom-inner-text", - 'width': "25px", - 'render': function(data, type, row, meta) { - if (data.pubmed_id != "N/A"){ - return '<a href="' + data.pubmed_link + '">' + data.pubmed_text + '</a>' - } else { - return data.pubmed_text - } - }, - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "Max LRS<a href=\"{{ url_for('glossary_blueprint.glossary') }}#LRS\" target=\"_blank\" style=\"color: white;\"><sup>?</sup></a>", - 'type': "natural-minus-na", - 'data': "LRS_score_repr", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "Max LRS Location", - 'type': "natural-minus-na", - 'width': "125px", - 'data': "max_lrs_text" - }, - { - 'title': "Additive Effect<a href=\"{{ url_for('glossary_blueprint.glossary') }}#A\" target=\"_blank\" style=\"color: white;\"><sup>?</sup></a>", - 'type': "natural-minus-na", - 'data': "additive", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - } - ], - "order": [[1, "asc" ]], - 'sDom': "iti", - "autoWidth": true, - "bSortClasses": false, - 'processing': true, - {% if trait_count > 20 %} - "scrollY": "100vh", - "scroller": true, - "scrollCollapse": true - {% else %} - "iDisplayLength": -1 - {% endif %} - } ); + if (!first_run){ + table_settings['autoWidth'] = false; + $('#table_container').css("width", String($('#trait_table').width() + width_change {% if trait_list|length > 20 %}+ 17{% endif %}) + "px"); // Change the container width by the change in width of the adjusted column, so the overall table size adjusts properly + } - $('#trait_table').append( - '<tfoot>' + - '<tr>' + - '<th></th>' + - '<th>Index</th>' + - '<th>Species</th> ' + - '<th>Group</th>' + - '<th>Record</th>' + - '<th>Description</th>' + - '<th>Authors</th>' + - '<th>Year</th>' + - '<th>Max LRS</th>' + - '<th>Max LRS Location</th>' + - '<th>Additive Effect</th>' + - '</tr>' + - '</tfoot>' - ); + let checked_rows = get_checked_rows(tableId); + trait_table = $('#' + tableId).DataTable(table_settings); + if (checked_rows.length > 0){ + recheck_rows(trait_table, checked_rows); + } - the_table.draw(); + if (first_run){ + {% if trait_list|length > 20 %} + $('#table_container').css("width", String($('#trait_table').width() + 17) + "px"); + {% endif %} + } + + trait_table.draw(); + } + + $('#redraw').click(function() { + var table = $('#' + tableId).DataTable(); + table.colReorder.reset() + }); }); - </script> - <script language="javascript" type="text/javascript" src="/static/new/javascript/search_results.js"></script> {% endblock %} diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html index 7b103305..3a490658 100755 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -1,17 +1,18 @@ {% extends "base.html" %} {% block title %}GeneNetwork{% endblock %} {% block css %} +<!-- UIkit CSS --> +<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/uikit@3.7.4/dist/css/uikit.min.css" /> + +<!-- UIkit JS --> +<script src="https://cdn.jsdelivr.net/npm/uikit@3.7.4/dist/js/uikit.min.js"></script> +<script src="https://cdn.jsdelivr.net/npm/uikit@3.7.4/dist/js/uikit-icons.min.js"></script> <style TYPE="text/css"> p.interact { display: none; } - - - .tweet { - padding:5px; color:#000; - } .media { @@ -26,7 +27,24 @@ border-radius: 5px; /*transform: scale(1.1); image small?*/ border:1px solid #c8ccc9; + } + + div#tweets { + margin:0 auto; + width: 590px; + height: 160px; + background: white; + position: relative; + border-radius:10px; + background-color: #F9F9F9; + } + + h2 { + margin-bottom: 0px; + } + ul { + margin-top: 0px; } </style> {% endblock %} @@ -40,12 +58,12 @@ <div class="col-xs-4" style="margin-right:50px; min-width: 530px; max-width: 550px;"> <section id="search"> - <div> - <h1>Select and search</h1> + <div class="page-header"> + <h2>Select and search</h2> </div> <form method="get" action="/search" target="_blank" name="SEARCHFORM"> <fieldset> - <div style="padding: 20px" class="form-horizontal"> + <div style="padding-left: 20px; padding-right: 20px;" class="form-horizontal"> <div class="form-group"> <label for="species" class="col-xs-1 control-label" style="width: 65px !important;">Species:</label> @@ -195,53 +213,74 @@ </div> <div class="col-xs-4" style="width: 600px !important;"> - <section id="affiliates"> - <div class="page-header"> - <h1>Affiliates</h1> - <ul> - <li><b><a href="http://gn1.genenetwork.org">GeneNetwork 1</a> at UTHSC</b></li> - <li><a href="https://systems-genetics.org/">Systems Genetics</a> at EPFL</li> - <li><a href="http://bnw.genenetwork.org/">Bayesian Network Web Server</a> at UTHSC</li> - <li><a href="https://www.geneweaver.org/">GeneWeaver</a></li> - <li><a href="https://phenogen.org/">PhenoGen</a> at University of Colorado</li> - <li><a href="http://www.webgestalt.org/">WebGestalt</a> at Baylor</li> - </ul> + <section id="tutorials"> + <div class="page-header"> + <h2>Tutorials</h2> + </div> + <div class="uk-grid-match uk-child-width-1-3@m" uk-grid> + <div> + <strong><a class="uk-link-text" href="/tutorials">Webinars & Courses</a></strong><br> + In-person courses, live webinars and webinar recordings<br> + <a href="/tutorials" class="uk-icon-link" uk-icon="laptop" ratio="2"></a> + </div> + <div> + <strong><a class="uk-link-text" href="/tutorials">Tutorials</a></strong><br> + Tutorials: Training materials in HTML, PDF and video formats<br> + <a href="/tutorials" class="uk-icon-link" uk-icon="file-text" ratio="2"></a> + </div> + <div> + <strong><a class="uk-link-text" href="/tutorials">Documentation</a></strong><br> + Online manuals, handbooks, fact sheets and FAQs<br> + <a href="/tutorials" class="uk-icon-link" uk-icon="album" ratio="2"></a> + </div> </div> </section> <section id="news-section"> <div class="page-header"> - <h1>News</h1> - <div id="tweets" style="height: 300px; overflow: scroll; overflow-x: hidden;"></div> - <div align="right"> - <a href="https://twitter.com/GeneNetwork2">more news items...</a> - </div> + <h2>News</h2> </div> - </section> - <section id="websites"> - <div class="page-header"> - <h1>Github</h1> - <ul> - <li><a href="https://github.com/genenetwork/genenetwork2">GN2 Source Code</a></li> - <li><a href="https://github.com/genenetwork/genenetwork">GN1 Source Code</a></li> - <li><a href="https://github.com/genenetwork/sysmaintenance">System Maintenance Code</a></li> - </ul> + <div id="tweets" style="height: 300px; overflow: scroll; overflow-x: hidden;"></div> + <div align="right"> + <a href="https://twitter.com/GeneNetwork2">more news items...</a> </div> - </section> + </section> <section id="websites"> <div class="page-header"> - <h1>Links</h1> + <h2>Links</h2> </div> - <h3>GeneNetwork v2:</h3> - <ul> - <li><a href="http://genenetwork.org/">Main website</a> at UTHSC</li> - </ul> - <h3>GeneNetwork v1:</h3> <ul> - <li><a href="http://gn1.genenetwork.org/">Main website</a> at UTHSC</li> - <li><span class="broken_link" href="http://artemis.uthsc.edu/">Time Machine</span>: Full GN versions from 2009 to 2016 (mm9)</li> + <li>Github</li> + <ul> + <li><a href="https://github.com/genenetwork/genenetwork2">GN2 Source Code</a></li> + <li><a href="https://github.com/genenetwork/genenetwork">GN1 Source Code</a></li> + <li><a href="https://github.com/genenetwork/sysmaintenance">System Maintenance Code</a></li> + </ul> + </ul> + <ul> + <li>GeneNetwork v2</li> + <ul> + <li><a href="http://genenetwork.org/">Main website</a> at UTHSC</li> + </ul> + </ul> + <ul> + <li>GeneNetwork v1</li> + <ul> + <li><a href="http://gn1.genenetwork.org/">Main website</a> at UTHSC</li> + <li><span class="broken_link" href="http://artemis.uthsc.edu/">Time Machine</span>: Full GN versions from 2009 to 2016 (mm9)</li> Cloud (EC2)</a></li> + </ul> + </ul> + <ul> + <li>Affiliates</li> + <ul> + <li><b><a href="http://gn1.genenetwork.org">GeneNetwork 1</a> at UTHSC</b></li> + <li><a href="https://systems-genetics.org/">Systems Genetics</a> at EPFL</li> + <li><a href="http://bnw.genenetwork.org/">Bayesian Network Web Server</a> at UTHSC</li> + <li><a href="https://www.geneweaver.org/">GeneWeaver</a></li> + <li><a href="https://phenogen.org/">PhenoGen</a> at University of Colorado</li> + <li><a href="http://www.webgestalt.org/">WebGestalt</a> at Baylor</li> + </ul> </ul> - <script type="text/javascript" src="//rf.revolvermaps.com/0/0/8.js?i=526mdlpknyd&m=0&c=ff0000&cr1=ffffff&f=arial&l=33" async="async"></script> </section> </div> </div> diff --git a/wqflask/wqflask/templates/jupyter_notebooks.html b/wqflask/wqflask/templates/jupyter_notebooks.html new file mode 100644 index 00000000..afc95a15 --- /dev/null +++ b/wqflask/wqflask/templates/jupyter_notebooks.html @@ -0,0 +1,28 @@ +{%extends "base.html"%} + +{%block title%} +Jupyter Notebooks +{%endblock%} + +{%block css%} +<link rel="stylesheet" type="text/css" href="/static/new/css/jupyter_notebooks.css" /> +{%endblock%} + +{%block content%} + +<div class="container"> + <h1>Current Notebooks</h1> + + {%for item in links:%} + <div class="jupyter-links"> + <a href="{{item.get('main_url')}}" + title="Access running notebook for '{{item.get('notebook_name')}}' on GeneNetwork" + class="main-link">{{item.get("notebook_name")}}</a> + <a href="{{item.get('src_link_url')}}" + title="Link to the notebook repository for {{item.get('notebook_name', '_')}}" + class="src-link">View Source</a> + </div> + {%endfor%} +</div> + +{%endblock%} diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index ccf810b0..b9e31ad0 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -66,11 +66,11 @@ {% endif %} {% endif %} {% else %} - <h1>Loading {{ start_vars.tool_used }} Results...</h1> + <h1> {{ start_vars.tool_used }} Computation in progress ...</h1> {% endif %} <br><br> <div style="text-align: center;"> - <img align="center" src="/static/gif/89.gif"> + <img align="center" src="/static/gif/waitAnima2.gif"> </div> {% if start_vars.vals_diff|length != 0 and start_vars.transform == "" %} <br><br> diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index f2d11e89..84db288c 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -12,7 +12,7 @@ {% endblock %} {% from "base_macro.html" import header %} {% block content %} - <div class="container" style="min-width: 900px;"> + <div class="container"> <form method="post" target="_blank" action="/run_mapping" name="marker_regression" id="marker_regression_form"> <input type="hidden" name="temp_uuid" value="{{ temp_uuid }}"> {% if temp_trait is defined %} @@ -39,6 +39,7 @@ <input type="hidden" name="maf" value="{{ maf }}"> <input type="hidden" name="use_loco" value="{{ use_loco }}"> <input type="hidden" name="selected_chr" value="{{ selectedChr }}"> + <input type="hidden" name="mapping_scale" value="{{ plotScale }}"> <input type="hidden" name="manhattan_plot" value="{{ manhattan_plot }}"> {% if manhattan_plot == True %} <input type="hidden" name="color_scheme" value="alternating"> diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index c499aa8f..f73cba17 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -5,6 +5,7 @@ <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='fontawesome/css/font-awesome.min.css') }}" /> <link rel="stylesheet" type="text/css" href="{{ url_for('js', filename='DataTablesExtensions/buttonStyles/css/buttons.dataTables.min.css') }}"> <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='fontawesome/css/all.min.css') }}"/> + <link href="https://code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css" rel="stylesheet" type="text/css" /> <link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" /> <link rel="stylesheet" type="text/css" href="static/new/css/trait_list.css" /> {% endblock %} @@ -125,8 +126,8 @@ {% endif %} </div> {% endif %} - <div id="table_container" {% if dataset.type == 'ProbeSet' or dataset.type == 'Publish' %}style="min-width: 1500px; max-width:100%;"{% endif %}> - <table class="table-hover table-striped cell-border" id='trait_table' style="float: left; width: {% if dataset.type == 'Geno' %}380px{% else %}100%{% endif %};"> + <div id="table_container" style="width: {% if dataset.type == 'Geno' %}270{% else %}100%{% endif %}px;"> + <table class="table-hover table-striped cell-border" id='trait_table' style="float: left;"> <tbody> <td colspan="100%" align="center"><br><b><font size="15">Loading...</font></b><br></td> </tbody> @@ -147,7 +148,6 @@ {% block js %} <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='js_alt/md5.min.js') }}"></script> - <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.dataTables.min.js') }}"></script> <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/scroller/js/dataTables.scroller.min.js') }}"></script> <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='jszip/jszip.min.js') }}"></script> @@ -157,6 +157,7 @@ <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='fontawesome/js/all.min.js') }}"></script> <script language="javascript" type="text/javascript" src="/static/new/javascript/search_results.js"></script> + <script language="javascript" type="text/javascript" src="/static/new/javascript/table_functions.js"></script> <script type='text/javascript'> var trait_list = {{ trait_list|safe }}; @@ -175,11 +176,222 @@ return params; }; - {% if results|count > 0 %} - //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters - trait_table = $('#trait_table').DataTable( { + {% if results|count > 0 and not too_many_results %} + var tableId = "trait_table"; + + var width_change = 0; //ZS: For storing the change in width so overall table width can be adjusted by that amount + + columnDefs = [ + { + 'data': null, + 'width': "5px", + 'orderDataType': "dom-checkbox", + 'targets': 0, + 'render': function(data, type, row, meta) { + return '<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + data.hmac + '">' + } + }, + { + 'title': "Index", + 'type': "natural", + 'width': "35px", + 'targets': 1, + 'data': "index" + } + {% if dataset.type == 'ProbeSet' %}, + { + 'title': "Record", + 'type': "natural-minus-na", + 'data': null, + 'width': "{{ max_widths.display_name * 8 }}px", + 'targets': 2, + 'render': function(data, type, row, meta) { + return '<a target="_blank" href="/show_trait?trait_id=' + data.display_name + '&dataset=' + data.dataset + '">' + data.display_name + '</a>' + } + }, + { + 'title': "Symbol", + 'type': "natural", + 'width': "{{ max_widths.symbol * 8 }}px", + 'targets': 3, + 'data': "symbol" + }, + { + 'title': "Description", + 'type': "natural", + 'data': null, + 'targets': 4, + 'render': function(data, type, row, meta) { + try { + return decodeURIComponent(escape(data.description)) + } catch(err){ + return escape(data.description) + } + } + }, + { + 'title': "<div style='text-align: right;'>Location</div>", + 'type': "natural-minus-na", + 'width': "125px", + 'targets': 5, + 'data': "location" + }, + { + 'title': "<div style='text-align: right;'>Mean</div>", + 'type': "natural-minus-na", + 'width': "40px", + 'data': "mean", + 'targets': 6, + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "<div style='text-align: right;'>Peak <a href=\"{{ url_for('glossary_blueprint.glossary') }}#LRS\" target=\"_blank\" style=\"color: white;\"> <i class=\"fa fa-info-circle\" aria-hidden=\"true\"></i></a></div><div style='text-align: right;'>LOD  </div>", + 'type': "natural-minus-na", + 'data': "lod_score", + 'width': "60px", + 'targets': 7, + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "<div style='text-align: right;'>Peak Location</div>", + 'type': "natural-minus-na", + 'width': "125px", + 'targets': 8, + 'data': "lrs_location" + }, + { + 'title': "<div style='text-align: right;'>Effect <a href=\"{{ url_for('glossary_blueprint.glossary') }}#A\" target=\"_blank\" style=\"color: white;\"> <i class=\"fa fa-info-circle\" aria-hidden=\"true\"></i></a></div><div style='text-align: right;'>Size  </div>", + 'type': "natural-minus-na", + 'data': "additive", + 'width': "65px", + 'targets': 9, + 'orderSequence': [ "desc", "asc"] + }{% elif dataset.type == 'Publish' %}, + { + 'title': "Record", + 'type': "natural-minus-na", + 'width': "{{ max_widths.display_name * 9 }}px", + 'data': null, + 'targets': 2, + 'render': function(data, type, row, meta) { + return '<a target="_blank" href="/show_trait?trait_id=' + data.name + '&dataset=' + data.dataset + '">' + data.display_name + '</a>' + } + }, + { + 'title': "Description", + 'type': "natural", + {% if (max_widths.description * 7) < 500 %} + 'width': "{{ max_widths.description * 7 }}px", + {% else %} + 'width': "500px", + {% endif %} + 'data': null, + 'targets': 3, + 'render': function(data, type, row, meta) { + try { + return decodeURIComponent(escape(data.description)) + } catch(err){ + return data.description + } + } + }, + { + 'title': "<div style='text-align: right;'>Mean</div>", + 'type': "natural-minus-na", + 'width': "60px", + 'data': "mean", + 'targets': 4, + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "Authors", + 'type': "natural", + {% if (max_widths.authors * 7) < 500 %} + 'width': "{{ max_widths.authors * 7 }}px", + {% else %} + 'width': "500px", + {% endif %} + 'data': null, + 'targets': 5, + 'render': function(data, type, row, meta) { + author_list = data.authors.split(",") + if (author_list.length >= 6) { + author_string = author_list.slice(0, 6).join(",") + ", et al." + } else{ + author_string = data.authors + } + return author_string + } + }, + { + 'title': "<div style='text-align: right;'>Year</div>", + 'type': "natural-minus-na", + 'data': null, + 'width': "50px", + 'targets': 6, + 'render': function(data, type, row, meta) { + if (data.pubmed_id != "N/A"){ + return '<a href="' + data.pubmed_link + '">' + data.pubmed_text + '</a>' + } else { + return data.pubmed_text + } + }, + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "<div style='text-align: right;'>Peak <a href=\"{{ url_for('glossary_blueprint.glossary') }}#LRS\" target=\"_blank\" style=\"color: white;\"> <i class=\"fa fa-info-circle\" aria-hidden=\"true\"></i></a></div><div style='text-align: right;'>LOD  </div>", + 'type': "natural-minus-na", + 'data': "lod_score", + 'targets': 7, + 'width': "60px", + 'orderSequence': [ "desc", "asc"] + }, + { + 'title': "<div style='text-align: right;'>Peak Location</div>", + 'type': "natural-minus-na", + 'width': "120px", + 'targets': 8, + 'data': "lrs_location" + }, + { + 'title': "<div style='text-align: right;'>Effect <a href=\"{{ url_for('glossary_blueprint.glossary') }}#A\" target=\"_blank\" style=\"color: white;\"> <i class=\"fa fa-info-circle\" aria-hidden=\"true\"></i></a></div><div style='text-align: right;'>Size  </div>", + 'type': "natural-minus-na", + 'width': "60px", + 'data': "additive", + 'targets': 9, + 'orderSequence': [ "desc", "asc"] + }{% elif dataset.type == 'Geno' %}, + { + 'title': "Record", + 'type': "natural-minus-na", + 'width': "{{ max_widths.display_name * 9 }}px", + 'data': null, + 'targets': 2, + 'render': function(data, type, row, meta) { + return '<a target="_blank" href="/show_trait?trait_id=' + data.display_name + '&dataset=' + data.dataset + '">' + data.display_name + '</a>' + } + }, + { + 'title': "<div style='text-align: right;'>Location</div>", + 'type': "natural-minus-na", + 'width': "120px", + 'targets': 2, + 'data': "location" + }{% endif %} + ]; + + loadDataTable(true); + + function loadDataTable(first_run=false){ + + if (!first_run){ + setUserColumnsDefWidths(tableId); + } + + //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters + table_settings = { 'drawCallback': function( settings ) { - $('#trait_table tr').off().on("click", function(event) { + $('#' + tableId + ' tr').off().on("click", function(event) { if (event.target.type !== 'checkbox' && event.target.tagName.toLowerCase() !== 'a') { var obj =$(this).find('input'); obj.prop('checked', !obj.is(':checked')); @@ -193,7 +405,7 @@ }); }, 'createdRow': function ( row, data, index ) { - $('td', row).eq(0).attr("style", "text-align: center; padding: 0px 10px 2px 13px;"); + $('td', row).eq(0).attr("style", "text-align: center; padding: 0px 10px 2px 10px;"); $('td', row).eq(1).attr("align", "right"); $('td', row).eq(1).attr('data-export', index+1); $('td', row).eq(2).attr('data-export', $('td', row).eq(2).text()); @@ -228,173 +440,60 @@ $('td', row).eq(3).attr('data-export', $('td', row).eq(3).text()); {% endif %} }, - 'data': trait_list, - 'columns': [ - { - 'data': null, - 'width': "10px", - 'orderDataType': "dom-checkbox", - 'orderable': false, - 'render': function(data, type, row, meta) { - return '<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + data.hmac + '">' - } - }, - { - 'title': "Index", - 'type': "natural", - 'width': "30px", - 'data': "index" - }, - { - 'title': "Record", - 'type': "natural-minus-na", - 'data': null, - 'width': "60px", - 'render': function(data, type, row, meta) { - return '<a target="_blank" href="/show_trait?trait_id=' + data.name + '&dataset=' + data.dataset + '">' + data.display_name + '</a>' - } - }{% if dataset.type == 'ProbeSet' %}, - { - 'title': "Symbol", - 'type': "natural", - 'width': "120px", - 'data': "symbol" - }, - { - 'title': "Description", - 'type': "natural", - 'data': null, - 'render': function(data, type, row, meta) { - try { - return decodeURIComponent(escape(data.description)) - } catch(err){ - return escape(data.description) - } - } - }, - { - 'title': "<div style='text-align: right;'>Location</div>", - 'type': "natural-minus-na", - 'width': "125px", - 'data': "location" - }, - { - 'title': "<div style='text-align: right;'>Mean</div>", - 'type': "natural-minus-na", - 'width': "30px", - 'data': "mean", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "<div style='text-align: right;'>Peak <a href=\"{{ url_for('glossary_blueprint.glossary') }}#LRS\" target=\"_blank\" style=\"color: white;\"> <i class=\"fa fa-info-circle\" aria-hidden=\"true\"></i></a></div><div style='text-align: right;'>LOD  </div>", - 'type': "natural-minus-na", - 'data': "lod_score", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "<div style='text-align: right;'>Peak Location</div>", - 'type': "natural-minus-na", - 'width': "125px", - 'data': "lrs_location" - }, - { - 'title': "<div style='text-align: right;'>Effect <a href=\"{{ url_for('glossary_blueprint.glossary') }}#A\" target=\"_blank\" style=\"color: white;\"> <i class=\"fa fa-info-circle\" aria-hidden=\"true\"></i></a></div><div style='text-align: right;'>Size  </div>", - 'type': "natural-minus-na", - 'data': "additive", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }{% elif dataset.type == 'Publish' %}, - { - 'title': "Description", - 'type': "natural", - 'width': "500px", - 'data': null, - 'render': function(data, type, row, meta) { - try { - return decodeURIComponent(escape(data.description)) - } catch(err){ - return data.description - } - } - }, - { - 'title': "<div style='text-align: right;'>Mean</div>", - 'type': "natural-minus-na", - 'width': "30px", - 'data': "mean", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "Authors", - 'type': "natural", - 'width': "300px", - 'data': null, - 'render': function(data, type, row, meta) { - author_list = data.authors.split(",") - if (author_list.length >= 6) { - author_string = author_list.slice(0, 6).join(",") + ", et al." - } else{ - author_string = data.authors - } - return author_string - } - }, - { - 'title': "<div style='text-align: right;'>Year</div>", - 'type': "natural-minus-na", - 'data': null, - 'width': "25px", - 'render': function(data, type, row, meta) { - if (data.pubmed_id != "N/A"){ - return '<a href="' + data.pubmed_link + '">' + data.pubmed_text + '</a>' - } else { - return data.pubmed_text - } - }, - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "<div style='text-align: right;'>Peak <a href=\"{{ url_for('glossary_blueprint.glossary') }}#LRS\" target=\"_blank\" style=\"color: white;\"> <i class=\"fa fa-info-circle\" aria-hidden=\"true\"></i></a></div><div style='text-align: right;'>LOD  </div>", - 'type': "natural-minus-na", - 'data': "lod_score", - 'width': "60px", - 'orderSequence': [ "desc", "asc"] - }, - { - 'title': "<div style='text-align: right;'>Peak Location</div>", - 'type': "natural-minus-na", - 'width': "120px", - 'data': "lrs_location" - }, - { - 'title': "<div style='text-align: right;'>Effect <a href=\"{{ url_for('glossary_blueprint.glossary') }}#A\" target=\"_blank\" style=\"color: white;\"> <i class=\"fa fa-info-circle\" aria-hidden=\"true\"></i></a></div><div style='text-align: right;'>Size  </div>", - 'type': "natural-minus-na", - 'width': "60px", - 'data': "additive", - 'orderSequence': [ "desc", "asc"] - }{% elif dataset.type == 'Geno' %}, - { - 'title': "<div style='text-align: right;'>Location</div>", - 'type': "natural-minus-na", - 'width': "120px", - 'data': "location" - }{% endif %} - ], + "data": trait_list, + "columns": columnDefs, "order": [[1, "asc" ]], - 'sDom': "iti", - "autoWidth": true, + "sDom": "iti", + "destroy": true, + "autoWidth": false, "bSortClasses": false, - {% if trait_list|length > 20 %} - "scrollY": "100vh", + "scrollY": "500px", + "scrollCollapse": true, + {% if trait_list|length > 5 %} "scroller": true, - "scrollCollapse": true + {% endif %} + "iDisplayLength": -1, + "initComplete": function (settings) { + //Add JQueryUI resizable functionality to each th in the ScrollHead table + $('#' + tableId + '_wrapper .dataTables_scrollHead thead th').resizable({ + handles: "e", + alsoResize: '#' + tableId + '_wrapper .dataTables_scrollHead table', //Not essential but makes the resizing smoother + resize: function( event, ui ) { + width_change = ui.size.width - ui.originalSize.width; + }, + stop: function () { + saveColumnSettings(tableId, trait_table); + loadDataTable(); + } + }); + } + } + + if (!first_run){ + $('#table_container').css("width", String($('#trait_table').width() + width_change {% if trait_list|length > 20 %}+ 17{% endif %}) + "px"); //ZS : Change the container width by the change in width of the adjusted column, so the overall table size adjusts properly + + let checked_rows = get_checked_rows(tableId); + trait_table = $('#' + tableId).DataTable(table_settings); + if (checked_rows.length > 0){ + recheck_rows(trait_table, checked_rows); + } + } else { + trait_table = $('#' + tableId).DataTable(table_settings); + trait_table.draw(); + } + + if (first_run){ + {% if trait_list|length > 20 %} + $('#table_container').css("width", String($('#trait_table').width() + 17) + "px"); {% else %} - "iDisplayLength": -1 + $('#table_container').css("width", String($('#trait_table').width()) + "px"); {% endif %} - } ); + } + } - trait_table.draw(); //ZS: This makes the table adjust its height properly on initial load + window.addEventListener('resize', function(){ + trait_table.columns.adjust(); + }); $('.toggle-vis').on( 'click', function (e) { e.preventDefault(); @@ -412,9 +511,8 @@ } } ); - $('#redraw').click(function() { - var table = $('#trait_table').DataTable(); + var table = $('#' + tableId).DataTable(); table.colReorder.reset() }); {% endif %} diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html index f3fa1332..7074e21e 100644 --- a/wqflask/wqflask/templates/show_trait.html +++ b/wqflask/wqflask/templates/show_trait.html @@ -5,9 +5,10 @@ <link rel="stylesheet" type="text/css" href="/static/new/css/box_plot.css" /> <link rel="stylesheet" type="text/css" href="/static/new/css/prob_plot.css" /> <link rel="stylesheet" type="text/css" href="/static/new/css/scatter-matrix.css" /> - <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='d3-tip/d3-tip.css') }}" /> + <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='d3-tip/d3-tip.css') }}" /> <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTables/css/jquery.dataTables.css') }}" /> <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='nouislider/nouislider.min.css') }}" /> + <link href="https://code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css" rel="stylesheet" type="text/css" /> <link rel="stylesheet" type="text/css" href="/static/new/css/trait_list.css" /> <link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" /> @@ -155,6 +156,7 @@ <script type="text/javascript" src="/static/new/javascript/show_trait.js"></script> <script type="text/javascript" src="/static/new/javascript/validation.js"></script> <script type="text/javascript" src="/static/new/javascript/get_covariates_from_collection.js"></script> + <script type="text/javascript" src="/static/new/javascript/table_functions.js"></script> <script type="text/javascript" charset="utf-8"> @@ -211,16 +213,6 @@ } }); - primary_table.on( 'order.dt search.dt draw.dt', function () { - primary_table.column(1, {search:'applied', order:'applied'}).nodes().each( function (cell, i) { - cell.innerHTML = i+1; - } ); - } ).draw(); - - $('#primary_searchbox').on( 'keyup', function () { - primary_table.search($(this).val()).draw(); - } ); - $('.toggle-vis').on('click', function (e) { e.preventDefault(); @@ -237,7 +229,6 @@ // Get the column API object var target_cols = $(this).attr('data-column').split(",") for (let i = 0; i < target_cols.length; i++){ - console.log("THE COL:", target_cols[i]) var column = primary_table.column( target_cols[i] ); toggle_column(column); @@ -248,11 +239,7 @@ } } ); - {% if sample_groups|length != 1 %} - $('#other_searchbox').on( 'keyup', function () { - other_table.search($(this).val()).draw(); - } ); - {% endif %} + $('#samples_primary, #samples_other').find("tr.outlier").css('background-color', 'orange') $('.edit_sample_checkbox:checkbox').change(function() { if ($(this).is(":checked")) { diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 2a21dd24..4e9ea0fb 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -234,16 +234,16 @@ {% endif %} {% endif %} <button type="button" id="view_in_gn1" class="btn btn-primary" title="View Trait in GN1" onclick="window.open('http://gn1.genenetwork.org/webqtl/main.py?cmd=show&db={{ this_trait.dataset.name }}&probeset={{ this_trait.name }}', '_blank')">Go to GN1</button> - {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} + {% if admin_status != None and admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} {% if this_trait.dataset.type == 'Publish' %} - <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('/trait/{{ this_trait.name }}/edit/inbredset-id/{{ this_trait.dataset.id }}', '_blank')">Edit</button> + <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('/datasets/{{ this_trait.dataset.group.id }}/traits/{{ this_trait.name }}?resource-id={{ resource_id }}', '_blank')">Edit</button> {% endif %} {% if this_trait.dataset.type == 'ProbeSet' %} - <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('/trait/edit/probeset-name/{{ this_trait.name }}', '_blank')">Edit</button> + <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('/datasets/traits/{{ this_trait.name }}?resource-id={{ resource_id }}', '_blank')">Edit</button> {% endif %} - {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} - <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('./resources/manage?resource_id={{ resource_id }}', '_blank')">Edit Privileges</button> + {% if admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} + <button type="button" id="edit_resource" class="btn btn-success" title="Edit Privileges" onclick="window.open('/resource-management/resources/{{ resource_id }}', '_blank')">Edit Privileges</button> {% endif %} {% endif %} </div> diff --git a/wqflask/wqflask/templates/show_trait_edit_data.html b/wqflask/wqflask/templates/show_trait_edit_data.html index 5939c953..e288e4d5 100644 --- a/wqflask/wqflask/templates/show_trait_edit_data.html +++ b/wqflask/wqflask/templates/show_trait_edit_data.html @@ -53,12 +53,12 @@ </div> </div> {% set outer_loop = loop %} - <div class="sample_group" style="width:{{ trait_table_width }}px;"> + <div class="sample_group"> <div style="position: relative;"> <div class="inline-div"><h3 style="float: left;">{{ sample_type.header }}<span name="transform_text"></span></h3></div> </div> - <div id="table_container"> - <table class="table-hover table-striped cell-border sample-table" id="samples_{{ sample_type.sample_group_type }}"> + <div id="{{ sample_type.sample_group_type }}_container" style="width: {{ trait_table_width }}px;"> + <table class="table-hover table-striped cell-border" id="samples_{{ sample_type.sample_group_type }}"> <tbody> <td colspan="100%" align="center"><br><b><font size="15">Loading...</font></b><br></td> </tbody> diff --git a/wqflask/wqflask/templates/test_wgcna_results.html b/wqflask/wqflask/templates/test_wgcna_results.html new file mode 100644 index 00000000..952f479e --- /dev/null +++ b/wqflask/wqflask/templates/test_wgcna_results.html @@ -0,0 +1,165 @@ +{% extends "base.html" %} +{% block title %}WCGNA results{% endblock %} +{% block content %} +<!-- Start of body --> + +<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/xterm/3.14.5/xterm.min.css" integrity="sha512-iLYuqv+v/P4u9erpk+KM83Ioe/l7SEmr7wB6g+Kg1qmEit8EShDKnKtLHlv2QXUp7GGJhmqDI+1PhJYLTsfb8w==" crossorigin="anonymous" referrerpolicy="no-referrer" /> + +<link rel="stylesheet" href="https://cdn.datatables.net/1.11.3/css/jquery.dataTables.min.css"> + + +<style type="text/css"> + + +.container { + min-height: 100vh; + width: 100vw; + padding: 20px; + +} + +.grid_container { + + + width: 80vw; + margin: auto; + padding: 20px; + + + display: grid; + grid-template-columns: repeat(7, 1fr); + /*grid-gap: 5px;*/ + border: 1px solid black; + grid-column-gap: 20px; + +} + +.control_sft_column { + text-align: center; +} + +.grid_container div:not(:last-child) { + border-right: 1px solid #000; +} + +.grid_container .control_sft_column h3 { + font-weight: bold; + font-size: 18px; +} + +.control_net_colors { + + display: flex; + flex-wrap: wrap; + justify-content: space-between; + align-items: center; + text-align: center; +} + + +.control_mod_eigens { + display: grid; + grid-template-columns: repeat(2, 200px); +} + +.control-image{ + display: block; + margin-left: auto; + margin-right: auto; + width: 80vw; +} +</style> +<div class="container"> + {% if error!='null' %} + <h4 style="text-align: center;">{{error}}</h4> + + {% else %} + <div> + <div > + <h2 style="text-align:center">Soft Thresholds </h2> + <div class="grid_container"> + + {% for key, value in results["data"]["output"]["soft_threshold"].items()%} + <div class="control_sft_column"> + <h3>{{key}}</h3> + {% for val in value %} + <p>{{val|round(3)}}</p> + {% endfor %} + </div> + {% endfor %} + </div> + </div> + + <div> + + {% if image["image_generated"] %} + <div > + <img class="control-image" src="data:image/jpeg;base64,{{ image['image_data']| safe }}"> + </div> + + {% endif %} +<!-- <div > + <img class="control-image" src="data:image/jpeg;base64,{{ results['data']['output']['image_data2']| safe }}"> + </div> --> + </div> + + <div> + <h2 style="text-align:center;"> Module eigen genes </h2> + <table id="eigens" class="display" width="80vw"></table> + </div> + + <div> + <h2 style="text-align:center;">Phenotype modules </h2> + + <table id="phenos" class="display" width="40vw" ></table> + </div> + </div> + +{% endif %} + +</div> + +{% endblock %} + +{% block js %} + +<script src="https://cdnjs.cloudflare.com/ajax/libs/xterm/3.14.5/xterm.min.js"></script> + +<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.js') }}"></script> +<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.dataTables.min.js') }}"></script> +<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/scroller/js/dataTables.scroller.min.js') }}"></script> + + +<script type="text/javascript"> + + +let results = {{results|safe}} + +let phenoModules = results["data"]["output"]["net_colors"] +let phenotypes = Object.keys(phenoModules) +let phenoMods = Object.values(phenoModules) + +let {col_names,mod_dataset} = {{data|safe}} + $('#eigens').DataTable( { + data: mod_dataset, + columns: col_names.map((name)=>{ + return { + title:name + } + }) + } ); + $('#phenos').DataTable( { + data:phenotypes.map((phenoName,idx)=>{ + return [phenoName,phenoMods[idx]] + }), + columns: [{ + title:"Phenotypes" + }, + { + title: "Modules" + }] + } ); + + +</script> +{% endblock %}
\ No newline at end of file diff --git a/wqflask/wqflask/templates/tool_buttons.html b/wqflask/wqflask/templates/tool_buttons.html index 3f9d8211..3ee5be19 100644 --- a/wqflask/wqflask/templates/tool_buttons.html +++ b/wqflask/wqflask/templates/tool_buttons.html @@ -18,13 +18,13 @@ BNW </button> -<!-- <button id="wgcna_setup" class="btn btn-primary submit_special" data-url="/wgcna_setup" title="WGCNA Analysis" > +<button id="wgcna_setup" class="btn btn-primary submit_special" data-url="/wgcna_setup" title="WGCNA Analysis" > WGCNA </button> <button id="ctl_setup" class="btn btn-primary submit_special" data-url="/ctl_setup" title="CTL Analysis" > CTL Maps -</button> --> +</button> <button id="heatmap" class="btn btn-primary submit_special" data-url="/heatmap" title="Heatmap" > MultiMap diff --git a/wqflask/wqflask/templates/tutorials.html b/wqflask/wqflask/templates/tutorials.html index 89143809..aa6a818d 100644 --- a/wqflask/wqflask/templates/tutorials.html +++ b/wqflask/wqflask/templates/tutorials.html @@ -3,7 +3,7 @@ {% block content %} <head> - <title>OPAR - OSGA webinar series</title> + <title>GeneNetwork Webinar Series, Tutorials and Short Video Tours</title> <meta charset="utf-8"> <meta name="viewport" content="width=device-width, initial-scale=1"> <!-- <link rel="stylesheet" href="uikit-3/css/uikit.min.css" />--> @@ -17,21 +17,37 @@ <script src="https://cdn.jsdelivr.net/npm/uikit@3.5.4/dist/js/uikit-icons.min.js"></script> <!-- DataTables--> - <!--<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/dt/jq-3.3.1/dt-1.10.21/cr-1.5.2/datatables.min.css"/>--> + <!-- <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/dt/jq-3.3.1/dt-1.10.21/cr-1.5.2/datatables.min.css"/> --> <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.22/css/jquery.dataTables.min.css"/> - <!-- <script type="text/javascript" src="https://cdn.datatables.net/v/dt/jq-3.3.1/dt-1.10.21/cr-1.5.2/datatables.min.js"></script>--> + <script type="text/javascript" src="https://cdn.datatables.net/v/dt/jq-3.3.1/dt-1.10.21/cr-1.5.2/datatables.min.js"></script> <script type="text/javascript" src="https://code.jquery.com/jquery-3.5.1.js"></script> <script type="text/javascript" src="https://cdn.datatables.net/1.10.22/js/jquery.dataTables.min.js"></script> </head> <body> - <div class="uk-margin-small uk-card uk-card-default uk-card-body"> -<h2 class="uk-heading-small">Webinar Series - Quantitative Genetics Tools for Mapping Trait Variation to Mechanisms, Therapeutics, and Interventions</h2> -<p>The NIDA Center of Excellence in Omics, Systems Genetics, and the Addictome has put together a webinar series, Quantitative Genetics Tools for Mapping Trait Variation to Mechanisms, Therapeutics, and Interventions. The goal of this series is to transverse the path from trait variance to QTL to gene variant to molecular networks to mechanisms to therapeutic and interventions. The target audience for this series are those new to the field of quantitative genetics, so please pass this information on to your trainees or colleagues.</p> + <!-- <div class="row">--> + <!-- <div class="col-lg-12"> + <p>Primary Sponsor</p> + <img class="img-responsive" src="images/illumina_logo.jpg" alt=""> <h2 class="page-header">Program (Preliminary)</h2> + </div>--> + <div class="col-lg-12"> - <table id="myTable" class="display"> + <ul id="myTab" class="nav nav-tabs nav-justified"> + <li class="active"><a href="#service-one" data-toggle="tab"><i class="fa fa-file-text-o"></i>Webinars</a> + </li> + <li class=""><a href="#service-two" data-toggle="tab"><i class="fa fa-file-text-o"></i>Short Video Tours</a> + </li> + <li class=""><a href="#service-three" data-toggle="tab"><i class="fa fa-file-text-o"></i>Tutorials</a> + </li> + <li class=""><a href="#service-four" data-toggle="tab"><i class="fa fa-file-text-o"></i>Documentation</a> + </li> + </ul> +<p></p> + <div id="myTabContent" class="tab-content"> + <div class="tab-pane fade active in" id="service-one"> + <table id="myTable" class="display"> <thead> <tr> <th>Title/Description</th> @@ -40,9 +56,7 @@ </thead> <tbody> <tr> - <td><p><h3>Webinar #01 - Introduction to Quantitative Trait Loci (QTL) Analysis</h3> - <p><i>Friday, May 8th, 2020<br> - 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT</i></p> + <td><p><h3>Introduction to Quantitative Trait Loci (QTL) Analysis</h3> <p>Goals of this webinar (trait variance to QTL):</p> <ul> <li>Define quantitative trait locus (QTL)</li> @@ -60,10 +74,7 @@ University of Tennessee Health Science Center <iframe width="560" height="315" src="https://www.youtube.com/embed/leY3kPmnLaI" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></td> </tr> <tr> - <td><p><h3>Webinar #02 - Mapping Addiction and Behavioral Traits and Getting at Causal Gene Variants with GeneNetwork</h3> - <p><i>Friday, May 22nd. 2020 - 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT</i> -</p> + <td><p><h3>Mapping Addiction and Behavioral Traits and Getting at Causal Gene Variants with GeneNetwork</h3> <p>Goals of this webinar (QTL to gene variant):</p> <ul> <li>Demonstrate mapping a quantitative trait using GeneNetwork (GN)</li> @@ -78,438 +89,10 @@ University of Tennessee Health Science Center </p></td> <td><iframe width="560" height="315" src="https://www.youtube.com/embed/LwpXzLHX9aM" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></td> </tr> - <tr> - <td><p><h3>Webinar #03 - Introduction to expression (e)QTL and their role in connecting QTL to genes and molecular networks</h3> - <p><i>Friday, June 12, 2020 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT</i> - <!--<p>1 hour presentation followed by 30 minutes of discussion</p>--> - <p>Goals of this webinar (QTL to gene/molecular networks):</p> - <ul> - <li>Define eQTL</li> - <li>Examine the role of eQTL in the relationship of genes and molecular networks with phenotypic QTL</li> - <li>eQTL for co-expression networks</li> - </ul> - <p>Presented by:<br> -Dr. Laura Saba<br> -Associate Professor<br> -Department of Pharmaceutical Sciences<br> -University of Colorado Anschutz Medical Campus -</p><p> -<a href="/pdf/webinar_flyer_2020-06-12.pdf" target="_blank">Webinar flyer (pdf)</a><br> -<a href="https://github.com/OSGA-OPAR/quant-genetics-webinars/blob/master/2020-06-12/README.md">Link to course material</a><br> -</td> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/8jiNHuOgr1A" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></td> - </tr> - <tr> - <td><p><h3>Webinar #04 - From Candidate Genes to Causal Variants—Strategies for and Examples of Identifying Genes and Sequence Variants in Rodent Populations</h3> - <p><i>Friday, June 26, 2020 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT</i> - - <p>Goals of this webinar (candidate genes to causal variants):</p> - <ul> - <li>To understand when it is practical or (just as often) not practical to try to "clone" the gene or nucleotide variant modulating trait variants -</li> - <li>To understand that defining the crucial causal nucleotide variant is usually a bonus and often not for the translational or even mechanistic utility of discoveries. -</li> - <li>To review new sequence-based methods to identify common and rare variants—the reduced complexity cross and epoch-effects in reference populations -</li> - </ul> - <p>Presented by:<br> -Dr. Rob Williams<br> -Professor and Chair<br> -Department of Genetics, Genomics, and Informatics<br> -University of Tennessee Health Science Center -</p> - -<p> -<a href="https://github.com/OSGA-OPAR/quant-genetics-webinars/blob/master/2020-06-26/README.md">Link to course material</a><br> -Link to course material in powerpoint pptx: [<a href="pdf/P30_Webinar_on_QTGenes_26Jun2020v3.pptx" target="_blank">P30_Webinar_on_QTGenes_26Jun2020v3.pptx</a>]<br> -</p></td> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/u9d5dYxM5q4" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></td> - </tr> - <tr> - <td><p><h3>Webinar #05 - Identifying genes from QTL using RNA expression and the PhenoGen website (<a href="http://phenogen.org" target="_blank">http://phenogen.org</a>)</h3> - <p><i>Friday, August 28, 2020 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT<br> -1-hour presentation followed by 30 minutes of discussion</i> - - <p>Goals of this webinar (candidate genes to causal variants): - <p>Demonstrate how to use the PhenoGen website to identify transcripts:</p> - <ul> - <li>Physically located within a QTL</li> - <li>Physically located within a QTL and expressed in brain</li> - <li>With a brain cis eQTL within the QTL</li> - <li>With any brain eQTL within the QTL</li> - <li>Within a co-expression network controlled from the same region as the QTL</li> - </ul> - <p>Presented by:<br> -Dr. Laura Saba<br> -Associate Professor<br> -Department of Pharmaceutical Sciences<br> -University of Colorado Anschutz Medical Campus -</p> -<p> -<a href="https://github.com/OSGA-OPAR/quant-genetics-webinars/blob/master/2020-08-28/README.md">Link to course material</a><br> -</p></td> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/9DJm5cJgVis" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></td> - </tr> - <tr> - <td><p><h3>Webinar #06 - Sex as a Biological Covariate in QTL Studies</h3> - <p><i>Friday, September 11th, 2020 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT <br> -1-hour presentation followed by 30 minutes of discussion</i> - - <p>Goals of this webinar (trait variance to QTL): - - <ul> - <li>Review QTL mapping </li> - <li>Understand the role of sex in QTL study design </li> - <li>Use sex as a covariate in QTL analysis </li> - <li>Understand X chromosome segregation in crosses</li> - <li>Make adjustments for X chromosome in QTL analysis </li> - </ul> - <p>Presented by:<br> -Dr. Saunak Sen<br> -Professor and Chief of Biostatistics<br> -Department of Preventative Medicine<br> -University of Tennessee Health Science Center -<p><a href="https://github.com/OSGA-OPAR/quant-genetics-webinars/blob/master/2020-09-11/README.md">Link to course material</a> -</p></td> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/dYeJcBbJjRU" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></td> - </tr> - <tr> - <td><h3>Webinar #07 - Introduction to Weighted Gene Co-expression Network Analysis</h3> - <p><i>Friday, September 25th at 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT <br> -1-hour presentation followed by 30 minutes of discussion </i> - - <p>Goals of this webinar (molecular networks): - <ul> - <li>Introduction and motivation for co-expression network analysis </li> - <li>Basics of weighted gene co-expression network analysis </li> - <li>Step-by-step guide to WGCNA using the wgcna package in R. </li> - </ul> - <p>Background reading available at: <a href="http://bit.ly/osga_wgcna">http://bit.ly/osga_wgcna</a></p> - <p>Presented by:<br> -Dr. Laura Saba<br> -Associate Professor<br> -Department of Pharmaceutical Sciences<br> -University of Colorado Anschutz Medical Campus -<p><a href="https://github.com/OSGA-OPAR/quant-genetics-webinars/blob/master/2020-09-11/README.md">Link to course material</a> -</p></td> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/OpWEHazyQLA" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></td> - </tr> - <tr> - <td><p><h3>Webinar #08 - Using genetic and non-genetic covariates in QTL studies</h3> - <p><i>Friday, October 9th at 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT <br> -1-hour presentation followed by 30 minutes of discussion </i> - - <p>Goals of this webinar (quantitative trait to genetic loci): - <ul> - <li>Identify covariates and mediators in QTL studies </li> - <li>Adjust for covariates in QTL scans </li> - <li>Review genetic relatedness in segregating populations </li> - <li>Adjust for genetic relatedness using linear mixed models </li> - </ul> - <!--<p>Background reading available at: <a href="http://bit.ly/osga_wgcna">http://bit.ly/osga_wgcna</a></p>--> - <p>Presented by:<br> -Dr. Saunak Sen<br> -Professor and Chief of Biostatistics<br> -Department of Preventative Medicine<br> -University of Tennessee Health Science Center -</p><p><a href="https://github.com/OSGA-OPAR/quant-genetics-webinars/blob/master/2020-10-09/README.md">Link to course material</a></td> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/1U_4DCSDq9U" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></td> - </tr> - <tr> - <td><p><h3>Webinar #09 - Introduction to GeneWeaver: Integrating and analyzing heterogeneous functional genomics data</h3> - <p><i>Friday, October 23th at 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT <br> -1-hour presentation followed by 30 minutes of discussion </i> - <p>Goals of this webinar: - <ul> - <li>Compare a user's gene list with multiple functional genomics data sets </li> - <li>Compare and contrast gene lists with data currently available and integrated in GeneWeaver</li> - <li>Explore functional relationships among genes and disease across species </li> - </ul> - <!--<p>Background reading available at: <a href="http://bit.ly/osga_wgcna">http://bit.ly/osga_wgcna</a></p>--> - <p>Presented by:<br> -Dr. Elissa Chesler<br> -Professor The Jackson Laboratory -</p> -<p> -Dr. Erich Baker<br> -Professor and Chair<br> -Department of Computer Science<br> -Baylor University -<p><a href="https://github.com/OSGA-OPAR/quant-genetics-webinars/blob/master/2020-10-23/README.md">Link to course material</a></p> -<!--After the presentation, the recording will be made available for download at <a href="http://opar.io">http://opar.io</a><br>--></p> - - -<p></td> - <!--<td><strong>After the presentation, the recording will be made available here.</strong></td>--> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/Vq3vocdMWLQ" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></td> - </tr> - - - - -<tr> - <td><p><h3>Webinar #10 - Sketching alternate realities: An introduction to causal inference in genetic studies</h3> - <p><i>Friday, November 20th at 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT<br> - 1-hour presentation followed by 30 minutes of discussion</i> - - <p>Goals of this webinar: - <p>Determination of cause is an important goal of biological studies, and genetic studies provide unique opportunities. In this introductory lecture we will frame causal inference as a missing data problem to clarify challenges, assumptions, and strategies necessary for assigning cause. We will survey the use of directed acyclic graphs (DAGs) to express causal information and to guide analytic strategies. - <ul> - <li>Express causal inference as a missing data problem (counterfactual framework) </li> - <li>Outline assumptions needed for causal inference </li> - <li>Express causal information as (directed acyclic) graphs </li> - <li>Outline how to use graphs to guide analytic strategy </li> - </ul> - - <p>Presented by:<br> -Dr. Saunak Sen<br> -Professor and Chief of Biostatistics<br> -Department of Preventative Medicine<br> -University of Tennessee Health Science Center -<p><a href="https://github.com/OSGA-OPAR/quant-genetics-webinars/blob/master/2020-11-20/README.md">Link to course material</a> -</p> -<!--<p>There is no fee associated with this webinar, but users are asked to register to receive the Zoom link and password. -Registration: <a href="https://bit.ly/osga_2020-11-20">https://bit.ly/osga_2020-11-20</a>--> -</td> - <!--<td><strong>After the presentation, the recording will be made available here.</strong></td>--> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/twJNYOL3qfA" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> - </tr> - <!--NEW WEBINAR STARTS HERE--> <tr> - <td><p><h3>Webinar #11 - Beginner's guide to bulk RNA-Seq analysis</h3> - <p><i>Friday, February 12th, 2021 at 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT <br> - 1-hour presentation followed by 30 minutes of discussion</i> - - <p>Goals of this webinar: - <p>The use of high throughput short read RNA sequencing has become common place in many scientific laboratories. The analysis tools for quantitating a transcriptome have matured becoming relatively simple to use. The goals of this webinar are: - <ul> - <li>To give a general overview of the popular Illumina technology for sequencing RNA. </li> - <li>To outline several of the key aspects to consider when designing an RNA-Seq study </li> - <li>To provide guidance on methods and tools for transforming reads to quantitative expression measurements. </li> - <li>To describe statistical models that are typically used for differential expression and why these specialized models are needed.</li> - </ul> - - <p>Presented by:<br> -Dr. Laura Saba<br> -Associate Professor<br> -Department of Pharmaceutical Sciences<br> -University of Colorado Anschutz Medical Campus -<p><a href="https://github.com/OSGA-OPAR/quant-genetics-webinars/blob/master/2021-02-12/README.md">Link to course material</a> -</p> -<p><a href="/pdf/webinar_flyer_2021-02-12.pdf" target="_blank">Webinar flyer (pdf)</a></p> -<!--<p>There is no fee associated with this webinar, but users are asked to register to receive the Zoom link and password. -Registration: <a href="http://bit.ly/osga_2021-02-12">http://bit.ly/osga_2021-02-12</a>--> -<p>This webinar series is sponsored by the NIDA Center of Excellence in Omics, Systems Genetics, and the Addictome (P30 DA044223) and the NIAAA-funded PhenoGen Website (R24 AA013162).</p> -</td> - <!--<td><strong>After the presentation, the recording will be made available here.</strong></td>--> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/WW94W-DBf2U" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></td> - </tr> - <!--WEBINAR ENDS HERE--> - - <!--NEW WEBINAR STARTS HERE--> - <tr> - <td><p><h3>Webinar #12 - From GWAS to gene: What are the essential analyses and how do we bring them together using heterogeneous stock rats?</h3> - <p><i>Friday, February 26th at 10am PST/ 11am MST/ 12pm CST/ 1pm EST<br> - 1-hour presentation followed by 30 minutes of discussion</i> - - <p>Goals of this webinar: - <p>Heterogeneous stock (HS) rats are an outbred population that was created in 1984 by intercrossing 8 inbred strains. The Center for GWAS in Outbred Rats (www.ratgenes.org) has developed a suite of analysis tools for analyzing genome wide association studies (GWAS) in HS rats - <ul> - <li>Explain the HS rat population and their history</li> - <li>Describe the automated pipeline that performs GWAS in HS rats</li> - <li>Explore the fine mapping of associated regions and explain the various secondary analyses that we use to prioritize genes within associated intervals</li> - </ul> - - <p>Presented by:<br> -Abraham A. Palmer, Ph.D.<br> -Professor & Vice Chair for Basic Research<br> -Department of Psychiatry<br> -University of California San Diego -</p> -<p><a href="/pdf/webinar_flyer_2021-02-26.pdf" target="_blank">Webinar flyer (pdf)</a></p> -<p><a href="https://github.com/OSGA-OPAR/quant-genetics-webinars/blob/master/2021-02-26/README.md">Link to course material</a> -<p>Link to course material in pptx:<a href="/pdf/Palmer_talk_2-26-21.pptx" target="_blank"> Palmer_talk_2-26-21.pptx</a></p> -<!--<p>There is no fee associated with this webinar, but users are asked to register to receive the Zoom link and password. -Registration: <a href="http://bit.ly/osga_2021-02-26 ">http://bit.ly/osga_2021-02-26</a>--> -<p>This webinar series is sponsored by the NIDA Center of Excellence in Omics, Systems Genetics, and the Addictome (P30 DA044223) and the NIDA-funded Center for GWAS in Outbred Rats (P50 DA037844).</p> -</td> - <!--<td><strong>After the presentation, the recording will be made available here.</strong></td>--> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/aWUxNZ9wS3E" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> - </tr> - <!--WEBINAR ENDS HERE--> - - <!--NEW WEBINAR STARTS HERE--> - <tr> - <td><p><h3>Webinar #13 - Become a UseR: A brief tour of R</h3> - <p><i>Friday, March 12th at 10am PST/ 11am MST/ 12pm CST/ 1pm EST<br> - 1-hour presentation followed by 30 minutes of discussion</i> - <p>We will introduce R programming language and outline the benefits of learning R. We will give a brief tour of basic concepts and tasks: variables, objects, functions, basic statistics, visualization, and data import/export. We will showcase a practical example demonstrating statistical analysis. - - <p>Goals of this webinar: - <ul> - <li>Why should one use/learn R?</li> - <li>How to install R/Rstudio</li> - <li>Learn about R basics: variables, programming, functions</li> - <li>Learn about the R package ecosystem that extends its capabilities</li> - <li>See a basic statistical analysis example</li> - <li>Learn about additional resources</li> - </ul> - - <p>Presented by:<br> -Gregory Farage, PhD and Saunak Sen, PhD<br> -Department of Preventive Medicine<br> -University of Tennessee Health Science Center -<p><a href="https://github.com/OSGA-OPAR/quant-genetics-webinars/blob/master/2021-03-12/README.md">Link to course material</a> -</p> -<p><a href="/pdf/webinar_flyer_2021-03-12.pdf" target="_blank">Webinar flyer (pdf)</a></p> -<!--<p>Link to course material in pptx:<a href="/pdf/Palmer_talk_2-26-21.pptx" target="_blank"> Palmer_talk_2-26-21.pptx</a></p>--> -<!--<p>There is no fee associated with this webinar, but users are asked to register to receive the Zoom link and password. -Registration: <a href="http://bit.ly/osga_2021-03-12">http://bit.ly/osga_2021-03-12</a></p>--> -</td> - <!--<td><strong>After the presentation, the recording will be made available here.</strong></td>--> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/25-X8_oXBSY" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> - </tr> - <!--WEBINAR ENDS HERE--> - - <!--NEW WEBINAR STARTS HERE--> - <tr> - <td><p><h3>Webinar #14 - Landing on Jupyter: A guided tour of interactive notebooks</h3> - <p><i>Friday, March 26th at 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT<br> - 1-hour presentation followed by 30 minutes of discussion</i> - <p>Jupyter is an interactive interface to data science and scientific computing across a variety of programming languages. We will present the Jupyter notebook, and explain some key concepts (e.g., kernel, cells). We will show how to create a new notebook; modify an existing notebook; save, export, and publish a notebook. We will discuss several possible use cases: developing code, writing reports, taking notes, and teaching/presenting. - - <p>Goals of this webinar: - <ul> - <li>Learn what Jupyter notebooks are</li> - <li>Learn how to install, configure, and use Jupyter notebooks</li> - <li>Learn how to use Jupyter notebooks for research, teaching, or code - development </li> - </ul> - - <p>Presented by:<br> -Gregory Farage, PhD and Saunak Sen, PhD<br> -Department of Preventive Medicine<br> -University of Tennessee Health Science Center -<p><a href="https://github.com/OSGA-OPAR/quant-genetics-webinars/blob/master/2021-03-26/README.md">Link to course material</a> -</p> -<p><a href="/pdf/webinar_flyer_2021-03-26.pdf" target="_blank">Webinar flyer (pdf)</a></p> -<!--<p>Link to course material in pptx:<a href="/pdf/Palmer_talk_2-26-21.pptx" target="_blank"> Palmer_talk_2-26-21.pptx</a></p>--> -<!--<p>There is no fee associated with this webinar, but users are asked to register to receive the Zoom link and password. -Registration: <a href="http://bit.ly/osga_2021-03-26">http://bit.ly/osga_2021-03-26</a>--> -</td> - <!--<td><strong>After the presentation, the recording will be made available here.</strong></td>--> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/GVzUNEmpanI" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></iframe> - </tr> - <!--WEBINAR ENDS HERE--> - - <!--NEW WEBINAR STARTS HERE--> - <tr> - <td><p><h3>Webinar #15 – Introduction to Metabolomics Platforms and Data Analysis</h3> - <p><i>Friday, April 9th at 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT <br> - 1-hour presentation followed by 30 minutes of discussion</i> - <p>Goals of this webinar: - <p>The use of metabolomics to profile small molecules is now widespread in biomedical research. The goals of this webinar are: - <ul> - <li>To describe research questions that can be addressed using metabolomics</li> - <li>To give a general overview of metabolomics technologies</li> - <li>To outline steps in a metabolomics data analysis pipeline</li> - <li>To provide information on common resources and databases</li> - </ul> - - <p>Presented by:<br> -Katerina Kechris, PhD <br> -Professor<br> -Department of Biostatistics and Informatics <br> -Colorado School of Public Health <br> -University of Colorado Anschutz Medical Campus -<p><a href="https://github.com/OSGA-OPAR/quant-genetics-webinars/blob/master/2021-04-09/README.md">Link to course material</a> -</p> -<p><a href="/pdf/webinar_flyer_2021-04-09.pdf" target="_blank">Webinar flyer (pdf)</a></p> -<!--<p>Link to course material in pptx:<a href="/pdf/Palmer_talk_2-26-21.pptx" target="_blank"> Palmer_talk_2-26-21.pptx</a></p> -<p>There is no fee associated with this webinar, but users are asked to register to receive the Zoom link and password. -Registration: <a href="https://bit.ly/osga_2021-04-09">https://bit.ly/osga_2021-04-09</a></p>--> -</td> - <!--<td><strong>After the presentation, the recording will be made available here.</strong></td>--> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/oB1Khk6mt_8" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></td> - </tr> - <!--WEBINAR ENDS HERE--> - - <!--NEW WEBINAR STARTS HERE--> - <tr> - <td><p><h3>Webinar #16 – Introduction to the Hybrid Rat Diversity Panel: A renewable rat panel for genetic studies of addiction-related traits</h3> - <p><i>Friday, April 23rd at 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT <br> - 1-hour presentation followed by 30 minutes of discussion</i> - <p>Goals of this webinar: - <p>The Hybrid Rat Diversity Panel (HRDP) is an inbred panel of rats that included two recombinant inbred panels and a panel of classic inbred strains. - <ul> - <li>To describe hybrid diversity panels, in particular the HRDP, including advantages and disadvantages when studying the role of genetics is substance use disorders, e.g., renewable genomes and the accumulation of behavioral and physiological phenotypes and high throughput omics data. </li> - <li>To outline current resources and resources that are being generated. </li> - <li>To demonstrate the utility of a renewable genetically diverse rodent population when exploring the interaction between genetics, drug exposure, and behavior. </li> - </ul> - - <p>Presented by:<br> - Hao Chen, PhD<br> -Associate Professor<br> -Department of Pharmacology, Addiction Science, and Toxicology <br> -University of Tennessee Health Science Center -<p> -Dr. Laura Saba<br> -Associate Professor<br> -Department of Pharmaceutical Sciences<br> -University of Colorado Anschutz Medical Campus -<p><a href="https://github.com/OSGA-OPAR/quant-genetics-webinars/blob/master/2021-04-23/README.md">Link to course material</a> -</p> -<p><a href="/pdf/webinar_flyer_2021-04-09.pdf" target="_blank">Webinar flyer (pdf)</a></p> -<!--<p>Link to course material in pptx:<a href="/pdf/Palmer_talk_2-26-21.pptx" target="_blank"> Palmer_talk_2-26-21.pptx</a></p> -<p>There is no fee associated with this webinar, but users are asked to register to receive the Zoom link and password. -Registration: <a href="http://bit.ly/osga_2021-04-23">http://bit.ly/osga_2021-04-23</a> -<p>This webinar series is sponsored by the NIDA Center of Excellence in Omics, Systems Genetics, and the Addictome (P30 DA044223) and RGAP: The heritable transcriptome and alcoholism (R24 AA013162). </p> -</td> - <td><strong>After the presentation, the recording will be made available here.</strong></td>--> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/dYIiv01IetQ" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></td> - </tr> - <!--WEBINAR ENDS HERE--> - - <!--NEW WEBINAR STARTS HERE--> - <tr> - <td><p><h3>Webinar #17 – Identifying sample mix-ups in eQTL data</h3> - <p><i>Friday, June 11th at 10am PDT/ 11am MDT/ 12pm CDT/ 1pm EDT <br> - 1-hour presentation followed by 30 minutes of discussion</i> - <p>Goals of this webinar: - <p>Sample mix-ups interfere with our ability to detect genotype-phenotype associations. However, the presence of numerous eQTL with strong effects provides the opportunity to not just identify sample mix-ups, but also to correct them. - <ul> - <li>To illustrate methods for identifying sample duplicates and errors in sex annotations. </li> - <li>To illustrate methods for identifying sample mix-ups in DNA and RNA samples from experimental cross data.</li> - </ul> - - <p>Presented by:<br> - Karl Broman, PhD <br> -Professor<br> -Department of Biostatistics & Medical Informatics <br> -University of Wisconsin-Madison -<p> - -<p><a href="/pdf/webinar_flyer_2021-06-11.pdf" target="_blank">Webinar flyer (pdf)</a></p> -<p>Link to course material:<a href="kbroman.org/Talk_OSGA2021" target="_blank">kbroman.org/Talk_OSGA2021</a></p> -<!--<p>There is no fee associated with this webinar, but users are asked to register to receive the Zoom link and password. -Registration: <a href="http://bit.ly/osga_2021-06-11">http://bit.ly/osga_2021-06-11</a>--> - <!--<p>This webinar series is sponsored by the NIDA Center of Excellence in Omics, Systems Genetics, and the Addictome (P30 DA044223) and RGAP: The heritable transcriptome and alcoholism (R24 AA013162). </p>--> -</td> - <!-- <td><strong>After the presentation, the recording will be made available here.</strong></td>--> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/h5gF7YnffeI" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></td> - </tr> - <!--WEBINAR ENDS HERE--> - - - - - <!--NEW WEBINAR STARTS HERE--> - <tr> - <td><p><h3>Bonus 1 - Data structure, disease risk, GXE, and causal modeling</h3> - <p><i>Friday, November 20th at 9am PDT/ 11pm CDT/ 12pm EDT<br> - 1-hour presentation followed by 30 minutes of discussion</i> + <td><p><h3>Data structure, disease risk, GXE, and causal modeling</h3> <p>Human disease is mainly due to complex interactions between genetic and environmental factors (GXE). We need to acquire the right "smart" data types—coherent and multiplicative data—required to make accurate predictions about risk and outcome for n = 1 individuals—a daunting task. We have developed large families of fully sequenced mice that mirror the genetic complexity of humans. We are using these Reference Populations to generate multiplicatively useful data and to build and test causal quantitative models of disease mechanisms with a special focus on diseases of aging, addiction, and neurological and psychiatric disease. @@ -530,9 +113,28 @@ Registration: <a href="https://bit.ly/osga_2020-11-20">https://bit.ly/osga_2020- </tr> <!--WEBINAR ENDS HERE--> + </tbody> + </table> + +<script> +$('#myTable').dataTable( { + "lengthMenu": [ 50, 75, 100 ] +} ); +</script> + </div> + <div class="tab-pane fade" id="service-two"> + <table id="myTable2" class="display"> + <thead> + <tr> + <th>Title/Description</th> + <th>Presentation</th> + </tr> + </thead> + <tbody> + <!--NEW WEBINAR STARTS HERE--> <tr> - <td><p><h3>Bonus 2 - Introduction to Gene Network</h3> + <td><p><h3>Introduction to Gene Network</h3> <p><i>Please note that this tutorial is based on GeneNetwork v1</i> <p>GeneNetwork is a group of linked data sets and tools used to study complex networks of genes, molecules, and higher order gene function and phenotypes. GeneNetwork combines more than 25 years of legacy data generated by hundreds of scientists together with sequence data (SNPs) and massive transcriptome data sets (expression genetic or eQTL data sets). The quantitative trait locus (QTL) mapping module that is built into GN is optimized for fast on-line analysis of traits that are controlled by combinations of gene variants and environmental factors. GeneNetwork can be used to study humans, mice (BXD, AXB, LXS, etc.), rats (HXB), Drosophila, and plant species (barley and Arabidopsis). Most of these population data sets are linked with dense genetic maps (genotypes) that can be used to locate the genetic modifiers that cause differences in expression and phenotypes, including disease susceptibility. @@ -550,39 +152,102 @@ Registration: <a href="https://bit.ly/osga_2020-11-20">https://bit.ly/osga_2020- <!--<p>This webinar series is sponsored by the NIDA Center of Excellence in Omics, Systems Genetics, and the Addictome (P30 DA044223). --> </td> <!--<td><strong>After the presentation, the recording will be made available here.</strong></td>--> - <td><iframe width="560" height="315" src="https://www.youtube.com/embed/B3g_0q-ldJ8" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></iframe> + <td> + <iframe width="560" height="315" src="https://www.youtube.com/embed/B3g_0q-ldJ8" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> </tr> <!--WEBINAR ENDS HERE--> + + <!--NEW WEBINAR STARTS HERE--> + <tr> + <td><p><h3>How to search in GeneNetwork</h3><br>Presented by Rob Williams University of Tennessee Health Science Center</td> + <!--<td><p><h3>How to search in GeneNetwork</h3> +</td>--> + <td> + <iframe width="560" height="315" src="https://www.youtube.com/embed/5exnkka5Tso" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> </td> + </tr> + <!--WEBINAR ENDS HERE--> + + <!--NEW WEBINAR STARTS HERE--> + <tr> + <td><p><h3>GeneNetwork.org: genetic analysis for all neuroscientists</h3><br>Presented by David G. Ashbrook Assistant Professor University of Tennessee Health Science Center +</td> + <td> + <iframe width="560" height="315" src="https://www.youtube.com/embed/JmlVLki09Q8" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></td> + </tr> + <!--WEBINAR ENDS HERE--> </tbody> </table> -</div> <script> -$('#myTable').dataTable( { +$('#myTable2').dataTable( { "lengthMenu": [ 50, 75, 100 ] } ); </script> + </div> + <div class="tab-pane fade" id="service-three"> + <table id="myTable3" class="display"> + <thead> + <tr> + <th>Title</th> + <th>Speaker</th> + <th>Video link</th> + </tr> + </thead> + <tbody> + <tr><td>Diallel Crosses, Artificial Intelligence, and Mouse Models of Alzheimer’s Disease</td> + <td>David G. Ashbrook<br>Assistant Professor<br>University of Tennessee Health Science Center</td> + <td><a href="https://www.youtube.com/watch?v=HKfYc8CJwqM">YouTube link</a></td> + </tr> -<!--<script> -$(document).ready( function () { -$('#myTable').DataTable({ - "lengthMenu": [ [50, 75, 100] + + </tbody> + </table> + <script> +$('#myTable3').dataTable( { + "lengthMenu": [ 50, 75, 100 ] } ); +</script> + </div> + + <div class="tab-pane fade" id="service-four"> + <table id="myTable4" class="display"> + <thead> + <tr> + <th>Title</th> + </tr> + </thead> + <tbody> + <tr><td><a href="https://www.biorxiv.org/content/10.1101/2020.12.23.424047v1">GeneNetwork: a continuously updated tool for systems genetics analyses</a></td></tr> + <tr><td><a href="https://www.biorxiv.org/content/10.1101/2021.05.24.445383v1">Old data and friends improve with age: Advancements with the updated tools of GeneNetwork</a></td> + <tr><td><a href="https://www.opar.io/pdf/Rat_HRDP_Brain_Proteomics_Wang_WIlliams_08Oct2021.pdf">A Primer on Brain Proteomics and protein-QTL Analysis for Substance Use Disorders</a></td></tr> + + </tbody> + </table> + <script> +$('#myTable4').dataTable( { + "lengthMenu": [ 50, 75, 100 ] } ); -</script>--> +</script> + </div> + </div> -<!--<script> -$(document).ready( function () { -$('#myTable').DataTable({ - "lengthMenu": [ [50, 75, 100, -1], [50, 75, 100, "All"] ]); -} ); -</script>--> + </div> + </div> -<!--<script> -$(document).ready( function () { -$('#myTable').DataTable(); -} ); -</script>--> +</div> + + <hr> + + + + </div> + <!-- /.container --> + + <!-- jQuery --> + <script src="js/jquery.js"></script> + + <!-- Bootstrap Core JavaScript --> + <script src="js/bootstrap.min.js"></script> </body> diff --git a/wqflask/wqflask/templates/wgcna_setup.html b/wqflask/wqflask/templates/wgcna_setup.html index c5461497..d7acd5f2 100644 --- a/wqflask/wqflask/templates/wgcna_setup.html +++ b/wqflask/wqflask/templates/wgcna_setup.html @@ -1,49 +1,142 @@ {% extends "base.html" %} {% block title %}WCGNA analysis{% endblock %} +{% block content %} +<!-- Start of body --> +<style type="text/css"> + +#terminal { + margin-top: 10px; +} + +</style> + +<link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='xterm/xterm.min.css') }}" /> -{% block content %} <!-- Start of body --> -<h1> WGCNA analysis parameters</h1> <div class="container"> - {% if request.form['trait_list'].split(",")|length < 4 %} - <div class="alert alert-danger" role="alert"> - <span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span> - <span class="sr-only">Error:</span> - <h2>Too few phenotypes as input</h2> - Please make sure you select enough phenotypes / genes to perform WGCNA. Your collection needs to contain at least 4 different phenotypes. You provided {{request.form['trait_list'].split(',')|length}} phenotypes as input. - </div> - {% else %} - <form action="/wgcna_results" method="post" class="form-horizontal"> - <input type="hidden" name="trait_list" id="trait_list" value= "{{request.form['trait_list']}}"> - <div class="form-group"> - <label for="SoftThresholds"> Soft threshold: </label> - <div class="col-sm-10"> - <input type="text" class="form-inline" name="SoftThresholds" id="SoftThresholds" value="1,2,3,4,5,6,7,8,9"> - </div> - </div> - <div class="form-group"> - <label for="MinModuleSize"> Minimum module size: </label> - <div class="col-sm-10"> - <input type="text" class="form-inline" name="MinModuleSize" id="MinModuleSize" value="30"> - </div> - </div> - <div class="form-group"> - <label for="TOMtype"> TOMtype: </label> - <div class="col-sm-10"> - <input type="text" class="form-inline" name="TOMtype" id="TOMtype" value="unsigned"> - </div> - </div> - <div class="form-group"> - <label for="mergeCutHeight"> mergeCutHeight: </label> - <div class="col-sm-10"> - <input type="text" class="form-inline" name="mergeCutHeight" id="mergeCutHeight" value="0.25"> - </div> - </div> - <div class="form-group"> - <div class="col-sm-10"> - <input type="submit" class="btn btn-primary" value="Run WGCNA using these settings" /> - </div> + <div class="col-md-5"> + <h1 class="mx-3 my-2 "> WGCNA analysis parameters</h1> + {% if request.form['trait_list'].split(",")|length < 4 %} <div class="alert alert-danger" role="alert"> + <span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span> + <span class="sr-only">Error:</span> + <h2>Too few phenotypes as input</h2> + Please make sure you select enough phenotypes / genes to perform WGCNA. Your collection needs to contain at least 4 different phenotypes. You provided {{request.form['trait_list'].split(',')|length}} phenotypes as input. </div> - </form> - {% endif %} + {% else %} + <form class="col-md-12" action="/wgcna_results" method="post" class="form-horizontal" id="wgcna_form"> + <input type="hidden" name="trait_list" id="trait_list" value="{{request.form['trait_list']}}"> + <div class="form-group row "> + <label for="SoftThresholds" class="col-md-3 col-form-label col-form-label-sm">Soft threshhold</label> + <div class="col-md-9"> + <input type="text" class="form-control form-control-md" value="1,2,3,4,5,6,7,8,9" id="SoftThresholds" name="SoftThresholds"> + </div> + </div> + <div class="form-group row "> + <label for="MinModuleSize" class="col-md-3 col-form-label col-form-label-sm">Minimum module size:</label> + <div class="col-md-9"> + <input type="text" class="form-control form-control-md" id="MinModuleSize" value="30" name="MinModuleSize"> + </div> + </div> + + <div class="form-group row"> + <label for="TOMtype" class="col-md-3 col-form-label col-form-label-sm">TOMtype:</label> + <div class="col-md-9"> + <select class="form-control" id="TOMtype" name="TOMtype"> + <option value="unsigned">unsigned</option> + <option value="signed">signed</option> + </select> + </div> + + </div> + <div class="form-group row "> + <label for="mergeCutHeight" class="col-md-3 col-form-label col-form-label-sm">mergeCutHeight:</label> + <div class="col-md-9"> + <input type="text" class="form-control form-control-md" id="mergeCutHeight" value="0.25" name="mergeCutHeight"> + </div> + </div> + + <div class="form-group row"> + <label for="corType" class="col-md-3 col-form-label col-form-label-sm">corType:</label> + <div class="col-md-9"> + <select class="form-control col-md-9" id="corType" name="corType"> + <option value="pearson">pearson</option> + <option value="bicor">bicor</option> + </select> + </div> + + </div> + <div class="form-group"> + <div class="text-center"> + <input type="submit" class="btn btn-primary" value="Run WGCNA using these settings" /> + </div> + </div> + + + + </form> + {% endif %} </div> -{% endblock %} +<div class="col-md-7"> + <div id="terminal" class="mt-2"> + </div> +</div> +</div> + +<script src="{{ url_for('js', filename='xterm/xterm.min.js') }}" type="text/javascript"></script> +<script src="{{ url_for('js', filename='jquery/jquery.min.js') }}" type="text/javascript"></script> +<script src="https://cdn.jsdelivr.net/npm/xterm-addon-fit@0.5.0/lib/xterm-addon-fit.min.js"></script> + +<script> +document.addEventListener('DOMContentLoaded', function() { +let term = new Terminal({ + cursorBlink: true, + lineHeight: 1.3, + scrollback: true, + macOptionIsMeta: true +}); + +let termDebugs = { + general: "Computation process to be displayed here....", + success: "Computation in process ......", + fail: "Too few phenotypes as input must be >=4" +} + +const fitAddon = new FitAddon.FitAddon() +term.loadAddon(fitAddon) + +term.open(document.getElementById('terminal')); +term.setOption('theme', { + background: '#300a24' +}); +term.writeln(termDebugs.general) + +wgcnaForm = document.querySelector("#wgcna_form") + +fitAddon.fit() +term.onData((data) => { + term.write(data) +}) + + +if (wgcnaForm) { +} else { + term.writeln(termDebugs.fail) +} + +$(document).on('submit', '#wgcna_form', function(e) { + term.writeln(termDebugs.success) + + e.preventDefault(); + var form = $(this); + $.ajax({ + type: 'POST', + url: '/wgcna_results', + data: form.serialize(), + success: function(data) { + document.write(data) + } + }) +}) +}) + +</script> +{% endblock %}
\ No newline at end of file diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py index 67e2e158..cef50cd9 100644 --- a/wqflask/wqflask/user_session.py +++ b/wqflask/wqflask/user_session.py @@ -10,7 +10,6 @@ from flask import (Flask, g, render_template, url_for, request, make_response, from wqflask import app from utility import hmac -#from utility.elasticsearch_tools import get_elasticsearch_connection from utility.redis_tools import get_redis_conn, get_user_id, get_user_by_unique_column, set_user_attribute, get_user_collections, save_collections Redis = get_redis_conn() @@ -23,7 +22,6 @@ THIRTY_DAYS = 60 * 60 * 24 * 30 @app.before_request def get_user_session(): - logger.info("@app.before_request get_session") g.user_session = UserSession() # ZS: I think this should solve the issue of deleting the cookie and redirecting to the home page when a user's session has expired if not g.user_session: diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 6936ce78..be3d9238 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -4,7 +4,6 @@ import MySQLdb import array import base64 import csv -import difflib import datetime import flask import io # Todo: Use cStringIO? @@ -20,8 +19,6 @@ import traceback import uuid import xlsxwriter -from itertools import groupby -from collections import namedtuple from zipfile import ZipFile from zipfile import ZIP_DEFLATED @@ -30,19 +27,12 @@ from wqflask import app from gn3.commands import run_cmd from gn3.computations.gemma import generate_hash_of_string from gn3.db import diff_from_dict -from gn3.db import fetchall -from gn3.db import fetchone from gn3.db import insert from gn3.db import update from gn3.db.metadata_audit import MetadataAudit from gn3.db.phenotypes import Phenotype from gn3.db.phenotypes import Probeset from gn3.db.phenotypes import Publication -from gn3.db.phenotypes import PublishXRef -from gn3.db.phenotypes import probeset_mapping -from gn3.db.traits import get_trait_csv_sample_data -from gn3.db.traits import update_sample_data - from flask import current_app from flask import g @@ -79,6 +69,7 @@ from wqflask.correlation_matrix import show_corr_matrix from wqflask.correlation import corr_scatter_plot # from wqflask.wgcna import wgcna_analysis # from wqflask.ctl import ctl_analysis +from wqflask.wgcna.gn3_wgcna import run_wgcna from wqflask.snp_browser import snp_browser from wqflask.search_results import SearchResultPage from wqflask.export_traits import export_search_results_csv @@ -118,43 +109,16 @@ logger = utility.logger.getLogger(__name__) @app.before_request def connect_db(): - logger.info("@app.before_request connect_db") db = getattr(g, '_database', None) if db is None: g.db = g._database = sqlalchemy.create_engine( SQL_URI, encoding="latin1") - logger.debug(g.db) - - -@app.before_request -def check_access_permissions(): - logger.debug("@app.before_request check_access_permissions") - if 'dataset' in request.args: - permissions = DEFAULT_PRIVILEGES - if request.args['dataset'] != "Temp": - dataset = create_dataset(request.args['dataset']) - - if dataset.type == "Temp": - permissions = DEFAULT_PRIVILEGES - elif 'trait_id' in request.args: - permissions = check_resource_availability( - dataset, request.args['trait_id']) - elif dataset.type != "Publish": - permissions = check_resource_availability(dataset) - - if type(permissions['data']) is list: - if 'view' not in permissions['data']: - return redirect(url_for("no_access_page")) - else: - if permissions['data'] == 'no-access': - return redirect(url_for("no_access_page")) @app.teardown_appcontext def shutdown_session(exception=None): db = getattr(g, '_database', None) if db is not None: - logger.debug("remove db_session") db_session.remove() g.db = None @@ -167,9 +131,8 @@ def handle_generic_exceptions(e): time_str = now.strftime('%l:%M%p UTC %b %d, %Y') # get the stack trace and send it to the logger exc_type, exc_value, exc_traceback = sys.exc_info() - formatted_lines = {f"{request.url} ({time_str}) " - f" {traceback.format_exc().splitlines()}"} - + formatted_lines = (f"{request.url} ({time_str}) \n" + f"{traceback.format_exc()}") _message_templates = { werkzeug.exceptions.NotFound: ("404: Not Found: " f"{time_str}: {request.url}"), @@ -178,8 +141,8 @@ def handle_generic_exceptions(e): werkzeug.exceptions.RequestTimeout: ("408: Request Timeout: " f"{time_str}: {request.url}")} # Default to the lengthy stack trace! - logger.error(_message_templates.get(exc_type, - formatted_lines)) + app.logger.error(_message_templates.get(exc_type, + formatted_lines)) # Handle random animations # Use a cookie to have one animation on refresh animation = request.cookies.get(err_msg[:32]) @@ -188,7 +151,7 @@ def handle_generic_exceptions(e): "./wqflask/static/gif/error") if fn.endswith(".gif")]) resp = make_response(render_template("error.html", message=err_msg, - stack=formatted_lines, + stack={formatted_lines}, error_image=animation, version=GN_VERSION)) @@ -376,6 +339,12 @@ def wcgna_setup(): return render_template("wgcna_setup.html", **request.form) +@app.route("/wgcna_results", methods=('POST',)) +def wcgna_results(): + """call the gn3 api to get wgcna response data""" + results = run_wgcna(dict(request.form)) + return render_template("test_wgcna_results.html", **results) + @app.route("/ctl_setup", methods=('POST',)) def ctl_setup(): # We are going to get additional user input for the analysis @@ -419,289 +388,6 @@ def submit_trait_form(): version=GN_VERSION) -@app.route("/trait/<name>/edit/inbredset-id/<inbredset_id>") -@edit_access_required -def edit_phenotype(name, inbredset_id): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - publish_xref = fetchone( - conn=conn, - table="PublishXRef", - where=PublishXRef(id_=name, - inbred_set_id=inbredset_id)) - phenotype_ = fetchone( - conn=conn, - table="Phenotype", - where=Phenotype(id_=publish_xref.phenotype_id)) - publication_ = fetchone( - conn=conn, - table="Publication", - where=Publication(id_=publish_xref.publication_id)) - json_data = fetchall( - conn, - "metadata_audit", - where=MetadataAudit(dataset_id=publish_xref.id_)) - - Edit = namedtuple("Edit", ["field", "old", "new", "diff"]) - Diff = namedtuple("Diff", ["author", "diff", "timestamp"]) - diff_data = [] - for data in json_data: - json_ = json.loads(data.json_data) - timestamp = json_.get("timestamp") - author = json_.get("author") - for key, value in json_.items(): - if isinstance(value, dict): - for field, data_ in value.items(): - diff_data.append( - Diff(author=author, - diff=Edit(field, - data_.get("old"), - data_.get("new"), - "\n".join(difflib.ndiff( - [data_.get("old")], - [data_.get("new")]))), - timestamp=timestamp)) - diff_data_ = None - if len(diff_data) > 0: - diff_data_ = groupby(diff_data, lambda x: x.timestamp) - return render_template( - "edit_phenotype.html", - diff=diff_data_, - publish_xref=publish_xref, - phenotype=phenotype_, - publication=publication_, - version=GN_VERSION, - ) - - -@app.route("/trait/edit/probeset-name/<dataset_name>") -@edit_access_required -def edit_probeset(dataset_name): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - probeset_ = fetchone(conn=conn, - table="ProbeSet", - columns=list(probeset_mapping.values()), - where=Probeset(name=dataset_name)) - json_data = fetchall( - conn, - "metadata_audit", - where=MetadataAudit(dataset_id=probeset_.id_)) - Edit = namedtuple("Edit", ["field", "old", "new", "diff"]) - Diff = namedtuple("Diff", ["author", "diff", "timestamp"]) - diff_data = [] - for data in json_data: - json_ = json.loads(data.json_data) - timestamp = json_.get("timestamp") - author = json_.get("author") - for key, value in json_.items(): - if isinstance(value, dict): - for field, data_ in value.items(): - diff_data.append( - Diff(author=author, - diff=Edit(field, - data_.get("old"), - data_.get("new"), - "\n".join(difflib.ndiff( - [data_.get("old")], - [data_.get("new")]))), - timestamp=timestamp)) - diff_data_ = None - if len(diff_data) > 0: - diff_data_ = groupby(diff_data, lambda x: x.timestamp) - return render_template( - "edit_probeset.html", - diff=diff_data_, - probeset=probeset_) - - -@app.route("/trait/update", methods=["POST"]) -@edit_access_required -def update_phenotype(): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - data_ = request.form.to_dict() - TMPDIR = current_app.config.get("TMPDIR") - author = g.user_session.record.get(b'user_name') - if 'file' not in request.files: - flash("No sample-data has been uploaded", "warning") - else: - file_ = request.files['file'] - trait_name = str(data_.get('dataset-name')) - phenotype_id = str(data_.get('phenotype-id', 35)) - SAMPLE_DATADIR = os.path.join(TMPDIR, "sample-data") - if not os.path.exists(SAMPLE_DATADIR): - os.makedirs(SAMPLE_DATADIR) - if not os.path.exists(os.path.join(SAMPLE_DATADIR, - "diffs")): - os.makedirs(os.path.join(SAMPLE_DATADIR, - "diffs")) - if not os.path.exists(os.path.join(SAMPLE_DATADIR, - "updated")): - os.makedirs(os.path.join(SAMPLE_DATADIR, - "updated")) - current_time = str(datetime.datetime.now().isoformat()) - new_file_name = (os.path.join(TMPDIR, - "sample-data/updated/", - (f"{author.decode('utf-8')}." - f"{trait_name}.{phenotype_id}." - f"{current_time}.csv"))) - uploaded_file_name = (os.path.join( - TMPDIR, - "sample-data/updated/", - (f"updated.{author.decode('utf-8')}." - f"{trait_name}.{phenotype_id}." - f"{current_time}.csv"))) - file_.save(new_file_name) - publishdata_id = "" - lines = [] - with open(new_file_name, "r") as f: - lines = f.read() - first_line = lines.split('\n', 1)[0] - publishdata_id = first_line.split("Id:")[-1].strip() - with open(new_file_name, "w") as f: - f.write(lines.split("\n\n")[-1]) - csv_ = get_trait_csv_sample_data(conn=conn, - trait_name=str(trait_name), - phenotype_id=str(phenotype_id)) - with open(uploaded_file_name, "w") as f_: - f_.write(csv_.split("\n\n")[-1]) - r = run_cmd(cmd=("csvdiff " - f"'{uploaded_file_name}' '{new_file_name}' " - "--format json")) - diff_output = (f"{TMPDIR}/sample-data/diffs/" - f"{trait_name}.{author.decode('utf-8')}." - f"{phenotype_id}.{current_time}.json") - with open(diff_output, "w") as f: - dict_ = json.loads(r.get("output")) - dict_.update({ - "author": author.decode('utf-8'), - "publishdata_id": publishdata_id, - "dataset_id": data_.get("dataset-name"), - "timestamp": datetime.datetime.now().strftime( - "%Y-%m-%d %H:%M:%S") - }) - f.write(json.dumps(dict_)) - flash("Sample-data has been successfully uploaded", "success") - # Run updates: - phenotype_ = { - "pre_pub_description": data_.get("pre-pub-desc"), - "post_pub_description": data_.get("post-pub-desc"), - "original_description": data_.get("orig-desc"), - "units": data_.get("units"), - "pre_pub_abbreviation": data_.get("pre-pub-abbrev"), - "post_pub_abbreviation": data_.get("post-pub-abbrev"), - "lab_code": data_.get("labcode"), - "submitter": data_.get("submitter"), - "owner": data_.get("owner"), - "authorized_users": data_.get("authorized-users"), - } - updated_phenotypes = update( - conn, "Phenotype", - data=Phenotype(**phenotype_), - where=Phenotype(id_=data_.get("phenotype-id"))) - diff_data = {} - if updated_phenotypes: - diff_data.update({"Phenotype": diff_from_dict(old={ - k: data_.get(f"old_{k}") for k, v in phenotype_.items() - if v is not None}, new=phenotype_)}) - publication_ = { - "abstract": data_.get("abstract"), - "authors": data_.get("authors"), - "title": data_.get("title"), - "journal": data_.get("journal"), - "volume": data_.get("volume"), - "pages": data_.get("pages"), - "month": data_.get("month"), - "year": data_.get("year") - } - updated_publications = update( - conn, "Publication", - data=Publication(**publication_), - where=Publication(id_=data_.get("pubmed-id", - data_.get("old_id_")))) - if updated_publications: - diff_data.update({"Publication": diff_from_dict(old={ - k: data_.get(f"old_{k}") for k, v in publication_.items() - if v is not None}, new=publication_)}) - if diff_data: - diff_data.update({"dataset_id": data_.get("dataset-name")}) - diff_data.update({"author": author.decode('utf-8')}) - diff_data.update({"timestamp": datetime.datetime.now().strftime( - "%Y-%m-%d %H:%M:%S")}) - insert(conn, - table="metadata_audit", - data=MetadataAudit(dataset_id=data_.get("dataset-name"), - editor=author.decode("utf-8"), - json_data=json.dumps(diff_data))) - flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success") - return redirect(f"/trait/{data_.get('dataset-name')}" - f"/edit/inbredset-id/{data_.get('inbred-set-id')}") - - -@app.route("/probeset/update", methods=["POST"]) -@edit_access_required -def update_probeset(): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - data_ = request.form.to_dict() - probeset_ = { - "id_": data_.get("id"), - "symbol": data_.get("symbol"), - "description": data_.get("description"), - "probe_target_description": data_.get("probe_target_description"), - "chr_": data_.get("chr"), - "mb": data_.get("mb"), - "alias": data_.get("alias"), - "geneid": data_.get("geneid"), - "homologeneid": data_.get("homologeneid"), - "unigeneid": data_.get("unigeneid"), - "omim": data_.get("OMIM"), - "refseq_transcriptid": data_.get("refseq_transcriptid"), - "blatseq": data_.get("blatseq"), - "targetseq": data_.get("targetseq"), - "strand_probe": data_.get("Strand_Probe"), - "probe_set_target_region": data_.get("probe_set_target_region"), - "probe_set_specificity": data_.get("probe_set_specificity"), - "probe_set_blat_score": data_.get("probe_set_blat_score"), - "probe_set_blat_mb_start": data_.get("probe_set_blat_mb_start"), - "probe_set_blat_mb_end": data_.get("probe_set_blat_mb_end"), - "probe_set_strand": data_.get("probe_set_strand"), - "probe_set_note_by_rw": data_.get("probe_set_note_by_rw"), - "flag": data_.get("flag") - } - updated_probeset = update( - conn, "ProbeSet", - data=Probeset(**probeset_), - where=Probeset(id_=data_.get("id"))) - - diff_data = {} - author = g.user_session.record.get(b'user_name') - if updated_probeset: - diff_data.update({"Probeset": diff_from_dict(old={ - k: data_.get(f"old_{k}") for k, v in probeset_.items() - if v is not None}, new=probeset_)}) - if diff_data: - diff_data.update({"probeset_name": data_.get("probeset_name")}) - diff_data.update({"author": author.decode('utf-8')}) - diff_data.update({"timestamp": datetime.datetime.now().strftime( - "%Y-%m-%d %H:%M:%S")}) - insert(conn, - table="metadata_audit", - data=MetadataAudit(dataset_id=data_.get("id"), - editor=author.decode("utf-8"), - json_data=json.dumps(diff_data))) - return redirect(f"/trait/edit/probeset-name/{data_.get('probeset_name')}") - - @app.route("/create_temp_trait", methods=('POST',)) def create_temp_trait(): logger.info(request.url) @@ -712,12 +398,9 @@ def create_temp_trait(): @app.route('/export_trait_excel', methods=('POST',)) def export_trait_excel(): """Excel file consisting of the sample data from the trait data and analysis page""" - logger.info("In export_trait_excel") - logger.info("request.form:", request.form) - logger.info(request.url) trait_name, sample_data = export_trait_data.export_sample_table( request.form) - + app.logger.info(request.url) logger.info("sample_data - type: %s -- size: %s" % (type(sample_data), len(sample_data))) @@ -836,8 +519,10 @@ def export_perm_data(): @app.route("/show_temp_trait", methods=('POST',)) def show_temp_trait_page(): - logger.info(request.url) - template_vars = show_trait.ShowTrait(request.form) + user_id = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") + template_vars = show_trait.ShowTrait(user_id=user_id, + kw=request.form) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, indent=" ") @@ -846,8 +531,10 @@ def show_temp_trait_page(): @app.route("/show_trait") def show_trait_page(): - logger.info(request.url) - template_vars = show_trait.ShowTrait(request.args) + user_id = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") + template_vars = show_trait.ShowTrait(user_id=user_id, + kw=request.args) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, indent=" ") @@ -1356,22 +1043,6 @@ def json_default_handler(obj): type(obj), repr(obj))) -@app.route("/trait/<trait_name>/sampledata/<phenotype_id>") -def get_sample_data_as_csv(trait_name: int, phenotype_id: int): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - csv_ = get_trait_csv_sample_data(conn, str(trait_name), - str(phenotype_id)) - return Response( - csv_, - mimetype="text/csv", - headers={"Content-disposition": - "attachment; filename=myplot.csv"} - ) - - @app.route("/admin/data-sample/diffs/") @edit_access_required def display_diffs_admin(): @@ -1401,65 +1072,3 @@ def display_diffs_users(): files=files) -@app.route("/data-samples/approve/<name>") -def approve_data(name): - sample_data = {} - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - TMPDIR = current_app.config.get("TMPDIR") - with open(os.path.join(f"{TMPDIR}/sample-data/diffs", - name), 'r') as myfile: - sample_data = json.load(myfile) - PUBLISH_ID = sample_data.get("publishdata_id") - modifications = [d for d in sample_data.get("Modifications")] - row_counts = len(modifications) - for modification in modifications: - if modification.get("Current"): - (strain_id, - strain_name, - value, se, count) = modification.get("Current").split(",") - update_sample_data( - conn=conn, - strain_name=strain_name, - strain_id=int(strain_id), - publish_data_id=int(PUBLISH_ID), - value=value, - error=se, - count=count - ) - insert(conn, - table="metadata_audit", - data=MetadataAudit( - dataset_id=name.split(".")[0], # use the dataset name - editor=sample_data.get("author"), - json_data=json.dumps(sample_data))) - if modifications: - # Once data is approved, rename it! - os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", name), - os.path.join(f"{TMPDIR}/sample-data/diffs", - f"{name}.approved")) - flash((f"Just updated data from: {name}; {row_counts} " - "row(s) modified!"), - "success") - return redirect("/admin/data-sample/diffs/") - - -@app.route("/data-samples/reject/<name>") -def reject_data(name): - TMPDIR = current_app.config.get("TMPDIR") - os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", name), - os.path.join(f"{TMPDIR}/sample-data/diffs", - f"{name}.rejected")) - flash(f"{name} has been rejected!", "success") - return redirect("/admin/data-sample/diffs/") - - -@app.route("/display-file/<name>") -def display_file(name): - TMPDIR = current_app.config.get("TMPDIR") - with open(os.path.join(f"{TMPDIR}/sample-data/diffs", - name), 'r') as myfile: - content = myfile.read() - return Response(content, mimetype='text/json') diff --git a/wqflask/wqflask/wgcna/gn3_wgcna.py b/wqflask/wqflask/wgcna/gn3_wgcna.py new file mode 100644 index 00000000..7bf5c62b --- /dev/null +++ b/wqflask/wqflask/wgcna/gn3_wgcna.py @@ -0,0 +1,111 @@ +"""module contains code to consume gn3-wgcna api +and process data to be rendered by datatables +""" + +import requests +from types import SimpleNamespace + +from utility.helper_functions import get_trait_db_obs +from utility.tools import GN3_LOCAL_URL + + +def fetch_trait_data(requestform): + """fetch trait data""" + db_obj = SimpleNamespace() + get_trait_db_obs(db_obj, + [trait.strip() + for trait in requestform['trait_list'].split(',')]) + + return process_dataset(db_obj.trait_list) + + +def process_dataset(trait_list): + """process datasets and strains""" + + input_data = {} + traits = [] + strains = [] + + for trait in trait_list: + traits.append(trait[0].name) + + input_data[trait[0].name] = {} + for strain in trait[0].data: + strains.append(strain) + input_data[trait[0].name][strain] = trait[0].data[strain].value + + return { + "input": input_data, + "trait_names": traits, + "sample_names": strains + } + + +def process_wgcna_data(response): + """function for processing modeigene genes + for create row data for datataba""" + mod_eigens = response["output"]["ModEigens"] + + sample_names = response["input"]["sample_names"] + + mod_dataset = [[sample] for sample in sample_names] + + for _, mod_values in mod_eigens.items(): + for (index, _sample) in enumerate(sample_names): + mod_dataset[index].append(round(mod_values[index], 3)) + + return { + "col_names": ["sample_names", *mod_eigens.keys()], + "mod_dataset": mod_dataset + } + + +def process_image(response): + """function to process image check if byte string is empty""" + image_data = response["output"]["image_data"] + return ({ + "image_generated": True, + "image_data": image_data + } if image_data else { + "image_generated": False + }) + + +def run_wgcna(form_data): + """function to run wgcna""" + + wgcna_api = f"{GN3_LOCAL_URL}/api/wgcna/run_wgcna" + + # parse form data + + trait_dataset = fetch_trait_data(form_data) + form_data["minModuleSize"] = int(form_data["MinModuleSize"]) + + form_data["SoftThresholds"] = [int(threshold.strip()) + for threshold in form_data['SoftThresholds'].rstrip().split(",")] + + try: + + response = requests.post(wgcna_api, json={ + "sample_names": list(set(trait_dataset["sample_names"])), + "trait_names": trait_dataset["trait_names"], + "trait_sample_data": list(trait_dataset["input"].values()), + **form_data + + } + ) + + status_code = response.status_code + response = response.json() + + return {"error": response} if status_code != 200 else { + "error": 'null', + "results": response, + "data": process_wgcna_data(response["data"]), + "image": process_image(response["data"]) + } + + except requests.exceptions.ConnectionError: + return { + "error": "A connection error to perform computation occurred" + } |
