diff options
53 files changed, 2332 insertions, 1816 deletions
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index af60c290..00000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: '[Bug] Bug Title' -labels: '' -assignees: '' - ---- - -**Describe the bug** -<!-- A clear and concise description of what the bug is. --> - -**To Reproduce** -<!-- Steps to reproduce the behavior --> - -**Expected behavior** -<!-- A clear and concise description of what you expected to happen. --> - -**Screenshots** -<!-- If applicable, add screenshots to help explain your problem. --> - -**Environment setup (please complete the following information):** -<!-- - OS: [e.g. Linux] --> -<!-- - Guix Version (optional) --> -<!-- - [Anything else you think is relevant] --> - -**Additional context** -<!-- Add any other context about the problem here. --> diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 813974c1..00000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -name: Feature request -about: Suggest a new feature for this project(Very Specific) -title: '[Feature] My-cool-feature' -labels: '' -assignees: '' - ---- - -## Is your feature request related to a problem? Please describe. -<!-- A clear and concise description of what the problem is. --> -<!-- Example: I'm always frustrated when [...] --> - -## Describe the solution you'd like -<!-- A clear and concise description of what you want to happen. --> - -## Describe alternatives you've considered -<!-- A clear and concise description of any alternative solutions or features you've considered. --> - -## User Stories (optional) -<!-- Example: --> -<!-- As a _[role or persona]_, I want _[goal/ need]_ so that _[why]_ --> -<!-- **Feature:** _[Brief description of feature]_ --> -<!-- _[Any additional descriptions on feature]_ --> -<!-- **Scenario:** -Please use _[Gherkin](https://cucumber.io/docs/gherkin/reference/)_ -here --> - -## Additional context -<!-- Add any other context or screenshots about the feature request here. --> diff --git a/.github/ISSUE_TEMPLATE/user_story.md b/.github/ISSUE_TEMPLATE/user_story.md deleted file mode 100644 index d46976ba..00000000 --- a/.github/ISSUE_TEMPLATE/user_story.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -name: User Story -about: Suggest an idea for this project -title: ['Feature] My-cool-new-feature' -labels: '' -assignees: '' - ---- -<!-- As a _[role or persona]_, I want _[goal/ need]_ so that _[why]_ --> -<!-- **Feature:** _[Brief description of feature]_ --> -<!-- _[Any additional descriptions on feature]_ --> -<!-- **Scenario:** Please use _[Gherkin](https://cucumber.io/docs/gherkin/reference/)_ here --> diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 0cf4557f..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: tests - -# Run actions when pushing to the testing branch or when you create a -# PR against it -on: - push: - branches: [ testing ] - pull_request: - branches: [ testing ] - -jobs: - unittest: - runs-on: ubuntu-latest - container: bonfacekilz/genenetwork2:latest - - steps: - # First start with mariadb set then checkout. The checkout gives - # the mysqld enough time to start - - name: Set up mariadb - run: | - mysql_install_db --user=mysql --datadir=/usr/local/mysql - # Wait for the mysqld_safe process to start - mysqld_safe --user=mysql --datadir=/usr/local/mysql & - - # Use v1 of checkout since v2 fails - - name: Checkout Project - uses: actions/checkout@v1 - - # Redis is required by some of the tests 6379 - - name: Start Redis - run: | - /gn2-profile/bin/screen -dmLS redisconn /gn2-profile/bin/redis-server - - # Initialise the tables - - name: Bootstrap tables - run: | - mysql -u root -e "SHOW DATABASES;" - mysql -u root -e "CREATE DATABASE db_webqtl_s;" - mysql -u root -e "CREATE USER 'gn2'@'localhost' IDENTIFIED BY 'mysql_password';" - mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO 'gn2'@'localhost';FLUSH PRIVILEGES;" - - - name: Start Genenetwork as a Background Task - run: | - /gn2-profile/bin/screen -dm bash -c "env GN2_PROFILE=/gn2-profile \ - TMPDIR=/tmp SERVER_PORT=5004 \ - WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \ - GN_PROXY_URL='http://localhost:8080' \ - GN3_LOCAL_URL='http://localhost:8081' \ - GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \ - etc/default_settings.py" - - - name: Run the unit tests - run: | - env GN2_PROFILE=/gn2-profile \ - TMPDIR=/tmp SERVER_PORT=5004 \ - WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \ - GN_PROXY_URL='http://localhost:8080' \ - GN3_LOCAL_URL='http://localhost:8081' \ - GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \ - etc/default_settings.py -c -m unittest discover -v - - # - name: Test for Broken Links - # run: | - # env GN2_PROFILE=/gn2-profile \ - # TMPDIR=/tmp\ - # WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \ - # GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \ - # etc/default_settings.py -c \ - # $PWD/test/requests/links_scraper/genelinks.py diff --git a/README.md b/README.md index 50fe49f6..d2df1673 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,38 @@ deploy GN2 and dependencies as a self contained unit on any machine. The database can be run separately as well as the source tree (for developers). See the [installation docs](doc/README.org). +## Configuration + +GeneNetwork2 comes with a [default configuration file](./etc/default_settings.py) +which can be used as a starting point. + +The recommended way to deal with the configurations is to **copy** this default configuration file to a location outside of the repository, say, + +```sh +.../genenetwork2$ cp etc/default_settings.py "${HOME}/configurations/gn2.py" +``` + +then change the appropriate values in the new file. You can then pass in the new +file as the configuration file when launching the application, + +```sh +.../genenetwork2$ bin/genenetwork "${HOME}/configurations/gn2.py" <command-to-run> +``` + +The other option is to override the configurations in `etc/default_settings.py` +by setting the configuration you want to override as an environment variable e.g. +to override the `SQL_URI` value, you could do something like: + +```sh +.../genenetwork2$ env SQL_URI="mysql://<user>:<passwd>@<host>:<port>/<db_name>" \ + bin/genenetwork "${HOME}/configurations/gn2.py" <command-to-run> +``` + +replacing the placeholders in the angle brackets with appropriate values. + +For a detailed breakdown of the configuration variables and their use, see the +[configuration documentation](doc/configurations.org) + ## Run Once having installed GN2 it can be run through a browser @@ -59,6 +91,20 @@ asserts sprinkled in the code base. Right now, the only tests running in CI are unittests. Please make sure the existing unittests are green when submitting a PR. +From the root directory of the repository, you can run the tests with something +like: + +```sh +env GN_PROFILE=~/opt/gn-latest SERVER_PORT=5300 \ + SQL_URI=<uri-to-override-the-default> \ + ./bin/genenetwork2 ./etc/default_settings.py \ + -m unittest -v +``` + +In the case where you use the default `etc/default_settings.py` configuration file, you can override any setting as demonstrated with the `SQL_URI` setting in the command above. + +In order to avoid having to set up a whole host of settings every time with the `env` command, you could copy the `etc/default_settings.py` file to a new location (outside the repository is best), and pass that to `bin/genenetwork2` instead. + See [./bin/genenetwork2](https://github.com/genenetwork/genenetwork2/blob/testing/doc/docker-container.org) for more details. diff --git a/doc/configurations.org b/doc/configurations.org new file mode 100644 index 00000000..54c97d52 --- /dev/null +++ b/doc/configurations.org @@ -0,0 +1,92 @@ +#+TITLE: Configurations for GeneNetwork2 Service +#+OPTIONS: toc:3 + +* Configuration Variables + +** Basic Configurations + +- *GN_VERSION*: The current version of GN2 +- *GN2_PROXY*: URL to the GN2 proxy service +- *GN_SERVER_URL*: URL to the GN3 REST API server +- *GN2_BASE_URL*: ?? +- *GN2_BRANCH_URL*: +- *SERVER_PORT*: The port on which to run GN2. This is passed on to Flask. +- *SECRET_HMAC_CODE*: +- *GENENETWORK_FILES*: Base directory for all static data files +- *HOME*: The home directory of the user that GN2 runs as + +** Redis Configurations + +- *REDIS_URL*: The url to use to connect to the running redis instance +- *USE_REDIS*: Redis caching + + +** Database (MySQL) Configurations + +- *SQL_URI*: The URI to the database server in the form + ~mysql://<username>:<password>@<host>:<port>/<dbname>~. Replace the items in + the angle brackets with the appropriate values for each item. +- *SQL_ALCHEMY_POOL_RECYCLE*: Probably obsoleted - the use of ORMs (ha/i)s + be(en/ing) phased off + +# ---- Flask configuration (see website) +** Flask-Specific Configurations +- *TRAP_BAD_REQUEST_ERRORS*: +- *SECURITY_CONFIRMABLE*: +- *SECURITY_TRACKABLE*: +- *SECURITY_REGISTERABLE*: +- *SECURITY_RECOVERABLE*: +- *SECURITY_EMAIL_SENDER*: +- *SECURITY_POST_LOGIN_VIEW*: + +** External Services + +*** GitHub Configurations + +- *GITHUB_CLIENT_ID*: Client identifier key for GitHub OAuth authentication +- *GITHUB_CLIENT_SECRET*: Client authentication token for github +- *GITHUB_AUTH_URL*: The url to use for authenticating the client (GN2) with + GitHub. Default: https://github.com/login/oauth/authorize +- *GITHUB_API_URL*: Once the client (GN2) is authenticated with GitHub, this URI + is the used to authenticate users using GitHub. The default value is + https://api.github.com/user + +*** ORCID Configurations + +- *ORCID_CLIENT_ID*: Client identifier key for ORCID OAuth authentication +- *ORCID_CLIENT_SECRET*: Client authentication token for ORCID +- *ORCID_AUTH_URL*: The url to use for authenticating the client (GN2) with + ORCID. Default: https://orcid.org/oauth/authorize +- *ORCID_TOKEN_URL*: The URI to acquire a token once GN2 has been authenticated + with ORCID. This token is used to authenticate users with the ORCID service. + The default value is https://orcid.org/oauth/token + +** Mail Configurations +- *SMTP_CONNECT*: +- *SMTP_USERNAME*: +- *SMTP_PASSWORD*: + +** Javascript Configurations +- *JS_GN_PATH*: Path to local javascript libraries for development purposes only. + See [[./development.org]] for more details + +** External Commands Configurations +- *REAPER_COMMAND*: The path to the reaper command/executable + +** Behavioral Settings +# ---- Behavioural settings (defaults) note that logger and log levels can +# be overridden at the module level and with enviroment settings +- *WEBSERVER_MODE*: Determines how the service is run. + If the value is ~DEBUG~, the service is run in debug mode - debug tools are + activated for the application. + If the value is ~DEV~, the service is run in development mode; logging is + activated. +- *WEBSERVER_BRANDING*: Probably unused - verify and remove. +- *WEBSERVER_DEPLOY*: Probably unused - verify and remove. +- *WEBSERVER_URL*: Probably unused - verify and remove. +- *LOG_LEVEL*: +- *LOG_LEVEL_DEBUG*: +- *LOG_SQL*: +- *LOG_SQL_ALCHEMY*: +- *LOG_BENCH*: +- *USE_GN_SERVER*: ?? diff --git a/doc/database.org b/doc/database.org index 99d2905a..32f1f8e0 100644 --- a/doc/database.org +++ b/doc/database.org @@ -1386,7 +1386,8 @@ JOIN Strain st ON pd.StrainId = st.Id LEFT JOIN PublishSE ps ON ps.DataId = pd.Id AND ps.StrainId = pd.StrainId LEFT JOIN NStrain ns ON ns.DataId = pd.Id AND ns.StrainId = pd.StrainId LEFT JOIN CaseAttributeXRefNew cxref ON - cxref.InbredSetId = px.InbredSetId + (cxref.InbredSetId = px.InbredSetId AND + cxref.StrainId = st.Id) LEFT JOIN CaseAttribute ca ON ca.Id = cxref.CaseAttributeId WHERE px.Id = 10006 AND px.PhenotypeId = 28409 LIMIT 10; #+end_src diff --git a/scripts/insert_expression_data.py b/scripts/insert_expression_data.py new file mode 100644 index 00000000..3d93c9f4 --- /dev/null +++ b/scripts/insert_expression_data.py @@ -0,0 +1,203 @@ +# !/usr/bin/python3 +"""This script use the nearest marker to the transcript as control, increasing permutation rounds according to the p-value""" + +######################################################################## +# Last Updated 3/11/2022 by Zach +######################################################################## +import csv +import string +import sys +import MySQLdb +import getpass +import time + +######################################################################## + +def translate_alias(str): + if str == "B6": + return "C57BL/6J" + elif str == "D2": + return "DBA/2J" + else: + return str + + +######################################################################## +# +# Indicate Data Start Position, ProbeFreezeId, gene_chip_id, DataFile +# +######################################################################## + +data_start = 1 + +gene_chip_id = int(input("Enter GeneChipId:")) +probeset_freeze_id = int(input("Enter ProbeSetFreezeId:")) +input_file_name = input("Enter file name with suffix:") + +try: + passwd = getpass.getpass('Please enter mysql password here : ') + conn = MySQLdb.Connect(db='db_webqtl', host='localhost', user='webqtlout', passwd=passwd) + + db = conn.cursor() + + print + "You have successfully connected to mysql.\n" +except: + print + "You entered incorrect password.\n" + sys.exit(0) + +time0 = time.time() + +######################################################################### +# +# Check if each line have same number of members +# generate the gene list of expression data here +# +######################################################################### +print +'Checking if each line have same number of members' + +gene_list = [] +strain_list = [] +trait_data = [] + +with open(input_file_name, "r") as csvfile: + reader = csv.DictReader(csvfile, delimiter="\t") + + kj = 0 + for line in reader: + trait_data.append(line) + + # Get the strain list; only need to get it once + if kj == 0: + strain_list = [item for item in line.keys() if item != "ProbeSetID"] + print("STRAIN LIST:", strain_list) + + gene_list.append(line['ProbeSetID']) + + if kj % 100000 == 0: + print(f"checked {kj} lines") + kj += 1 + +gene_list.sort() + +print(f"used {time.time() - time0} seconds") +######################################################################### +# +# Check if each strain exist in database +# generate the string id list of expression data here +# +######################################################################### +print('Checking if each strain exists in database') + +strain_list = map(translate_alias, strain_list) + +strain_ids = {} +for item in strain_list: + try: + db.execute(f'select Id from Strain where Name = "{item}" AND SpeciesId=1') + strain_ids[item] = db.fetchone()[0] + except: + print(f"{item} does not exist, check the if the strain name is correct") + sys.exit(0) + +print(f"Used {time.time() - time0} seconds") + +######################################################################## +# +# Check if each ProbeSet exist in database +# +######################################################################## +print("Check if each ProbeSet exists in database") + + +# Check whether ProbeSetIDs are Name or TargetId (if not Name, assume to be TargetId) +id_type = "TargetId" +db.execute(f"select Id from ProbeSet where Name='{gene_list[0]}' and ChipId={gene_chip_id}") +if len(db.fetchall()): + id_type = "Name" + +## Get Name/TargetId + ID list from database +db.execute(f"select {id_type}, Id from ProbeSet where ChipId={gene_chip_id} order by {id_type}") +records_from_db = db.fetchall() + +record_names = [item[0] for item in records_from_db] +record_names.sort() + +# Compare gene_list with gene_names +invalid_records = [] +lowercase_records = [name2.lower() for name2 in record_names] +for name in gene_list: + if name.lower() not in lowercase_records: + invalid_records.append(name) + +if len(invalid_records): + with open("ProbeSetError.txt", "wb") as error_fh: + for item in invalid_records: + error_fh.write(f"{item} doesn't exist, cheeck if the ProbeSet name is correct \n") + sys.exit(0) + +print(f"used {time.time() - time0} seconds") +######################################################################### +# +# Insert data into database +# +######################################################################### +print("getting ProbeSet Name + Id") +record_ids = {} +for record in records_from_db: + record_ids[record[0]] = record[1] + +print(f"used {time.time() - time0} seconds") + +print("inserting data") + +# Get old max dataId +db.execute('select max(Id) from ProbeSetData') +latest_data_id = int(db.fetchone()[0]) +print(f"Latest DataId = {latest_data_id}") + +# Insert data +probeset_data_values = [] +probeset_xref_values = [] +for i, item in enumerate(trait_data): + latest_data_id += 1 + + + probeset_id = item['ProbeSetID'] + item.pop('ProbeSetID') + sample_data = item + for strain in sample_data: + probeset_data_values.append(f"({latest_data_id},{strain_ids[strain]},{float(sample_data[strain])})") + + probeset_xref_values.append(f"({probeset_freeze_id},{record_ids[probeset_id]},{latest_data_id})") + + # Insert into tables for every 100 traits + if i % 100 == 0: + data_query = f"INSERT INTO ProbeSetData VALUES {','.join(probeset_data_values)}" + db.execute(data_query) + + xref_query = ( + "INSERT INTO ProbeSetXRef(ProbeSetFreezeId, ProbeSetId, DataId) " + f"VALUES {','.join(probeset_xref_values)}") + db.execute(xref_query) + + probeset_data_values = [] + probeset_xref_values = [] + + print(f"Inserted {i} lines") + print(f"Used {time.time() - time0} seconds") + +# Insert the remainder (since the loop above only inserts every 100 traits) +if len(probeset_data_values): + data_query = f"INSERT INTO ProbeSetData VALUES {','.join(probeset_data_values)}" + db.execute(data_query) + + xref_query = ( + "INSERT INTO ProbeSetXRef(ProbeSetFreezeId, ProbeSetId, DataId) " + f"VALUES {','.join(probeset_xref_values)}") + db.execute(xref_query) + +conn.commit() +conn.close() diff --git a/scripts/maintenance/QTL_Reaper_v6.py b/scripts/maintenance/QTL_Reaper_v6.py index 35f2d1a1..20fd8e3b 100755 --- a/scripts/maintenance/QTL_Reaper_v6.py +++ b/scripts/maintenance/QTL_Reaper_v6.py @@ -106,3 +106,6 @@ for ProbeSetFreezeId in ProbeSetFreezeIds: print(ProbeSetFreezeIds) + +cursor.close() +con.close() diff --git a/scripts/maintenance/Update_Case_Attributes_MySQL_tab.py b/scripts/maintenance/Update_Case_Attributes_MySQL_tab.py index bf796df4..a3cd1c35 100644 --- a/scripts/maintenance/Update_Case_Attributes_MySQL_tab.py +++ b/scripts/maintenance/Update_Case_Attributes_MySQL_tab.py @@ -10,18 +10,13 @@ import time import csv ######################################################################## -mydb = MySQLdb.connect(host='localhost', - user='username', - passwd='', - db='db_webqtl') -cursor = mydb.cursor() +with MySQLdb.connect( + host='localhost', user='username', passwd='', db='db_webqtl') as mydb: + with mydb.cursor() as cursor: -csv_data = csv.reader(file('GN711_pvalues.txt'), delimiter ="\t") -for row in csv_data: - - cursor.execute("""UPDATE ProbeSetXRef SET pValue = %s WHERE ProbeSetFreezeId = %s AND ProbeSetId = %s """, - (row)) -#close the connection to the database. -mydb.commit() -cursor.close() -print("Done") \ No newline at end of file + csv_data = csv.reader(file('GN711_pvalues.txt'), delimiter ="\t") + for row in csv_data: + cursor.execute( + """UPDATE ProbeSetXRef SET pValue = %s WHERE ProbeSetFreezeId = %s AND ProbeSetId = %s """, + (row)) +print("Done") diff --git a/scripts/maintenance/readProbeSetMean_v7.py b/scripts/maintenance/readProbeSetMean_v7.py index 43f084f4..56adcdfd 100755 --- a/scripts/maintenance/readProbeSetMean_v7.py +++ b/scripts/maintenance/readProbeSetMean_v7.py @@ -269,4 +269,5 @@ if len(values1) > 0: cmd = 'insert into ProbeSetXRef(ProbeSetFreezeId, ProbeSetId, DataId) values %s' % cmd db.execute(cmd) +db.close() con.close() diff --git a/scripts/maintenance/readProbeSetSE_v7.py b/scripts/maintenance/readProbeSetSE_v7.py index 2cfe2e07..88a347bf 100755 --- a/scripts/maintenance/readProbeSetSE_v7.py +++ b/scripts/maintenance/readProbeSetSE_v7.py @@ -251,4 +251,5 @@ if len(DataValues) > 0: cmd = 'insert ProbeSetSE values %s' % DataValues db.execute(cmd) +db.close() con.close() diff --git a/scripts/maintenance/utilities.py b/scripts/maintenance/utilities.py index 886410c2..1fe14809 100644 --- a/scripts/maintenance/utilities.py +++ b/scripts/maintenance/utilities.py @@ -1,16 +1,6 @@ -import MySQLdb import re import configparser -def get_cursor(): - host = 'tux.uthsc.edu' - user = 'webqtlout' - passwd = 'webqtlout' - db = 'db_webqtl' - con = MySQLdb.Connect(db=db, host=host, user=user, passwd=passwd) - cursor = con.cursor() - return cursor, con - def clearspaces(s, default=None): if s: s = re.sub('\s+', ' ', s) diff --git a/test/requests/test-website.py b/test/requests/test-website.py index d619a7d5..71055fca 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -14,16 +14,11 @@ import link_checker import sys # Imports for integration tests -from wqflask import app -from test_login_local import TestLoginLocal -from test_login_orcid import TestLoginOrcid -from test_login_github import TestLoginGithub -from test_registration import TestRegistration -from test_forgot_password import TestForgotPassword from unittest import TestSuite, TextTestRunner, TestLoader print("Mechanical Rob firing up...") + def run_all(args_obj, parser): print("") print("Running all tests.") @@ -35,38 +30,20 @@ def run_all(args_obj, parser): check_mapping(args_obj, parser) # TODO: Add other functions as they are created. + def print_help(args_obj, parser): print(parser.format_help()) + def dummy(args_obj, parser): print("Not implemented yet.") -def integration_tests(args_obj, parser): - gn2_url = args_obj.host - run_integration_tests(gn2_url, es_url) def initTest(klass, gn2_url, es_url): loader = TestLoader() methodNames = loader.getTestCaseNames(klass) return [klass(mname, gn2_url, es_url) for mname in methodNames] -def integration_suite(gn2_url, es_url): - test_cases = [ - TestRegistration - , TestLoginLocal - , TestLoginGithub - , TestLoginOrcid - , TestForgotPassword - ] - the_suite = TestSuite() - for case in test_cases: - the_suite.addTests(initTest(case, gn2_url, es_url)) - return the_suite - -def run_integration_tests(gn2_url, es_url): - runner = TextTestRunner() - runner.run(integration_suite(gn2_url, es_url)) - desc = """ This is Mechanical-Rob - an automated web server tester for @@ -76,34 +53,63 @@ parser = argparse.ArgumentParser(description=desc) parser.add_argument("--fail", help="Fail and stop on any error", action="store_true") -parser.add_argument("-d", "--database", metavar="DB", type=str - , default="db_webqtl_s" - , help="Use database (default db_webqtl_s)") - -parser.add_argument("host", metavar="HOST", type=str - , default="http://localhost:5003" - , help="The url to the web server") - -parser.add_argument("-a", "--all", dest="accumulate", action="store_const" - , const=run_all, default=print_help - , help="Runs all tests.") - -parser.add_argument("-l", "--link-checker", dest="accumulate" - , action='store_const', const=check_links, default=print_help - , help="Checks for dead links.") - -parser.add_argument("-f", "--main-functionality", dest="accumulate" - , action='store_const', const=check_main_web_functionality - , default=print_help - , help="Checks for main web functionality.") - -parser.add_argument("-m", "--mapping", dest="accumulate" - , action="store_const", const=check_mapping, default=print_help - , help="Checks for mapping.") +parser.add_argument( + "-d", + "--database", + metavar="DB", + type=str, + default="db_webqtl_s", + help="Use database (default db_webqtl_s)", +) + +parser.add_argument( + "host", + metavar="HOST", + type=str, + default="http://localhost:5003", + help="The url to the web server", +) + +parser.add_argument( + "-a", + "--all", + dest="accumulate", + action="store_const", + const=run_all, + default=print_help, + help="Runs all tests.", +) + +parser.add_argument( + "-l", + "--link-checker", + dest="accumulate", + action="store_const", + const=check_links, + default=print_help, + help="Checks for dead links.", +) + +parser.add_argument( + "-f", + "--main-functionality", + dest="accumulate", + action="store_const", + const=check_main_web_functionality, + default=print_help, + help="Checks for main web functionality.", +) + +parser.add_argument( + "-m", + "--mapping", + dest="accumulate", + action="store_const", + const=check_mapping, + default=print_help, + help="Checks for mapping.", +) -parser.add_argument("-i", "--integration-tests", dest="accumulate" - , action="store_const", const=integration_tests, default=print_help - , help="Runs integration tests.") args = parser.parse_args() diff --git a/test/requests/test_registration.py b/test/requests/test_registration.py deleted file mode 100644 index 5d08bf58..00000000 --- a/test/requests/test_registration.py +++ /dev/null @@ -1,35 +0,0 @@ -import sys -import requests - -class TestRegistration(ParametrizedTest): - - - def testRegistrationPage(self): - data = { - "email_address": "test@user.com", - "full_name": "Test User", - "organization": "Test Organisation", - "password": "test_password", - "password_confirm": "test_password" - } - requests.post(self.gn2_url+"/n/register", data) - response = self.es.search( - index="users" - , doc_type="local" - , body={ - "query": {"match": {"email_address": "test@user.com"}}}) - self.assertEqual(len(response["hits"]["hits"]), 1) - - -def main(gn2, es): - import unittest - suite = unittest.TestSuite() - suite.addTest(TestRegistration(methodName="testRegistrationPage", gn2_url=gn2, es_url=es)) - runner = unittest.TextTestRunner() - runner.run(suite) - -if __name__ == "__main__": - if len(sys.argv) < 3: - raise Exception("Required arguments missing") - else: - main(sys.argv[1], sys.argv[2]) diff --git a/wqflask/maintenance/gen_ind_genofiles.py b/wqflask/maintenance/gen_ind_genofiles.py new file mode 100644 index 00000000..8b958efa --- /dev/null +++ b/wqflask/maintenance/gen_ind_genofiles.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""A script that generates the genotype files for groups of individuals, using an existing strain genotype file as a basis + +Example commands: +python3 gen_ind_genofiles.py + /home/zas1024/gn2-zach/genotype_files/genotype/ + /home/zas1024/gn2-zach/new_geno/ + BXD-Micturition.geno + BXD.json +python3 gen_ind_genofiles.py + /home/zas1024/gn2-zach/genotype_files/genotype + /home/zas1024/gn2-zach/new_geno/ + BXD-Micturition.geno + BXD.2.geno BXD.4.geno BXD.5.geno + +""" + +import json +import os +import sys +from typing import List + +import MySQLdb + +def conn(): + return MySQLdb.Connect(db=os.environ.get("DB_NAME"), + user=os.environ.get("DB_USER"), + passwd=os.environ.get("DB_PASS"), + host=os.environ.get("DB_HOST")) + +def main(args): + + # Directory in which .geno files are located + geno_dir = args[1] + + # Directory in which to output new files + out_dir = args[2] + + # The individuals group that we want to generate a .geno file for + target_file = geno_dir + args[3] + + # The source group(s) we're generating the .geno files from + # This can be passed as either a specific .geno file (or set of files as multiple arguments), + # or as a JSON file containing a set of .geno files (and their corresponding file names and sample lists) + geno_json = {} + source_files = [] + if ".json" in args[4]: + geno_json = json.load(open(geno_dir + args[4], "r")) + par_f1s = { + "mat": geno_json['mat'], + "pat": geno_json['pat'], + "f1s": geno_json['f1s'] + } + + # List of file titles and locations from JSON + source_files = [{'title': genofile['title'], 'location': geno_dir + genofile['location']} for genofile in geno_json['genofile']] + else: + par_f1s = {} + # List of files directly taken from command line arguments, with titles just set to the filename + for group in args[4:]: + file_name = geno_dir + group + ".geno" if ".geno" not in group else group + source_files.append({'title': file_name[:-5], 'location': file_name}) + + if len(source_files) > 1: + # Generate a JSON file pointing to the new target genotype files, in situations where there are multiple source .geno files + target_json_loc = out_dir + ".".join(args[3].split(".")[:-1]) + ".json" + target_json = {'genofile': []} + + # Generate the output .geno files + for source_file in source_files: + filename, samples = generate_new_genofile(source_file['location'], target_file, par_f1s, out_dir) + + target_json['genofile'].append({ + 'location': filename.split("/")[-1], + 'title': source_file['title'], + 'sample_list': samples + }) + + json.dump(target_json, open(target_json_loc, "w")) + +def get_strain_for_sample(sample): + query = ( + "SELECT CaseAttributeXRefNew.Value " + "FROM CaseAttributeXRefNew, Strain " + "WHERE CaseAttributeXRefNew.CaseAttributeId=11 " + "AND CaseAttributeXRefNew.StrainId = Strain.Id " + "AND Strain.Name = %(name)s" ) + + with conn().cursor() as cursor: + cursor.execute(query, {"name": sample.strip()}) + return cursor.fetchone()[0] + +def generate_new_genofile(source_genofile, target_genofile, par_f1s, out_dir): + source_samples = group_samples(source_genofile) + source_genotypes = strain_genotypes(source_genofile) + target_samples = group_samples(target_genofile) + strain_pos_map = map_strain_pos_to_target_group(source_samples, target_samples, par_f1s) + + if len(source_genofile.split("/")[-1].split(".")) > 2: + # The number in the source genofile; for example 4 in BXD.4.geno + source_num = source_genofile.split("/")[-1].split(".")[-2] + target_filename = ".".join(target_genofile.split("/")[-1].split(".")[:-1]) + "." + source_num + ".geno" + else: + target_filename = ".".join(target_genofile.split("/")[-1].split(".")[:-1]) + ".geno" + + file_location = out_dir + target_filename + + with open(file_location, "w") as fh: + for metadata in ["name", "type", "mat", "pat", "het", "unk"]: + fh.write("@" + metadata + ":" + source_genotypes[metadata] + "\n") + + header_line = ["Chr", "Locus", "cM", "Mb"] + target_samples + fh.write("\t".join(header_line)) + + for marker in source_genotypes['markers']: + line_items = [ + marker['Chr'], + marker['Locus'], + marker['cM'], + marker['Mb'] + ] + + for pos in strain_pos_map: + if isinstance(pos, int): + line_items.append(marker['genotypes'][pos]) + else: + if pos in ["mat", "pat"]: + line_items.append(source_genotypes[pos]) + elif pos == "f1s": + line_items.append("H") + else: + line_items.append("U") + + fh.write("\t".join(line_items) + "\n") + + return file_location, target_samples + +def map_strain_pos_to_target_group(source_samples, target_samples, par_f1s): + """ + Retrieve corresponding strain position for each sample in the target group + + This is so the genotypes from the base genofile can be mapped to the samples in the target group + + For example: + Base strains: BXD1, BXD2, BXD3 + Target samples: BXD1_1, BXD1_2, BXD2_1, BXD3_1, BXD3_2, BXD3_3 + Returns: [0, 0, 1, 2, 2, 2] + """ + pos_map = [] + for sample in target_samples: + sample_strain = get_strain_for_sample(sample) + if sample_strain in source_samples: + pos_map.append(source_samples.index(sample_strain)) + else: + val = "U" + for key in par_f1s.keys(): + if sample_strain in par_f1s[key]: + val = key + pos_map.append(val) + + return pos_map + +def group_samples(target_file: str) -> List: + """ + Get the group samples from its "dummy" .geno file (which still contains the sample list) + """ + + sample_list = [] + with open(target_file, "r") as target_geno: + for i, line in enumerate(target_geno): + # Skip header lines + if line[0] in ["#", "@"] or not len(line): + continue + + line_items = line.split("\t") + sample_list = [item for item in line_items if item not in ["Chr", "Locus", "Mb", "cM"]] + break + + return sample_list + +def strain_genotypes(strain_genofile: str) -> List: + """ + Read genotypes from source strain .geno file + + :param strain_genofile: string of genofile filename + :return: a list of dictionaries representing each marker's genotypes + + Example output: [ + { + 'Chr': '1', + 'Locus': 'marker1', + 'Mb': '10.0', + 'cM': '8.0', + 'genotypes': [('BXD1', 'B'), ('BXD2', 'D'), ('BXD3', 'H'), ...] + }, + ... + ] + """ + + geno_dict = {} + + geno_start_col = None + header_columns = [] + sample_list = [] + markers = [] + with open(strain_genofile, "r") as source_geno: + for i, line in enumerate(source_geno): + if line[0] == "@": + metadata_type = line[1:].split(":")[0] + if metadata_type in ['name', 'type', 'mat', 'pat', 'het', 'unk']: + geno_dict[metadata_type] = line.split(":")[1].strip() + + continue + + # Skip other header lines + if line[0] == "#" or not len(line): + continue + + line_items = line.split("\t") + if "Chr" in line_items: # Header row + # Get the first column index containing genotypes + header_columns = line_items + for j, item in enumerate(line_items): + if item not in ["Chr", "Locus", "Mb", "cM"]: + geno_start_col = j + break + + sample_list = line_items[geno_start_col:] + if not geno_start_col: + print("Check .geno file - expected columns not found") + sys.exit() + else: # Marker rows + this_marker = { + 'Chr': line_items[header_columns.index("Chr")], + 'Locus': line_items[header_columns.index("Locus")], + 'Mb': line_items[header_columns.index("Mb")], + 'cM': line_items[header_columns.index("cM")], + 'genotypes': [item.strip() for item in line_items][geno_start_col:] + } + + markers.append(this_marker) + + geno_dict['markers'] = markers + + return geno_dict + +if __name__ == "__main__": + main(sys.argv) + diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py index db65a11f..9f4b670d 100644 --- a/wqflask/maintenance/gen_select_dataset.py +++ b/wqflask/maintenance/gen_select_dataset.py @@ -39,21 +39,13 @@ from wqflask import app from utility.tools import locate, locate_ignore_error, TEMPDIR, SQL_URI -import MySQLdb - import simplejson as json import urllib.parse -#import sqlalchemy as sa - from pprint import pformat as pf -#Engine = sa.create_engine(zach_settings.SQL_URI) - -# build MySql database connection - -#conn = Engine.connect() +from wqflask.database import database_connection def parse_db_uri(): @@ -71,19 +63,19 @@ def parse_db_uri(): return db_conn_info -def get_species(): +def get_species(cursor): """Build species list""" - #Cursor.execute("select Name, MenuName from Species where Species.Name != 'macaque monkey' order by OrderId") - Cursor.execute("select Name, MenuName from Species order by OrderId") - species = list(Cursor.fetchall()) + #cursor.execute("select Name, MenuName from Species where Species.Name != 'macaque monkey' order by OrderId") + cursor.execute("select Name, MenuName from Species order by OrderId") + species = list(cursor.fetchall()) return species -def get_groups(species): +def get_groups(cursor, species): """Build groups list""" groups = {} for species_name, _species_full_name in species: - Cursor.execute("""select InbredSet.Name, InbredSet.FullName from InbredSet, + cursor.execute("""select InbredSet.Name, InbredSet.FullName from InbredSet, Species, ProbeFreeze, GenoFreeze, PublishFreeze where Species.Name = '%s' and InbredSet.SpeciesId = Species.Id and @@ -92,7 +84,7 @@ def get_groups(species): or ProbeFreeze.InbredSetId = InbredSet.Id) group by InbredSet.Name order by InbredSet.FullName""" % species_name) - results = Cursor.fetchall() + results = cursor.fetchall() groups[species_name] = list(results) return groups @@ -273,13 +265,13 @@ def build_datasets(species, group, type_name): return datasets -def main(): +def main(cursor): """Generates and outputs (as json file) the data for the main dropdown menus on the home page""" parse_db_uri() - species = get_species() - groups = get_groups(species) + species = get_species(cursor) + groups = get_groups(cursor, species) types = get_types(groups) datasets = get_datasets(types) @@ -316,6 +308,6 @@ def _test_it(): if __name__ == '__main__': - Conn = MySQLdb.Connect(**parse_db_uri()) - Cursor = Conn.cursor() - main() + with database_connection() as conn: + with conn.cursor() as cursor: + main(cursor) diff --git a/wqflask/maintenance/generate_probesetfreeze_file.py b/wqflask/maintenance/generate_probesetfreeze_file.py index e964c8ed..f43f952b 100644 --- a/wqflask/maintenance/generate_probesetfreeze_file.py +++ b/wqflask/maintenance/generate_probesetfreeze_file.py @@ -8,20 +8,11 @@ import os import collections import csv -import MySQLdb - from base import webqtlConfig from pprint import pformat as pf - -def get_cursor(): - con = MySQLdb.Connect(db=webqtlConfig.DB_UPDNAME, - host=webqtlConfig.MYSQL_UPDSERVER, - user=webqtlConfig.DB_UPDUSER, - passwd=webqtlConfig.DB_UPDPASSWD) - cursor = con.cursor() - return cursor +from wqflask.database import database_connection def show_progress(process, counter): @@ -116,13 +107,14 @@ def main(): "(Oct08)_RankInv_Beta.txt") dataset_name = "Eye_AXBXA_1008_RankInv" - cursor = get_cursor() - strains = get_strains(cursor) - print("Getting probset_vals") - probeset_vals = get_probeset_vals(cursor, dataset_name) - print("Finished getting probeset_vals") - trimmed_strains = trim_strains(strains, probeset_vals) - write_data_matrix_file(trimmed_strains, probeset_vals, filename) + with database_connection as conn: + with conn.cursor() as cursor: + strains = get_strains(cursor) + print("Getting probset_vals") + probeset_vals = get_probeset_vals(cursor, dataset_name) + print("Finished getting probeset_vals") + trimmed_strains = trim_strains(strains, probeset_vals) + write_data_matrix_file(trimmed_strains, probeset_vals, filename) if __name__ == '__main__': diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py index 32780ca6..90ec72de 100644 --- a/wqflask/maintenance/quantile_normalize.py +++ b/wqflask/maintenance/quantile_normalize.py @@ -1,6 +1,5 @@ import sys sys.path.insert(0, './') -import MySQLdb import urllib.parse import numpy as np @@ -9,6 +8,7 @@ import pandas as pd from flask import Flask, g, request from wqflask import app +from wqflask.database import database_connection def parse_db_uri(): @@ -52,7 +52,7 @@ def quantileNormalize(df_input): return df -def set_data(dataset_name): +def set_data(cursor, dataset_name): orig_file = "/home/zas1024/cfw_data/" + dataset_name + ".txt" sample_list = [] @@ -80,8 +80,8 @@ def set_data(dataset_name): ProbeSetFreeze.Id = ProbeSetXRef.ProbeSetFreezeId and ProbeSetXRef.ProbeSetId = ProbeSet.Id and ProbeSet.Name = '%s'""" % (dataset_name, line1.split('\t')[0]) - Cursor.execute(query) - result_info = Cursor.fetchone() + cursor.execute(query) + result_info = cursor.fetchone() yield { "_index": "traits", @@ -99,15 +99,14 @@ def set_data(dataset_name): if __name__ == '__main__': - Conn = MySQLdb.Connect(**parse_db_uri()) - Cursor = Conn.cursor() + with database_connection as conn: + with conn.cursor() as cursor: + success, _ = bulk(es, set_data(cursor, sys.argv[1])) - success, _ = bulk(es, set_data(sys.argv[1])) - - response = es.search( - index="traits", doc_type="trait", body={ - "query": {"match": {"name": "ENSMUSG00000028982"}} - } - ) + response = es.search( + index="traits", doc_type="trait", body={ + "query": {"match": {"name": "ENSMUSG00000028982"}} + } + ) - print(response) + print(response) diff --git a/wqflask/maintenance/set_resource_defaults.py b/wqflask/maintenance/set_resource_defaults.py index 0f472494..22d73ba3 100644 --- a/wqflask/maintenance/set_resource_defaults.py +++ b/wqflask/maintenance/set_resource_defaults.py @@ -30,10 +30,9 @@ from utility.tools import SQL_URI from utility.redis_tools import get_redis_conn, get_user_id, add_resource, get_resources, get_resource_info Redis = get_redis_conn() -import MySQLdb - import urllib.parse +from wqflask.database import database_connection from utility.logger import getLogger logger = getLogger(__name__) @@ -53,14 +52,14 @@ def parse_db_uri(): return db_conn_info -def insert_probeset_resources(default_owner_id): +def insert_probeset_resources(cursor, default_owner_id): current_resources = Redis.hgetall("resources") - Cursor.execute(""" SELECT + cursor.execute(""" SELECT ProbeSetFreeze.Id, ProbeSetFreeze.Name, ProbeSetFreeze.confidentiality, ProbeSetFreeze.public FROM ProbeSetFreeze""") - resource_results = Cursor.fetchall() + resource_results = cursor.fetchall() for i, resource in enumerate(resource_results): resource_ob = {} resource_ob['name'] = resource[1] @@ -80,9 +79,9 @@ def insert_probeset_resources(default_owner_id): add_resource(resource_ob, update=False) -def insert_publish_resources(default_owner_id): +def insert_publish_resources(cursor, default_owner_id): current_resources = Redis.hgetall("resources") - Cursor.execute(""" SELECT + cursor.execute(""" SELECT PublishXRef.Id, PublishFreeze.Id, InbredSet.InbredSetCode FROM PublishXRef, PublishFreeze, InbredSet, Publication @@ -91,7 +90,7 @@ def insert_publish_resources(default_owner_id): InbredSet.Id = PublishXRef.InbredSetId AND Publication.Id = PublishXRef.PublicationId""") - resource_results = Cursor.fetchall() + resource_results = cursor.fetchall() for resource in resource_results: if resource[2]: resource_ob = {} @@ -114,14 +113,14 @@ def insert_publish_resources(default_owner_id): continue -def insert_geno_resources(default_owner_id): +def insert_geno_resources(cursor, default_owner_id): current_resources = Redis.hgetall("resources") - Cursor.execute(""" SELECT + cursor.execute(""" SELECT GenoFreeze.Id, GenoFreeze.ShortName, GenoFreeze.confidentiality FROM GenoFreeze""") - resource_results = Cursor.fetchall() + resource_results = cursor.fetchall() for i, resource in enumerate(resource_results): resource_ob = {} resource_ob['name'] = resource[1] @@ -147,15 +146,15 @@ def insert_geno_resources(default_owner_id): def insert_resources(default_owner_id): current_resources = get_resources() print("START") - insert_publish_resources(default_owner_id) + insert_publish_resources(cursor, default_owner_id) print("AFTER PUBLISH") - insert_geno_resources(default_owner_id) + insert_geno_resources(cursor, default_owner_id) print("AFTER GENO") - insert_probeset_resources(default_owner_id) + insert_probeset_resources(cursor, default_owner_id) print("AFTER PROBESET") -def main(): +def main(cursor): """Generates and outputs (as json file) the data for the main dropdown menus on the home page""" Redis.delete("resources") @@ -166,6 +165,6 @@ def main(): if __name__ == '__main__': - Conn = MySQLdb.Connect(**parse_db_uri()) - Cursor = Conn.cursor() - main() + with database_connection() as conn: + with conn.cursor() as cursor: + main(cursor) diff --git a/wqflask/tests/unit/base/test_data_set.py b/wqflask/tests/unit/base/test_data_set.py index 66ad361d..505449a5 100644 --- a/wqflask/tests/unit/base/test_data_set.py +++ b/wqflask/tests/unit/base/test_data_set.py @@ -66,6 +66,7 @@ class TestDataSetTypes(unittest.TestCase): '"HC_M2_0606_P": "ProbeSet", ' '"BXDPublish": "Publish"}')) + @unittest.skip("Too complicated") @mock.patch('base.data_set.g') def test_set_dataset_key_mrna(self, db_mock): with app.app_context(): @@ -92,6 +93,7 @@ class TestDataSetTypes(unittest.TestCase): + "WHERE ProbeSetFreeze.Name = \"Test\" ") ) + @unittest.skip("Too complicated") @mock.patch('base.data_set.g') def test_set_dataset_key_pheno(self, db_mock): with app.app_context(): @@ -121,6 +123,7 @@ class TestDataSetTypes(unittest.TestCase): "InfoFiles.InfoPageName = PublishFreeze.Name") ) + @unittest.skip("Too complicated") @mock.patch('base.data_set.g') def test_set_dataset_other_pheno(self, db_mock): with app.app_context(): @@ -151,6 +154,7 @@ class TestDataSetTypes(unittest.TestCase): "PublishFreeze.InbredSetId = InbredSet.Id") ) + @unittest.skip("Too complicated") @mock.patch('base.data_set.g') def test_set_dataset_geno(self, db_mock): with app.app_context(): diff --git a/wqflask/tests/unit/base/test_species.py b/wqflask/tests/unit/base/test_species.py index 9b5c023c..87f89607 100644 --- a/wqflask/tests/unit/base/test_species.py +++ b/wqflask/tests/unit/base/test_species.py @@ -60,6 +60,7 @@ class TestIndChromosome(unittest.TestCase): self.assertEqual(test_ind_chromosome.mb_length, 10) +@unittest.skip("Too complicated") class TestChromosomes(unittest.TestCase): """Tests for Chromosomes class""" maxDiff = None diff --git a/wqflask/tests/unit/base/test_trait.py b/wqflask/tests/unit/base/test_trait.py index 826ccefd..71fe0a44 100644 --- a/wqflask/tests/unit/base/test_trait.py +++ b/wqflask/tests/unit/base/test_trait.py @@ -106,49 +106,57 @@ class TestRetrieveTraitInfo(unittest.TestCase): self.assertEqual(test_trait.authors, "Jane Doe かいと") + + @unittest.skip("Too complicated") @mock.patch('base.trait.requests.get') - @mock.patch('base.trait.g') + @mock.patch('base.trait.database_connection') @mock.patch('base.trait.get_resource_id') def test_retrieve_trait_info_with_non_empty_lrs(self, resource_id_mock, - g_mock, + mock_db, requests_mock): """Test retrieve trait info when lrs has a value""" resource_id_mock.return_value = 1 - g_mock.db.execute.return_value.fetchone = mock.Mock() - g_mock.db.execute.return_value.fetchone.side_effect = [ - [1, 2, 3, 4], # trait_info = g.db.execute(query).fetchone() - [1, 2.37, 3, 4, 5], # trait_qtl = g.db.execute(query).fetchone() - [2.7333, 2.1204] # trait_info = g.db.execute(query).fetchone() - ] - requests_mock.return_value = None - - mock_dataset = mock.MagicMock() - type(mock_dataset).display_fields = mock.PropertyMock( - return_value=["a", "b", "c", "d"]) - type(mock_dataset).type = "ProbeSet" - type(mock_dataset).name = "RandomName" - - mock_trait = MockTrait( - dataset=mock_dataset, - pre_publication_description="test_string" - ) - trait_attrs = { - "description": "some description", - "probe_target_description": "some description", - "cellid": False, - "chr": 2.733, - "mb": 2.1204 - } - - for key, val in list(trait_attrs.items()): - setattr(mock_trait, key, val) - test_trait = retrieve_trait_info(trait=mock_trait, - dataset=mock_dataset, - get_qtl_info=True) - self.assertEqual(test_trait.LRS_score_repr, - "2.4") - + conn = mock.MagicMock() + mock_db.return_value.__enter__.return_value = conn + with conn.cursor() as cursor: + cursor.fetchone.side_effect = [ + # trait_info = g.db.execute(query).fetchone() + [1, 2, 3, 4], + # trait_qtl = g.db.execute(query).fetchone() + [1, 2.37, 3, 4, 5], + # trait_info = g.db.execute(query).fetchone() + [2.7333, 2.1204] + ] + requests_mock.return_value = None + + mock_dataset = mock.MagicMock() + type(mock_dataset).display_fields = mock.PropertyMock( + return_value=["a", "b", "c", "d"]) + type(mock_dataset).type = "ProbeSet" + type(mock_dataset).name = "RandomName" + + mock_trait = MockTrait( + dataset=mock_dataset, + pre_publication_description="test_string" + ) + trait_attrs = { + "description": "some description", + "probe_target_description": "some description", + "cellid": False, + "chr": 2.733, + "mb": 2.1204 + } + + for key, val in list(trait_attrs.items()): + setattr(mock_trait, key, val) + test_trait = retrieve_trait_info(trait=mock_trait, + dataset=mock_dataset, + get_qtl_info=True) + self.assertEqual(test_trait.LRS_score_repr, + "2.4") + + @unittest.skip("Too complicated") @mock.patch('base.trait.requests.get') @mock.patch('base.trait.g') @mock.patch('base.trait.get_resource_id') @@ -193,7 +201,8 @@ class TestRetrieveTraitInfo(unittest.TestCase): "N/A") self.assertEqual(test_trait.LRS_location_repr, "Chr2: 3.000000") - + + @unittest.skip("Too complicated") @mock.patch('base.trait.requests.get') @mock.patch('base.trait.g') @mock.patch('base.trait.get_resource_id') diff --git a/wqflask/tests/unit/wqflask/api/test_correlation.py b/wqflask/tests/unit/wqflask/api/test_correlation.py index 1089a36f..53e1b9a2 100644 --- a/wqflask/tests/unit/wqflask/api/test_correlation.py +++ b/wqflask/tests/unit/wqflask/api/test_correlation.py @@ -20,12 +20,7 @@ class MockDataset(AttributeSetter): return None def retrieve_genes(self, id=None): - return { - "TT-1": "GH-1", - "TT-2": "GH-2", - "TT-3": "GH-3" - - } + return {"TT-1": "GH-1", "TT-2": "GH-2", "TT-3": "GH-3"} class TestCorrelations(unittest.TestCase): @@ -37,93 +32,112 @@ class TestCorrelations(unittest.TestCase): self.app_context.pop() def test_init_corr_params(self): - start_vars = { - "return_count": "3", - "type": "T1", - "method": "spearman" - } + start_vars = {"return_count": "3", "type": "T1", "method": "spearman"} corr_params_results = init_corr_params(start_vars=start_vars) - expected_results = { - "return_count": 3, - "type": "T1", - "method": "spearman" - } + expected_results = {"return_count": 3, "type": "T1", "method": "spearman"} self.assertEqual(corr_params_results, expected_results) - @mock.patch("wqflask.api.correlation.g") + @mock.patch("wqflask.api.correlation.database_connection") def test_convert_to_mouse_gene_id(self, mock_db): - - results = convert_to_mouse_gene_id(species="Other", gene_id="") - self.assertEqual(results, None) - - rat_species_results = convert_to_mouse_gene_id( - species="rat", gene_id="GH1") - - mock_db.db.execute.return_value.fetchone.side_effect = [ - AttributeSetter({"mouse": "MG-1"}), AttributeSetter({"mouse": "MG-2"})] - - self.assertEqual(convert_to_mouse_gene_id( - species="mouse", gene_id="MG-4"), "MG-4") - self.assertEqual(convert_to_mouse_gene_id( - species="rat", gene_id="R1"), "MG-1") - self.assertEqual(convert_to_mouse_gene_id( - species="human", gene_id="H1"), "MG-2") - - @mock.patch("wqflask.api.correlation.g") + conn = mock.MagicMock() + mock_db.return_value.__enter__.return_value = conn + with conn.cursor() as cursor: + cursor.fetchone.side_effect = [("MG-1",), ("MG-2",)] + + self.assertEqual( + convert_to_mouse_gene_id(species="Other", gene_id=""), None + ) + self.assertEqual( + convert_to_mouse_gene_id(species="mouse", gene_id="MG-4"), "MG-4" + ) + self.assertEqual( + convert_to_mouse_gene_id(species="rat", gene_id="R1"), "MG-1" + ) + self.assertEqual( + convert_to_mouse_gene_id(species="human", gene_id="H1"), "MG-2" + ) + + @mock.patch("wqflask.api.correlation.database_connection") @mock.patch("wqflask.api.correlation.convert_to_mouse_gene_id") - def test_do_literature_correlation_for_all_traits(self, mock_convert_to_mouse_geneid, mock_db): - mock_convert_to_mouse_geneid.side_effect = [ - "MG-1", "MG-2;", "MG-3", "MG-4"] - - trait_geneid_dict = { - "TT-1": "GH-1", - "TT-2": "GH-2", - "TT-3": "GH-3" - - } - mock_db.db.execute.return_value.fetchone.side_effect = [AttributeSetter( - {"value": "V1"}), AttributeSetter({"value": "V2"}), AttributeSetter({"value": "V3"})] - - this_trait = AttributeSetter({"geneid": "GH-1"}) - - target_dataset = AttributeSetter( - {"group": AttributeSetter({"species": "rat"})}) - results = do_literature_correlation_for_all_traits( - this_trait=this_trait, target_dataset=target_dataset, trait_geneid_dict=trait_geneid_dict, corr_params={}) - - expected_results = {'TT-1': ['GH-1', 0], - 'TT-2': ['GH-2', 'V1'], 'TT-3': ['GH-3', 'V2']} - self.assertEqual(results, expected_results) + def test_do_literature_correlation_for_all_traits( + self, mock_convert_to_mouse_geneid, mock_db + ): + mock_convert_to_mouse_geneid.side_effect = ["MG-1", "MG-2;", "MG-3", "MG-4"] + + trait_geneid_dict = {"TT-1": "GH-1", "TT-2": "GH-2", "TT-3": "GH-3"} + conn = mock.MagicMock() + mock_db.return_value.__enter__.return_value = conn + with conn.cursor() as cursor: + cursor.fetchone.side_effect = [("V1",), ("V2",), ("V3",)] + this_trait = AttributeSetter({"geneid": "GH-1"}) + target_dataset = AttributeSetter( + {"group": AttributeSetter({"species": "rat"})} + ) + results = do_literature_correlation_for_all_traits( + this_trait=this_trait, + target_dataset=target_dataset, + trait_geneid_dict=trait_geneid_dict, + corr_params={}, + ) + expected_results = { + "TT-1": ["GH-1", 0], + "TT-2": ["GH-2", "V1"], + "TT-3": ["GH-3", "V2"], + } + self.assertEqual(results, expected_results) @mock.patch("wqflask.api.correlation.corr_result_helpers.normalize_values") def test_get_sample_r_and_p_values(self, mock_normalize): group = AttributeSetter( - {"samplelist": ["S1", "S2", "S3", "S4", "S5", "S6", "S7"]}) + {"samplelist": ["S1", "S2", "S3", "S4", "S5", "S6", "S7"]} + ) target_dataset = AttributeSetter({"group": group}) target_vals = [3.4, 6.2, 4.1, 3.4, 1.2, 5.6] - trait_data = {"S1": AttributeSetter({"value": 2.3}), "S2": AttributeSetter({"value": 1.1}), - "S3": AttributeSetter( - {"value": 6.3}), "S4": AttributeSetter({"value": 3.6}), "S5": AttributeSetter({"value": 4.1}), - "S6": AttributeSetter({"value": 5.0})} + trait_data = { + "S1": AttributeSetter({"value": 2.3}), + "S2": AttributeSetter({"value": 1.1}), + "S3": AttributeSetter({"value": 6.3}), + "S4": AttributeSetter({"value": 3.6}), + "S5": AttributeSetter({"value": 4.1}), + "S6": AttributeSetter({"value": 5.0}), + } this_trait = AttributeSetter({"data": trait_data}) - mock_normalize.return_value = ([2.3, 1.1, 6.3, 3.6, 4.1, 5.0], - [3.4, 6.2, 4.1, 3.4, 1.2, 5.6], 6) - mock_normalize.side_effect = [([2.3, 1.1, 6.3, 3.6, 4.1, 5.0], - [3.4, 6.2, 4.1, 3.4, 1.2, 5.6], 6), - ([2.3, 1.1, 6.3, 3.6, 4.1, 5.0], - [3.4, 6.2, 4.1, 3.4, 1.2, 5.6], 6), - ([2.3, 1.1, 1.4], [3.4, 6.2, 4.1], 3)] - - results_pearsonr = get_sample_r_and_p_values(this_trait=this_trait, this_dataset={ - }, target_vals=target_vals, target_dataset=target_dataset, type="pearson") - results_spearmanr = get_sample_r_and_p_values(this_trait=this_trait, this_dataset={ - }, target_vals=target_vals, target_dataset=target_dataset, type="spearman") - results_num_overlap = get_sample_r_and_p_values(this_trait=this_trait, this_dataset={ - }, target_vals=target_vals, target_dataset=target_dataset, type="pearson") + mock_normalize.return_value = ( + [2.3, 1.1, 6.3, 3.6, 4.1, 5.0], + [3.4, 6.2, 4.1, 3.4, 1.2, 5.6], + 6, + ) + mock_normalize.side_effect = [ + ([2.3, 1.1, 6.3, 3.6, 4.1, 5.0], [3.4, 6.2, 4.1, 3.4, 1.2, 5.6], 6), + ([2.3, 1.1, 6.3, 3.6, 4.1, 5.0], [3.4, 6.2, 4.1, 3.4, 1.2, 5.6], 6), + ([2.3, 1.1, 1.4], [3.4, 6.2, 4.1], 3), + ] + + results_pearsonr = get_sample_r_and_p_values( + this_trait=this_trait, + this_dataset={}, + target_vals=target_vals, + target_dataset=target_dataset, + type="pearson", + ) + results_spearmanr = get_sample_r_and_p_values( + this_trait=this_trait, + this_dataset={}, + target_vals=target_vals, + target_dataset=target_dataset, + type="spearman", + ) + results_num_overlap = get_sample_r_and_p_values( + this_trait=this_trait, + this_dataset={}, + target_vals=target_vals, + target_dataset=target_dataset, + type="pearson", + ) expected_pearsonr = [-0.21618688834430866, 0.680771605997119, 6] expected_spearmanr = [-0.11595420713048969, 0.826848213385815, 6] for i, val in enumerate(expected_pearsonr): @@ -136,18 +150,26 @@ class TestCorrelations(unittest.TestCase): def test_calculate_results(self, literature_correlation): literature_correlation.return_value = { - 'TT-1': ['GH-1', 0], 'TT-2': ['GH-2', 3], 'TT-3': ['GH-3', 1]} + "TT-1": ["GH-1", 0], + "TT-2": ["GH-2", 3], + "TT-3": ["GH-3", 1], + } - this_dataset = MockDataset( - {"group": AttributeSetter({"species": "rat"})}) - target_dataset = MockDataset( - {"group": AttributeSetter({"species": "rat"})}) + this_dataset = MockDataset({"group": AttributeSetter({"species": "rat"})}) + target_dataset = MockDataset({"group": AttributeSetter({"species": "rat"})}) this_trait = AttributeSetter({"geneid": "GH-1"}) corr_params = {"type": "literature"} sorted_results = calculate_results( - this_trait=this_trait, this_dataset=this_dataset, target_dataset=target_dataset, corr_params=corr_params) - expected_results = {'TT-2': ['GH-2', 3], - 'TT-3': ['GH-3', 1], 'TT-1': ['GH-1', 0]} + this_trait=this_trait, + this_dataset=this_dataset, + target_dataset=target_dataset, + corr_params=corr_params, + ) + expected_results = { + "TT-2": ["GH-2", 3], + "TT-3": ["GH-3", 1], + "TT-1": ["GH-1", 0], + } self.assertTrue(isinstance(sorted_results, OrderedDict)) self.assertEqual(dict(sorted_results), expected_results) diff --git a/wqflask/tests/unit/wqflask/snp_browser/test_snp_browser.py b/wqflask/tests/unit/wqflask/snp_browser/test_snp_browser.py index 89442c47..a09d1538 100644 --- a/wqflask/tests/unit/wqflask/snp_browser/test_snp_browser.py +++ b/wqflask/tests/unit/wqflask/snp_browser/test_snp_browser.py @@ -17,89 +17,190 @@ class TestSnpBrowser(unittest.TestCase): self.app_context.pop() def test_get_header_list(self): - empty_columns = {"snp_source": "false", "conservation_score": "true", "gene_name": "false", - "transcript": "false", "exon": "false", "domain_2": "true", "function": "false", "function_details": "true"} + empty_columns = { + "snp_source": "false", + "conservation_score": "true", + "gene_name": "false", + "transcript": "false", + "exon": "false", + "domain_2": "true", + "function": "false", + "function_details": "true", + } strains = {"mouse": ["S1", "S2", "S3", "S4", "S5"], "rat": []} - expected_results = ([['Index', 'SNP ID', 'Chr', 'Mb', 'Alleles', 'ConScore', - 'Domain 1', 'Domain 2', 'Details'], - ['S1', 'S2', 'S3', 'S4', 'S5']], 5, - ['index', 'snp_name', 'chr', 'mb_formatted', 'alleles', - 'conservation_score', 'domain_1', 'domain_2', - 'function_details', 'S1', 'S2', 'S3', 'S4', 'S5']) + expected_results = ( + [ + [ + "Index", + "SNP ID", + "Chr", + "Mb", + "Alleles", + "ConScore", + "Domain 1", + "Domain 2", + "Details", + ], + ["S1", "S2", "S3", "S4", "S5"], + ], + 5, + [ + "index", + "snp_name", + "chr", + "mb_formatted", + "alleles", + "conservation_score", + "domain_1", + "domain_2", + "function_details", + "S1", + "S2", + "S3", + "S4", + "S5", + ], + ) results_with_snp = get_header_list( - variant_type="SNP", strains=strains, species="Mouse", empty_columns=empty_columns) + variant_type="SNP", + strains=strains, + species="Mouse", + empty_columns=empty_columns, + ) results_with_indel = get_header_list( - variant_type="InDel", strains=strains, species="rat", empty_columns=[]) + variant_type="InDel", strains=strains, species="rat", empty_columns=[] + ) expected_results_with_indel = ( - ['Index', 'ID', 'Type', 'InDel Chr', 'Mb Start', - 'Mb End', 'Strand', 'Size', 'Sequence', 'Source'], 0, - ['index', 'indel_name', 'indel_type', 'indel_chr', 'indel_mb_s', - 'indel_mb_e', 'indel_strand', 'indel_size', 'indel_sequence', 'source_name']) + [ + "Index", + "ID", + "Type", + "InDel Chr", + "Mb Start", + "Mb End", + "Strand", + "Size", + "Sequence", + "Source", + ], + 0, + [ + "index", + "indel_name", + "indel_type", + "indel_chr", + "indel_mb_s", + "indel_mb_e", + "indel_strand", + "indel_size", + "indel_sequence", + "source_name", + ], + ) self.assertEqual(expected_results, results_with_snp) self.assertEqual(expected_results_with_indel, results_with_indel) - @mock.patch("wqflask.snp_browser.snp_browser.g") + @mock.patch("wqflask.snp_browser.snp_browser.database_connection") def test_get_gene_id(self, mock_db): - mock_db.db.execute.return_value.fetchone.return_value = "517d729f-aa13-4413-a885-40a3f7ff768a" - db_query_value = """ - SELECT - geneId - FROM - GeneList - WHERE - SpeciesId = c9c0f59e-1259-4cba-91e6-831ef1a99c83 AND geneSymbol = 'INSR' - """ - results = get_gene_id( - species_id="c9c0f59e-1259-4cba-91e6-831ef1a99c83", gene_name="INSR") - mock_db.db.execute.assert_called_once_with(db_query_value) - self.assertEqual(results, "517d729f-aa13-4413-a885-40a3f7ff768a") + db_query_value = ( + "SELECT geneId FROM GeneList WHERE " "SpeciesId = %s AND geneSymbol = %s" + ) + conn = mock.MagicMock() + mock_db.return_value.__enter__.return_value = conn + with conn.cursor() as cursor: + cursor.fetchone.return_value = ( + ("517d729f-aa13-4413" "-a885-40a3f7ff768a"), + ) - @mock.patch("wqflask.snp_browser.snp_browser.g") + results = get_gene_id( + species_id="c9c0f59e-1259-4cba-91e6-831ef1a99c83", gene_name="INSR" + ) + cursor.execute.assert_called_once_with( + db_query_value, ("c9c0f59e-1259-4cba-91e6-831ef1a99c83", "INSR") + ) + self.assertEqual(results, "517d729f-aa13-4413-a885-40a3f7ff768a") + + @mock.patch("wqflask.snp_browser.snp_browser.database_connection") def test_gene_id_name_dict(self, mock_db): no_gene_names = [] - self.assertEqual("", get_gene_id_name_dict( - species_id="fregb343bui43g4", gene_name_list=no_gene_names)) - gene_name_list = ["GH1", "GH2", "GH3"] - mock_db.db.execute.return_value.fetchall.side_effect = [[], [("fsdf43-fseferger-f22", "GH1"), ("1sdf43-fsewferger-f22", "GH2"), - ("fwdj43-fstferger-f22", "GH3")]] - no_results = get_gene_id_name_dict( - species_id="ret3-32rf32", gene_name_list=gene_name_list) - results_found = get_gene_id_name_dict( - species_id="ret3-32rf32", gene_name_list=gene_name_list) - expected_found = {'GH1': 'fsdf43-fseferger-f22', - 'GH2': '1sdf43-fsewferger-f22', 'GH3': 'fwdj43-fstferger-f22'} - db_query_value = """ - SELECT - geneId, geneSymbol - FROM - GeneList - WHERE - SpeciesId = ret3-32rf32 AND geneSymbol in ('GH1','GH2','GH3') - """ - mock_db.db.execute.assert_called_with(db_query_value) - self.assertEqual(results_found, expected_found) - self.assertEqual(no_results, {}) + conn = mock.MagicMock() + mock_db.return_value.__enter__.return_value = conn + with conn.cursor() as cursor: + cursor.fetchall.side_effect = [ + [], + [ + ("fsdf43-fseferger-f22", "GH1"), + ("1sdf43-fsewferger-f22", "GH2"), + ("fwdj43-fstferger-f22", "GH3"), + ], + ] + self.assertEqual( + "", + get_gene_id_name_dict( + species_id="fregb343bui43g4", gene_name_list=no_gene_names + ), + ) + gene_name_list = ["GH1", "GH2", "GH3"] + no_results = get_gene_id_name_dict( + species_id="ret3-32rf32", gene_name_list=gene_name_list + ) + results_found = get_gene_id_name_dict( + species_id="ret3-32rf32", gene_name_list=gene_name_list + ) + expected_found = { + "GH1": "fsdf43-fseferger-f22", + "GH2": "1sdf43-fsewferger-f22", + "GH3": "fwdj43-fstferger-f22", + } + db_query_value = ( + "SELECT geneId, geneSymbol FROM GeneList WHERE " + "SpeciesId = %s AND geneSymbol in (%s, %s, %s)" + ) + cursor.execute.assert_called_with( + db_query_value, ("ret3-32rf32", "GH1", "GH2", "GH3") + ) + self.assertEqual(results_found, expected_found) + self.assertEqual(no_results, {}) - @mock.patch("wqflask.snp_browser.snp_browser.g") + @mock.patch("wqflask.snp_browser.snp_browser.database_connection") def test_check_if_in_gene(self, mock_db): - mock_db.db.execute.return_value.fetchone.side_effect = [ - ("fsdf-232sdf-sdf", "GHA"), ""] - results_found = check_if_in_gene( - species_id="517d729f-aa13-4413-a885-40a3f7ff768a", chr="CH1", mb=12.09) - db_query_value = """SELECT geneId, geneSymbol - FROM GeneList - WHERE SpeciesId = 517d729f-aa13-4413-a885-40a3f7ff768a AND chromosome = 'CH1' AND - (txStart < 12.09 AND txEnd > 12.09); """ - gene_not_found = check_if_in_gene( - species_id="517d729f-aa13-4413-a885-40a3f7ff768a", chr="CH1", mb=12.09) - mock_db.db.execute.assert_called_with(db_query_value) - self.assertEqual(gene_not_found, "") + conn = mock.MagicMock() + mock_db.return_value.__enter__.return_value = conn + with conn.cursor() as cursor: + cursor.fetchone.side_effect = [("fsdf-232sdf-sdf", "GHA"), ""] + results_found = check_if_in_gene( + species_id="517d729f-aa13-4413-a885-40a3f7ff768a", chr_="CH1", mb=12.09 + ) + self.assertEqual(results_found, ["fsdf-232sdf-sdf", "GHA"]) + db_query_value = ( + "SELECT geneId, geneSymbol FROM GeneList " + "WHERE SpeciesId = %s AND chromosome = %s " + "AND (txStart < %s AND txEnd > %s)" + ) + gene_not_found = check_if_in_gene( + species_id="517d729f-aa13-4413-a885-40a3f7ff768a", chr_="CH1", mb=12.09 + ) + cursor.execute.assert_has_calls( + [ + mock.call( + db_query_value, + ("517d729f-aa13-4413-a885-40a3f7ff768a", "CH1", 12.09, 12.09), + ), + mock.call( + db_query_value, + ("517d729f-aa13-4413-a885-40a3f7ff768a", "CH1", 12.09, 12.09), + ), + ] + ) + self.assertEqual(gene_not_found, "") - @mock.patch("wqflask.snp_browser.snp_browser.g") + @mock.patch("wqflask.snp_browser.snp_browser.database_connection") def test_get_browser_sample_lists(self, mock_db): - mock_db.db.execute.return_value.fetchall.return_value = [] - - results = get_browser_sample_lists(species_id="12") - self.assertEqual(results, {'mouse': [], 'rat': []}) + conn = mock.MagicMock() + mock_db.return_value.__enter__.return_value = conn + with conn.cursor() as cursor: + cursor.execute.return_value.fetchall.return_value = [] + results = get_browser_sample_lists(species_id="12") + self.assertEqual(results, {"mouse": [], "rat": []}) diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index ab8b9e66..118a7ff3 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -11,6 +11,8 @@ from utility import formatting from gn3.authentication import DataRole, AdminRole +from wqflask.database import parse_db_url + from wqflask.group_manager import group_management from wqflask.resource_manager import resource_management from wqflask.metadata_edits import metadata_edit @@ -29,17 +31,6 @@ from wqflask.jupyter_notebooks import jupyter_notebooks app = Flask(__name__) -# Helper function for getting the SQL objects -def parse_db_url(sql_uri: str) -> Tuple: - """Parse SQL_URI env variable from an sql URI - e.g. 'mysql://user:pass@host_name/db_name' - - """ - parsed_db = urlparse(sql_uri) - return (parsed_db.hostname, parsed_db.username, - parsed_db.password, parsed_db.path[1:]) - - # See http://flask.pocoo.org/docs/config/#configuring-from-files # Note no longer use the badly named WQFLASK_OVERRIDES (nyi) app.config.from_envvar('GN2_SETTINGS') diff --git a/wqflask/wqflask/api/correlation.py b/wqflask/wqflask/api/correlation.py index 9b875c99..ab1e772a 100644 --- a/wqflask/wqflask/api/correlation.py +++ b/wqflask/wqflask/api/correlation.py @@ -3,11 +3,9 @@ import scipy from base import data_set from base.trait import create_trait, retrieve_sample_data -from flask import g from utility import corr_result_helpers -from utility.db_tools import escape from wqflask.correlation import correlation_functions - +from wqflask.database import database_connection def do_correlation(start_vars): assert('db' in start_vars) @@ -125,22 +123,24 @@ def do_literature_correlation_for_all_traits(this_trait, target_dataset, trait_g target_dataset.group.species.lower(), gene_id) if mouse_gene_id and str(mouse_gene_id).find(";") == -1: - result = g.db.execute( - """SELECT value - FROM LCorrRamin3 - WHERE GeneId1='%s' and - GeneId2='%s' - """ % (escape(mouse_gene_id), escape(input_trait_mouse_gene_id)) - ).fetchone() - if not result: - result = g.db.execute("""SELECT value - FROM LCorrRamin3 - WHERE GeneId2='%s' and - GeneId1='%s' - """ % (escape(mouse_gene_id), escape(input_trait_mouse_gene_id)) - ).fetchone() + result = "" + with database_connection() as conn: + with conn.cursor() as cursor: + cursor.execute( + ("SELECT value FROM LCorrRamin3 " + "WHERE GeneId1=%s AND GeneId2=%s"), + (mouse_gene_id, + input_trait_mouse_gene_id)) + result = cursor.fetchone() + if not result: + cursor.execute( + ("SELECT value FROM LCorrRamin3 " + "WHERE GeneId2=%s AND GeneId1=%s"), + (mouse_gene_id, + input_trait_mouse_gene_id)) + result = cursor.fetchone() if result: - lit_corr = result.value + lit_corr = result[0] lit_corr_data[trait] = [gene_id, lit_corr] else: lit_corr_data[trait] = [gene_id, 0] @@ -195,30 +195,24 @@ def convert_to_mouse_gene_id(species=None, gene_id=None): return None mouse_gene_id = None - - if species == 'mouse': - mouse_gene_id = gene_id - - elif species == 'rat': - - query = """SELECT mouse - FROM GeneIDXRef - WHERE rat='%s'""" % escape(gene_id) - - result = g.db.execute(query).fetchone() - if result != None: - mouse_gene_id = result.mouse - - elif species == 'human': - - query = """SELECT mouse - FROM GeneIDXRef - WHERE human='%s'""" % escape(gene_id) - - result = g.db.execute(query).fetchone() - if result != None: - mouse_gene_id = result.mouse - + with database_connection() as conn: + with conn.cursor() as cursor: + if species == 'mouse': + mouse_gene_id = gene_id + elif species == 'rat': + cursor.execute( + ("SELECT mouse FROM GeneIDXRef " + "WHERE rat=%s"), gene_id) + result = cursor.fetchone() + if result: + mouse_gene_id = result[0] + elif species == 'human': + cursor.execute( + "SELECT mouse FROM GeneIDXRef " + "WHERE human=%s", gene_id) + result = cursor.fetchone() + if result: + mouse_gene_id = result[0] return mouse_gene_id diff --git a/wqflask/wqflask/api/router.py b/wqflask/wqflask/api/router.py index a739e5a9..3d33cc87 100644 --- a/wqflask/wqflask/api/router.py +++ b/wqflask/wqflask/api/router.py @@ -6,7 +6,6 @@ import csv import json import datetime import requests -import MySQLdb from zipfile import ZipFile, ZIP_DEFLATED @@ -24,6 +23,8 @@ from wqflask.api import correlation, mapping, gen_menu from utility.tools import flat_files +from wqflask.database import database_connection + import utility.logger logger = utility.logger.getLogger(__name__) @@ -847,11 +848,8 @@ def get_genotypes(group_name, file_format="csv", dataset_name=None): @app.route("/api/v_{}/gen_dropdown".format(version), methods=("GET",)) def gen_dropdown_menu(): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - results = gen_menu.gen_dropdown_json(conn) + with database_connection() as conn: + results = gen_menu.gen_dropdown_json(conn) if len(results) > 0: return flask.jsonify(results) diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 76ef5ca4..815bb7c1 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -189,6 +189,30 @@ def delete_collection(): return redirect(url_for('list_collections')) +def trait_info_str(trait): + """Provide a string representation for given trait""" + def __trait_desc(trt): + if trait.dataset.type == "Geno": + return f"Marker: {trt.name}" + return trt.description_display or "N/A" + + def __symbol(trt): + return (trt.symbol or trt.abbreviation or "N/A")[:20] + + def __lrs(trt): + return ( + f"{float(trait.LRS_score_repr):0.3f}" if float(trait.LRS_score_repr) > 0 + else f"{trait.LRS_score_repr}") + + def __location(trt): + if hasattr(trt, "location_repr"): + return trt.location_repr + return None + + return "{}|||{}|||{}|||{}|||{}|||{:0.3f}|||{}|||{}".format( + trait.name, trait.dataset.name, __trait_desc(trait), __symbol(trait), + __location(trait), trait.mean, __lrs(trait), trait.LRS_location_repr) + @app.route("/collections/view") def view_collection(): params = request.args @@ -222,14 +246,15 @@ def view_collection(): collection_info = dict( trait_obs=trait_obs, uc=uc, - heatmap_data_url=f"{GN_SERVER_URL}heatmaps/clustered") + heatmap_data_url=f"{GN_SERVER_URL}api/heatmaps/clustered") if "json" in params: return json.dumps(json_version) else: - return render_template("collections/view.html", - **collection_info - ) + return render_template( + "collections/view.html", + trait_info_str=trait_info_str, + **collection_info) @app.route("/collections/change_name", methods=('POST',)) diff --git a/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py index cb88eb53..438d2276 100644 --- a/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py +++ b/wqflask/wqflask/comparison_bar_chart/comparison_bar_chart.py @@ -26,8 +26,6 @@ from utility import webqtlUtil, helper_functions, corr_result_helpers import utility.webqtlUtil # this is for parallel computing only. from wqflask.correlation import correlation_functions -from MySQLdb import escape_string as escape - from flask import Flask, g diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index 975a53b8..cb2f4470 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -6,29 +6,40 @@ from pathlib import Path from base.data_set import query_table_timestamp from base.webqtlConfig import TMPDIR +from json.decoder import JSONDecodeError + def fetch_all_cached_metadata(dataset_name): """in a gvein dataset fetch all the traits metadata""" file_name = generate_filename(dataset_name, suffix="metadata") - file_path = os.path.join(TMPDIR, file_name) + file_path = Path(TMPDIR, file_name) try: with open(file_path, "r+") as file_handler: dataset_metadata = json.load(file_handler) + return (file_path, dataset_metadata) except FileNotFoundError: - Path(file_path).touch(exist_ok=True) - return (file_path, {}) + pass + + except JSONDecodeError: + file_path.unlink() + + file_path.touch(exist_ok=True) + + return (file_path, {}) def cache_new_traits_metadata(dataset_metadata: dict, new_traits_metadata, file_path: str): """function to cache the new traits metadata""" - if bool(new_traits_metadata): - dataset_metadata.update(new_traits_metadata) - + if (dataset_metadata == {} and new_traits_metadata == {}): + return + + dataset_metadata.update(new_traits_metadata) + with open(file_path, "w+") as file_handler: json.dump(dataset_metadata, file_handler) diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py index 11f8d287..9f659ae6 100644 --- a/wqflask/wqflask/database.py +++ b/wqflask/wqflask/database.py @@ -1,12 +1,33 @@ # Module to initialize sqlalchemy with flask +import os +import sys +from string import Template +from typing import Tuple +from urllib.parse import urlparse +import importlib + +import MySQLdb from sqlalchemy import create_engine from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.ext.declarative import declarative_base -from utility.tools import SQL_URI +def read_from_pyfile(pyfile, setting): + orig_sys_path = sys.path[:] + sys.path.insert(0, os.path.dirname(pyfile)) + module = importlib.import_module(os.path.basename(pyfile).strip(".py")) + sys.path = orig_sys_path[:] + return module.__dict__.get(setting) + +def sql_uri(): + """Read the SQL_URI from the environment or settings file.""" + return os.environ.get( + "SQL_URI", read_from_pyfile( + os.environ.get( + "GN2_SETTINGS", os.path.abspath("../etc/default_settings.py")), + "SQL_URI")) -engine = create_engine(SQL_URI, encoding="latin1") +engine = create_engine(sql_uri(), encoding="latin1") db_session = scoped_session(sessionmaker(autocommit=False, autoflush=False, @@ -16,3 +37,17 @@ Base.query = db_session.query_property() # Initialise the db Base.metadata.create_all(bind=engine) + +def parse_db_url(sql_uri: str) -> Tuple: + """ + Parse SQL_URI env variable from an sql URI + e.g. 'mysql://user:pass@host_name/db_name' + """ + parsed_db = urlparse(sql_uri) + return (parsed_db.hostname, parsed_db.username, + parsed_db.password, parsed_db.path[1:]) + +def database_connection(): + """Returns a database connection""" + host, user, passwd, db_name = parse_db_url(sql_uri()) + return MySQLdb.Connect(db=db_name, user=user, passwd=passwd, host=host) diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 5c182260..1e245d6a 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -1,6 +1,7 @@ -import string -import requests import json +import re +import requests +import string from flask import Flask, g @@ -82,7 +83,7 @@ class MrnaAssaySearch(DoSearch): DoSearch.search_types['ProbeSet'] = "MrnaAssaySearch" base_query = """ - SELECT + SELECT DISTINCT ProbeSetFreeze.`Name`, ProbeSetFreeze.`FullName`, ProbeSet.`Name`, @@ -137,15 +138,17 @@ class MrnaAssaySearch(DoSearch): search_string = escape(self.search_term[0]) if self.search_term[0] != "*": - match_clause = """((MATCH (ProbeSet.Name, + if re.search("\w{1,2}\-\w+|\w+\-\w{1,2}", self.search_term[0]): + search_string = f'"{search_string}*"' + + match_clause = f"""((MATCH (ProbeSet.Name, ProbeSet.description, ProbeSet.symbol, alias, GenbankId, UniGeneId, Probe_Target_Description) - AGAINST ('%s' IN BOOLEAN MODE))) AND - """ % (search_string) + AGAINST ('{search_string}' IN BOOLEAN MODE))) AND """ else: match_clause = "" @@ -986,8 +989,6 @@ def get_aliases(symbol, species): if __name__ == "__main__": # Usually this will be used as a library, but call it from the command line for testing # And it runs the code below - - import MySQLdb import sys from base import webqtlConfig @@ -995,15 +996,11 @@ if __name__ == "__main__": from utility import webqtlUtil from db import webqtlDatabaseFunction - db_conn = MySQLdb.Connect(db=webqtlConfig.DB_NAME, - host=webqtlConfig.MYSQL_SERVER, - user=webqtlConfig.DB_USER, - passwd=webqtlConfig.DB_PASSWD) - cursor = db_conn.cursor() - - dataset_name = "HC_M2_0606_P" - dataset = create_dataset(db_conn, dataset_name) + from wqflask.database import database_connection - results = PvalueSearch(['0.005'], '<', dataset, cursor, db_conn).run() + with database_connection() as db_conn: + with db_conn.cursor() as cursor: + dataset_name = "HC_M2_0606_P" + dataset = create_dataset(db_conn, dataset_name) - db_conn.close() + results = PvalueSearch(['0.005'], '<', dataset, cursor, db_conn).run() diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index bceb9f3b..30acf4d4 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -1,13 +1,11 @@ import datetime import json import os -import re from collections import namedtuple from itertools import groupby from typing import Dict -import MySQLdb import difflib import redis @@ -21,15 +19,18 @@ from flask import render_template from flask import request from flask import url_for +from wqflask.database import database_connection from wqflask.decorators import edit_access_required from wqflask.decorators import edit_admins_access_required from wqflask.decorators import login_required from gn3.authentication import AdminRole -from gn3.authentication import DataRole from gn3.authentication import get_highest_user_access_role -from gn3.authentication import get_user_membership -from gn3.commands import run_cmd +from gn3.csvcmp import create_dirs_if_not_exists +from gn3.csvcmp import csv_diff +from gn3.csvcmp import extract_invalid_csv_headers +from gn3.csvcmp import get_allowable_sampledata_headers +from gn3.csvcmp import remove_insignificant_edits from gn3.db import diff_from_dict from gn3.db import fetchall from gn3.db import fetchone @@ -41,10 +42,10 @@ from gn3.db.phenotypes import Probeset from gn3.db.phenotypes import Publication from gn3.db.phenotypes import PublishXRef from gn3.db.phenotypes import probeset_mapping -from gn3.db.traits import get_trait_csv_sample_data -from gn3.db.traits import update_sample_data -from gn3.db.traits import delete_sample_data -from gn3.db.traits import insert_sample_data +from gn3.db.sample_data import delete_sample_data +from gn3.db.sample_data import get_trait_csv_sample_data +from gn3.db.sample_data import insert_sample_data +from gn3.db.sample_data import update_sample_data metadata_edit = Blueprint('metadata_edit', __name__) @@ -178,50 +179,41 @@ def edit_probeset(conn, name): @edit_access_required @login_required def display_phenotype_metadata(dataset_id: str, name: str): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - _d = edit_phenotype(conn=conn, name=name, dataset_id=dataset_id) - return render_template( - "edit_phenotype.html", - diff=_d.get("diff"), - publish_xref=_d.get("publish_xref"), - phenotype=_d.get("phenotype"), - publication=_d.get("publication"), - dataset_id=dataset_id, - resource_id=request.args.get("resource-id"), - version=os.environ.get("GN_VERSION"), - ) + with database_connection() as conn: + _d = edit_phenotype(conn=conn, name=name, dataset_id=dataset_id) + return render_template( + "edit_phenotype.html", + diff=_d.get("diff"), + publish_xref=_d.get("publish_xref"), + phenotype=_d.get("phenotype"), + publication=_d.get("publication"), + dataset_id=dataset_id, + resource_id=request.args.get("resource-id"), + headers=get_allowable_sampledata_headers(conn), + version=os.environ.get("GN_VERSION"), + ) @metadata_edit.route("/traits/<name>") @edit_access_required @login_required def display_probeset_metadata(name: str): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - _d = edit_probeset(conn=conn, name=name) - return render_template( - "edit_probeset.html", - diff=_d.get("diff"), - probeset=_d.get("probeset"), - name=name, - resource_id=request.args.get("resource-id"), - version=os.environ.get("GN_VERSION"), - ) + with database_connection as conn: + _d = edit_probeset(conn=conn, name=name) + return render_template( + "edit_probeset.html", + diff=_d.get("diff"), + probeset=_d.get("probeset"), + name=name, + resource_id=request.args.get("resource-id"), + version=os.environ.get("GN_VERSION"), + ) @metadata_edit.route("/<dataset_id>/traits/<name>", methods=("POST",)) @edit_access_required @login_required def update_phenotype(dataset_id: str, name: str): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) data_ = request.form.to_dict() TMPDIR = current_app.config.get("TMPDIR") author = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") @@ -230,72 +222,68 @@ def update_phenotype(dataset_id: str, name: str): if not (file_ := request.files.get("file")): flash("No sample-data has been uploaded", "warning") else: - if not os.path.exists(SAMPLE_DATADIR := os.path.join(TMPDIR, "sample-data")): - os.makedirs(SAMPLE_DATADIR) - if not os.path.exists(os.path.join(SAMPLE_DATADIR, - "diffs")): - os.makedirs(os.path.join(SAMPLE_DATADIR, - "diffs")) - if not os.path.exists(os.path.join(SAMPLE_DATADIR, - "updated")): - os.makedirs(os.path.join(SAMPLE_DATADIR, - "updated")) + create_dirs_if_not_exists([ + SAMPLE_DATADIR := os.path.join(TMPDIR, "sample-data"), + DIFF_DATADIR := os.path.join(SAMPLE_DATADIR, "diffs"), + UPLOAD_DATADIR := os.path.join(SAMPLE_DATADIR, "updated") + ]) + current_time = str(datetime.datetime.now().isoformat()) _file_name = (f"{author}.{request.args.get('resource-id')}." f"{current_time}") - new_file_name = (os.path.join(TMPDIR, - f"sample-data/updated/{_file_name}.csv")) - uploaded_file_name = (os.path.join( - TMPDIR, "sample-data/updated/", - f"{_file_name}.csv.uploaded")) - file_.save(new_file_name) - with open(uploaded_file_name, "w") as f_: - f_.write(get_trait_csv_sample_data( - conn=conn, - trait_name=str(name), - phenotype_id=str(phenotype_id))) - r = run_cmd(cmd=("csvdiff " - f"'{uploaded_file_name}' '{new_file_name}' " - "--format json")) - json_data = json.loads(r.get("output")) - - # Only consider values where |ε| < 0.001; otherwise, use the - # old value in "Original". - _modifications = [] - for m in json_data.get("Modifications"): - _original = m.get("Original").split(",") - _current = m.get("Current").split(",") - for i, (x, y) in enumerate(zip(_original, _current)): - if (x.replace('.', '').isdigit() - and y.replace('.', '').isdigit() - and abs(float(x) - float(y)) < 0.001): - _current[i] = x - if not (__o:=",".join(_original)) == (__c:=",".join(_current)): - _modifications.append( - { - "Original": __o, - "Current": __c, - }) - json_data['Modifications'] = _modifications - + diff_data = {} + with database_connection() as conn: + diff_data = remove_insignificant_edits( + diff_data=csv_diff( + base_csv=(base_csv := get_trait_csv_sample_data( + conn=conn, + trait_name=str(name), + phenotype_id=str(phenotype_id))), + delta_csv=(delta_csv := file_.read().decode()), + tmp_dir=TMPDIR), + epsilon=0.001) + headers = get_allowable_sampledata_headers(conn) + invalid_headers = extract_invalid_csv_headers( + allowed_headers=headers, + csv_text=delta_csv) + if invalid_headers: + flash("You have invalid headers: " + f"""{', '.join(invalid_headers)}. Valid headers """ + f"""are: {', '.join(headers)}""", + "warning") + return redirect( + f"/datasets/{dataset_id}/traits/{name}" + f"?resource-id={request.args.get('resource-id')}") # Edge case where the csv file has not been edited! - if not any(json_data.values()): - flash(f"You have not modified the csv file you downloaded!", + if not any(diff_data.values()): + flash("You have not modified the csv file you downloaded!", "warning") return redirect(f"/datasets/{dataset_id}/traits/{name}" f"?resource-id={request.args.get('resource-id')}") - diff_output = (f"{TMPDIR}/sample-data/diffs/" - f"{_file_name}.json") - with open(diff_output, "w") as f: - dict_ = json_data - dict_.update({ + + with open(os.path.join( + UPLOAD_DATADIR, + f"{_file_name}.csv"), "w") as f_: + f_.write(base_csv) + with open(os.path.join( + UPLOAD_DATADIR, + f"{_file_name}.delta.csv"), "w") as f_: + f_.write(delta_csv) + + with open(os.path.join(DIFF_DATADIR, + f"{_file_name}.json"), "w") as f: + diff_data.update({ "trait_name": str(name), "phenotype_id": str(phenotype_id), + "dataset_id": name, + "resource_id": request.args.get('resource-id'), "author": author, - "timestamp": datetime.datetime.now().strftime( - "%Y-%m-%d %H:%M:%S") + "timestamp": (datetime + .datetime + .now() + .strftime("%Y-%m-%d %H:%M:%S")), }) - f.write(json.dumps(dict_)) + f.write(json.dumps(diff_data)) flash("Sample-data has been successfully uploaded", "success") # Run updates: phenotype_ = { @@ -310,10 +298,12 @@ def update_phenotype(dataset_id: str, name: str): "owner": data_.get("owner"), "authorized_users": data_.get("authorized-users"), } - updated_phenotypes = update( - conn, "Phenotype", - data=Phenotype(**phenotype_), - where=Phenotype(id_=data_.get("phenotype-id"))) + updated_phenotypes = "" + with database_connection() as conn: + updated_phenotypes = update( + conn, "Phenotype", + data=Phenotype(**phenotype_), + where=Phenotype(id_=data_.get("phenotype-id"))) diff_data = {} if updated_phenotypes: diff_data.update({"Phenotype": diff_from_dict(old={ @@ -329,11 +319,13 @@ def update_phenotype(dataset_id: str, name: str): "month": data_.get("month"), "year": data_.get("year") } - updated_publications = update( - conn, "Publication", - data=Publication(**publication_), - where=Publication(id_=data_.get("pubmed-id", - data_.get("old_id_")))) + updated_publications = "" + with database_connection() as conn: + updated_publications = update( + conn, "Publication", + data=Publication(**publication_), + where=Publication(id_=data_.get("pubmed-id", + data_.get("old_id_")))) if updated_publications: diff_data.update({"Publication": diff_from_dict(old={ k: data_.get(f"old_{k}") for k, v in publication_.items() @@ -349,11 +341,12 @@ def update_phenotype(dataset_id: str, name: str): .now() .strftime("%Y-%m-%d %H:%M:%S")), }) - insert(conn, - table="metadata_audit", - data=MetadataAudit(dataset_id=name, - editor=author, - json_data=json.dumps(diff_data))) + with database_connection() as conn: + insert(conn, + table="metadata_audit", + data=MetadataAudit(dataset_id=name, + editor=author, + json_data=json.dumps(diff_data))) flash(f"Diff-data: \n{diff_data}\nhas been uploaded", "success") return redirect(f"/datasets/{dataset_id}/traits/{name}" f"?resource-id={request.args.get('resource-id')}") @@ -363,76 +356,71 @@ def update_phenotype(dataset_id: str, name: str): @edit_access_required @login_required def update_probeset(name: str): - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - data_ = request.form.to_dict() - probeset_ = { - "id_": data_.get("id"), - "symbol": data_.get("symbol"), - "description": data_.get("description"), - "probe_target_description": data_.get("probe_target_description"), - "chr_": data_.get("chr"), - "mb": data_.get("mb"), - "alias": data_.get("alias"), - "geneid": data_.get("geneid"), - "homologeneid": data_.get("homologeneid"), - "unigeneid": data_.get("unigeneid"), - "omim": data_.get("OMIM"), - "refseq_transcriptid": data_.get("refseq_transcriptid"), - "blatseq": data_.get("blatseq"), - "targetseq": data_.get("targetseq"), - "strand_probe": data_.get("Strand_Probe"), - "probe_set_target_region": data_.get("probe_set_target_region"), - "probe_set_specificity": data_.get("probe_set_specificity"), - "probe_set_blat_score": data_.get("probe_set_blat_score"), - "probe_set_blat_mb_start": data_.get("probe_set_blat_mb_start"), - "probe_set_blat_mb_end": data_.get("probe_set_blat_mb_end"), - "probe_set_strand": data_.get("probe_set_strand"), - "probe_set_note_by_rw": data_.get("probe_set_note_by_rw"), - "flag": data_.get("flag") - } - diff_data = {} - author = ((g.user_session.record.get(b"user_id") or b"").decode("utf-8") - or g.user_session.record.get("user_id") or "") - if (updated_probeset := update( - conn, "ProbeSet", - data=Probeset(**probeset_), - where=Probeset(id_=data_.get("id")))): - diff_data.update({"Probeset": diff_from_dict(old={ - k: data_.get(f"old_{k}") for k, v in probeset_.items() - if v is not None}, new=probeset_)}) - if diff_data: - diff_data.update({"probeset_name": data_.get("probeset_name")}) - diff_data.update({"author": author}) - diff_data.update({"resource_id": request.args.get('resource-id')}) - diff_data.update({"timestamp": datetime.datetime.now().strftime( - "%Y-%m-%d %H:%M:%S")}) - insert(conn, - table="metadata_audit", - data=MetadataAudit(dataset_id=data_.get("id"), - editor=author, - json_data=json.dumps(diff_data))) - return redirect(f"/datasets/traits/{name}" - f"?resource-id={request.args.get('resource-id')}") + with database_connection as conn: + data_ = request.form.to_dict() + probeset_ = { + "id_": data_.get("id"), + "symbol": data_.get("symbol"), + "description": data_.get("description"), + "probe_target_description": data_.get("probe_target_description"), + "chr_": data_.get("chr"), + "mb": data_.get("mb"), + "alias": data_.get("alias"), + "geneid": data_.get("geneid"), + "homologeneid": data_.get("homologeneid"), + "unigeneid": data_.get("unigeneid"), + "omim": data_.get("OMIM"), + "refseq_transcriptid": data_.get("refseq_transcriptid"), + "blatseq": data_.get("blatseq"), + "targetseq": data_.get("targetseq"), + "strand_probe": data_.get("Strand_Probe"), + "probe_set_target_region": data_.get("probe_set_target_region"), + "probe_set_specificity": data_.get("probe_set_specificity"), + "probe_set_blat_score": data_.get("probe_set_blat_score"), + "probe_set_blat_mb_start": data_.get("probe_set_blat_mb_start"), + "probe_set_blat_mb_end": data_.get("probe_set_blat_mb_end"), + "probe_set_strand": data_.get("probe_set_strand"), + "probe_set_note_by_rw": data_.get("probe_set_note_by_rw"), + "flag": data_.get("flag") + } + diff_data = {} + author = ((g.user_session.record.get(b"user_id") + or b"").decode("utf-8") + or g.user_session.record.get("user_id") or "") + if update(conn, "ProbeSet", + data=Probeset(**probeset_), + where=Probeset(id_=data_.get("id"))): + diff_data.update({"Probeset": diff_from_dict(old={ + k: data_.get(f"old_{k}") for k, v in probeset_.items() + if v is not None}, new=probeset_)}) + if diff_data: + diff_data.update({"probeset_name": data_.get("probeset_name")}) + diff_data.update({"author": author}) + diff_data.update({"resource_id": request.args.get('resource-id')}) + diff_data.update({"timestamp": datetime.datetime.now().strftime( + "%Y-%m-%d %H:%M:%S")}) + insert(conn, + table="metadata_audit", + data=MetadataAudit(dataset_id=data_.get("id"), + editor=author, + json_data=json.dumps(diff_data))) + return redirect(f"/datasets/traits/{name}" + f"?resource-id={request.args.get('resource-id')}") @metadata_edit.route("/<dataset_id>/traits/<phenotype_id>/csv") @login_required -def get_sample_data_as_csv(dataset_id: str, phenotype_id: int): - return Response( - get_trait_csv_sample_data( - conn=MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")), - trait_name=str(dataset_id), - phenotype_id=str(phenotype_id)), - mimetype="text/csv", - headers={"Content-disposition": - f"attachment; filename=sample-data-{dataset_id}.csv"} - ) +def get_sample_data_as_csv(dataset_id: str, phenotype_id: int): + with database_connection() as conn: + return Response( + get_trait_csv_sample_data( + conn=conn, + trait_name=str(dataset_id), + phenotype_id=str(phenotype_id)), + mimetype="text/csv", + headers={"Content-disposition": + f"attachment; filename=sample-data-{dataset_id}.csv"} + ) @metadata_edit.route("/diffs") @@ -490,88 +478,77 @@ def reject_data(resource_id: str, file_name: str): @metadata_edit.route("<resource_id>/diffs/<file_name>/approve") @edit_admins_access_required @login_required -def approve_data(resource_id:str, file_name: str): +def approve_data(resource_id: str, file_name: str): sample_data = {file_name: str} - conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) TMPDIR = current_app.config.get("TMPDIR") with open(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), 'r') as myfile: sample_data = json.load(myfile) - for modification in ( - modifications := [d for d in sample_data.get("Modifications")]): - if modification.get("Current"): - (strain_name, - value, se, count) = modification.get("Current").split(",") - update_sample_data( - conn=conn, - trait_name=sample_data.get("trait_name"), - strain_name=strain_name, - phenotype_id=int(sample_data.get("phenotype_id")), - value=value, - error=se, - count=count) + with database_connection() as conn: + for modification in ( + modifications := [d for d in + sample_data.get("Modifications")]): + if modification.get("Current"): + update_sample_data( + conn=conn, + trait_name=sample_data.get("trait_name"), + original_data=modification.get("Original"), + updated_data=modification.get("Current"), + csv_header=sample_data.get("Columns", + "Strain Name,Value,SE,Count"), + phenotype_id=int(sample_data.get("phenotype_id"))) n_deletions = 0 - for deletion in (deletions := [d for d in sample_data.get("Deletions")]): - strain_name, _, _, _ = deletion.split(",") - __deletions, _, _ = delete_sample_data( - conn=conn, - trait_name=sample_data.get("trait_name"), - strain_name=strain_name, - phenotype_id=int(sample_data.get("phenotype_id"))) - if __deletions: - n_deletions += 1 - # Remove any data that already exists from sample_data deletes - else: - sample_data.get("Deletions").remove(deletion) + with database_connection() as conn: + for data in [d for d in sample_data.get("Deletions")]: + __deletions = delete_sample_data( + conn=conn, + trait_name=sample_data.get("trait_name"), + data=data, + csv_header=sample_data.get("Columns", + "Strain Name,Value,SE,Count"), + phenotype_id=int(sample_data.get("phenotype_id"))) + if __deletions: + n_deletions += 1 + # Remove any data that already exists from sample_data deletes + else: + sample_data.get("Deletions").remove(data) n_insertions = 0 - for insertion in ( - insertions := [d for d in sample_data.get("Additions")]): - (strain_name, - value, se, count) = insertion.split(",") - __insertions, _, _ = insert_sample_data( - conn=conn, - trait_name=sample_data.get("trait_name"), - strain_name=strain_name, - phenotype_id=int(sample_data.get("phenotype_id")), - value=value, - error=se, - count=count) - if __insertions: - n_insertions += 1 - # Remove any data that already exists from sample_data inserts - else: - sample_data.get("Additions").remove(insertion) + with database_connection() as conn: + for data in [d for d in sample_data.get("Additions")]: + if insert_sample_data( + conn=conn, + trait_name=sample_data.get("trait_name"), + data=data, + csv_header=sample_data.get("Columns", + "Strain Name,Value,SE,Count"), + phenotype_id=int(sample_data.get("phenotype_id"))): + n_insertions += 1 if any([sample_data.get("Additions"), sample_data.get("Modifications"), sample_data.get("Deletions")]): - insert(conn, - table="metadata_audit", - data=MetadataAudit( - dataset_id=sample_data.get("trait_name"), - editor=sample_data.get("author"), - json_data=json.dumps(sample_data))) + with database_connection() as conn: + insert(conn, + table="metadata_audit", + data=MetadataAudit( + dataset_id=sample_data.get("trait_name"), + editor=sample_data.get("author"), + json_data=json.dumps(sample_data))) # Once data is approved, rename it! os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), os.path.join(f"{TMPDIR}/sample-data/diffs", f"{file_name}.approved")) - message = "" if n_deletions: flash(f"# Deletions: {n_deletions}", "success") if n_insertions: - flash("# Additions: {len(modifications)", "success") + flash(f"# Additions: {len(modifications)}", "success") if len(modifications): - flash("# Modifications: {len(modifications)}", "success") + flash(f"# Modifications: {len(modifications)}", "success") else: # Edge case where you need to automatically reject the file os.rename(os.path.join(f"{TMPDIR}/sample-data/diffs", file_name), os.path.join(f"{TMPDIR}/sample-data/diffs", f"{file_name}.rejected")) flash(("Automatically rejecting this file since no " "changes could be applied."), "warning") - return redirect(url_for('metadata_edit.list_diffs')) - diff --git a/wqflask/wqflask/parser.py b/wqflask/wqflask/parser.py index bd1c4407..7a808ac9 100644 --- a/wqflask/wqflask/parser.py +++ b/wqflask/wqflask/parser.py @@ -33,7 +33,7 @@ def parse(pstring): pstring = re.split(r"""(?:(\w+\s*=\s*[\('"\[][^)'"]*[\)\]'"]) | # LRS=(1 2 3), cisLRS=[4 5 6], etc (\w+\s*[=:\>\<][\w\*]+) | # wiki=bar, GO:foobar, etc (".*?") | ('.*?') | # terms in quotes, i.e. "brain weight" - ([\w\*\?]+)) # shh, brain, etc """, pstring, + ([\w\*\?\-]+)) # shh, brain, etc """, pstring, flags=re.VERBOSE) pstring = [item.strip() for item in pstring if item and item.strip()] diff --git a/wqflask/wqflask/partial_correlations_views.py b/wqflask/wqflask/partial_correlations_views.py index 6bc5efee..659b49e9 100644 --- a/wqflask/wqflask/partial_correlations_views.py +++ b/wqflask/wqflask/partial_correlations_views.py @@ -1,142 +1,31 @@ +import json +import math +import requests +from functools import reduce from typing import Union, Tuple -import MySQLdb -from gn3.db.traits import retrieve_trait_info -from flask import flash, request, current_app, render_template -from gn3.computations.partial_correlations import partial_correlations_entry +from flask import ( + flash, + request, + url_for, + redirect, + current_app, + render_template) from wqflask import app from utility.tools import GN_SERVER_URL +from wqflask.database import database_connection +from gn3.db.partial_correlations import traits_info -def parse_trait(trait_str: str) -> Union[dict, None]: - keys = ( - "name", "dataset", "symbol", "description", "location", "mean_expr", - "max_lrs", "data_hmac") - parts = tuple(part.strip() for part in trait_str.split(":::")) - if len(parts) == len(keys): - return dict(zip(keys, parts)) - return None - -def process_step_select_primary( - primary_trait: dict, control_traits: Tuple[dict, ...], - target_traits: Tuple[dict, ...], - traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ - str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], - str]: - if primary_trait is None: - flash("You must select a primary trait", "alert-danger") - return ( - "select-primary", primary_trait, control_traits, target_traits, - traits_list, corr_method) - - return ( - "select-controls", primary_trait, control_traits, target_traits, - tuple( - trait for trait in traits_list - if trait["data_hmac"] != primary_trait["data_hmac"]), - corr_method) - -def process_step_select_controls( - primary_trait: dict, control_traits: Tuple[dict, ...], - target_traits: Tuple[dict, ...], - traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ - str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], - str]: - if len(control_traits) == 0 or len(control_traits) > 3: - flash( - ("You must select a minimum of one control trait, up to a maximum " - "of three control traits."), - "alert-danger") - return ( - "select-controls", primary_trait, control_traits, target_traits, - traits_list, corr_method) - - hmacs =(primary_trait["data_hmac"],) + tuple( - trait["data_hmac"] for trait in control_traits) - return ( - "select-corr-method", primary_trait, control_traits, target_traits, - tuple( - trait for trait in traits_list if trait["data_hmac"] not in hmacs), - corr_method) - -def process_step_select_targets( - primary_trait: dict, control_traits: Tuple[dict, ...], - target_traits: Tuple[dict, ...], - traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ - str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], - str]: - if len(target_traits) == 0: - flash( - "You must select at least one target trait.", "alert-danger") - return ( - "select-targets", primary_trait, control_traits, target_traits, - traits_list, corr_method) - - hmacs =(primary_trait["data_hmac"],) + tuple( - trait["data_hmac"] for trait in (control_traits + target_traits)) - return ( - "select-corr-method", primary_trait, control_traits, target_traits, - tuple( - trait for trait in traits_list if trait["data_hmac"] not in hmacs), - corr_method) - -def process_step_select_corr_method( - primary_trait: dict, control_traits: Tuple[dict, ...], - target_traits: Tuple[dict, ...], - traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ - str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], - str]: - methods = ( - "genetic correlation, pearson's r", - "genetic correlation, spearman's rho", - "sgo literature correlation", - "tissue correlation, pearson's r", - "tissue correlation, spearman's rho") - if corr_method.lower() not in methods: - flash( - "Selected method is unknown.", "alert-danger") - return ( - "select-corr-method", primary_trait, control_traits, target_traits, - traits_list, corr_method) - - hmacs =(primary_trait["data_hmac"],) + tuple( - trait["data_hmac"] for trait in (control_traits + target_traits)) - return ( - "run-correlation", primary_trait, control_traits, target_traits, - tuple( - trait for trait in traits_list if trait["data_hmac"] not in hmacs), - corr_method) - -def process_step( - step: str, primary_trait: dict, control_traits: Tuple[dict, ...], - target_traits: Tuple[dict, ...], traits_list: Tuple[dict, ...], - corr_method: str) -> Tuple[ - str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], - str]: - processor_functions = { - # "select-traits": lambda arg: arg, - "select-primary": process_step_select_primary, - "select-controls": process_step_select_controls, - "select-targets": process_step_select_targets, - "select-corr-method": process_step_select_corr_method - } - return processor_functions[(step or "select-primary")]( - primary_trait, control_traits, target_traits, traits_list, corr_method) - -def sequence_of_traits(trait_strs) -> Tuple[dict, ...]: - return tuple(filter( - lambda trt: trt is not None, - (parse_trait(tstr.strip()) for tstr in trait_strs))) - -def publish_target_dabases(conn, group, threshold): +def publish_target_databases(conn, groups, threshold): query = ( "SELECT PublishFreeze.FullName,PublishFreeze.Name " "FROM PublishFreeze, InbredSet " "WHERE PublishFreeze.InbredSetId = InbredSet.Id " - "AND InbredSet.Name = %(group)s " - "AND PublishFreeze.public > %(threshold)s") + f"AND InbredSet.Name IN ({', '.join(['%s'] * len(groups))}) " + "AND PublishFreeze.public > %s") with conn.cursor() as cursor: - cursor.execute(query, {"group": group, "threshold": threshold}) + cursor.execute(query, tuple(groups) + (threshold,)) res = cursor.fetchall() if res: return tuple( @@ -144,15 +33,15 @@ def publish_target_dabases(conn, group, threshold): return tuple() -def geno_target_databases(conn, group, threshold): +def geno_target_databases(conn, groups, threshold): query = ( "SELECT GenoFreeze.FullName,GenoFreeze.Name " "FROM GenoFreeze, InbredSet " "WHERE GenoFreeze.InbredSetId = InbredSet.Id " - "AND InbredSet.Name = %(group)s " - "AND GenoFreeze.public > %(threshold)s") + f"AND InbredSet.Name IN ({', '.join(['%s'] * len(groups))}) " + "AND GenoFreeze.public > %s") with conn.cursor() as cursor: - cursor.execute(query, {"group": group, "threshold": threshold}) + cursor.execute(query, tuple(groups) + (threshold,)) res = cursor.fetchall() if res: return tuple( @@ -160,27 +49,26 @@ def geno_target_databases(conn, group, threshold): return tuple() -def probeset_target_databases(conn, group, threshold): +def probeset_target_databases(conn, groups, threshold): query1 = "SELECT Id, Name FROM Tissue order by Name" - query2 = ( - "SELECT ProbeFreeze.TissueId, ProbeSetFreeze.FullName, ProbeSetFreeze.Name " - "FROM ProbeSetFreeze, ProbeFreeze, InbredSet " - "WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id " - "AND ProbeFreeze.TissueId IN %(tissue_ids)s " - "AND ProbeSetFreeze.public > %(threshold)s " - "AND ProbeFreeze.InbredSetId = InbredSet.Id " - "AND InbredSet.Name like %(group)s " - "ORDER BY ProbeSetFreeze.CreateTime desc, ProbeSetFreeze.AvgId") with conn.cursor() as cursor: cursor.execute(query1) tissue_res = cursor.fetchall() if tissue_res: tissue_ids = tuple(row[0] for row in tissue_res) - cursor.execute( - query2,{ - "tissue_ids": tissue_ids, "group": f"{group}%%", - "threshold": threshold - }) + groups_clauses = ["InbredSet.Name like %s"] * len(groups) + query2 = ( + "SELECT ProbeFreeze.TissueId, ProbeSetFreeze.FullName, " + "ProbeSetFreeze.Name " + "FROM ProbeSetFreeze, ProbeFreeze, InbredSet " + "WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id " + "AND ProbeFreeze.TissueId IN " + f"({', '.join(['%s'] * len(tissue_ids))}) " + "AND ProbeSetFreeze.public > %s " + "AND ProbeFreeze.InbredSetId = InbredSet.Id " + f"AND ({' OR '.join(groups_clauses)}) " + "ORDER BY ProbeSetFreeze.CreateTime desc, ProbeSetFreeze.AvgId") + cursor.execute(query2, tissue_ids + (threshold,) + tuple(groups)) db_res = cursor.fetchall() if db_res: databases = tuple( @@ -197,70 +85,201 @@ def probeset_target_databases(conn, group, threshold): return tuple() -def target_databases(conn, step, trait, threshold): +def target_databases(conn, traits, threshold): """ Retrieves the names of possible target databases from the database. """ - if step != "select-corr-method": - return None - - trait_info = retrieve_trait_info( - threshold, f"{trait['dataset']}::{trait['name']}", conn) - group = trait_info["group"] + trait_info = traits_info( + conn, threshold, + tuple(f"{trait['dataset']}::{trait['trait_name']}" for trait in traits)) + groups = tuple(set(row["db"]["group"] for row in trait_info)) return ( - publish_target_dabases(conn, group, threshold) + - geno_target_databases(conn, group, threshold) + - probeset_target_databases(conn, group, threshold)) + publish_target_databases(conn, groups, threshold) + + geno_target_databases(conn, groups, threshold) + + probeset_target_databases(conn, groups, threshold)) + +def primary_error(args): + if len(args["primary_trait"]) == 0 or len(args["primary_trait"]) > 1: + return { + **args, + "errors": (args.get("errors", tuple()) + + ("You must provide one, and only one primary trait",))} + return args + +def controls_error(args): + if len(args["control_traits"]) == 0 or len(args["control_traits"]) > 3: + return { + **args, + "errors": ( + args.get("errors", tuple()) + + (("You must provide at least one control trait, and a maximum " + "of three control traits"),))} + return args + +def target_db_error(args): + if not args["target_db"]: + return { + **args, + "errors": ( + args.get("errors", tuple()) + + ("The target database must be provided",))} + return args + +def method_error(args): + methods = ( + "genetic correlation, pearson's r", + "genetic correlation, spearman's rho", + "sgo literature correlation", + "tissue correlation, pearson's r", + "tissue correlation, spearman's rho") + if not args["method"] or args["method"].lower() not in methods: + return { + **args, + "errors": ( + args.get("errors", tuple()) + + ("Invalid correlation method provided",))} + return args -def pcorrelations(conn, values): - if values["step"] != "run-correlation": - return None +def criteria_error(args): + try: + int(args.get("criteria", "invalid")) + return args + except ValueError: + return { + **args, + "errors": ( + args.get("errors", tuple()) + + ("Invalid return number provided",))} - def trait_fullname(trait): - return f"{trait['dataset']}::{trait['name']}" +def errors(args): + return criteria_error(method_error(target_db_error(controls_error( + primary_error(args))))) - return partial_correlations_entry( - conn, trait_fullname(values["primary_trait"]), - tuple(trait_fullname(trait) for trait in values["control_traits"]), - values["method"], values["criteria"], values["target_db"]) +def __classify_args(acc, item): + if item[1].startswith("primary_"): + return { + **acc, + "primary_trait": (acc.get("primary_trait", tuple()) + (item,))} + if item[1].startswith("controls_"): + return {**acc, "control_traits": (acc.get("control_traits", tuple()) + (item,))} + if item[0] == "target_db": + return {**acc, "target_db": item[1]} + if item[0] == "method": + return {**acc, "method": item[1]} + if item[0] == "criteria": + return {**acc, "criteria": item[1]} + return acc + +def __build_args(raw_form, traits): + args = reduce(__classify_args, raw_form.items(), {}) + return { + **args, + "primary_trait": [ + item for item in traits if item["trait_name"] in + (name[1][8:] for name in args["primary_trait"])], + "control_traits": [ + item for item in traits if item["trait_name"] in + (name[1][9:] for name in args["control_traits"])] + } + +def parse_trait(trait_str): + return dict(zip( + ("trait_name", "dataset", "description", "symbol", "location", "mean", + "lrs", "lrs_location"), + trait_str.strip().split("|||"))) + +def response_error_message(response): + error_messages = { + 404: ("We could not connect to the API server at this time. " + "Try again later."), + 500: ("The API server experienced a problem. We will be working on a " + "fix. Please try again later.") + } + return error_messages.get( + response.status_code, + "General API server error!!") + +def render_error(error_message): + return render_template( + "partial_correlations/pcorrs_error.html", + message = error_message) + +def handle_200_response(response): + if response["status"] == "success": + return redirect( + url_for( + "poll_partial_correlation_results", + command_id=response["results"]), + code=303) + return render_error(response["results"]) + +def handle_response(response): + if response.status_code != 200: + return render_template( + "partial_correlations/pcorrs_error.html", + message = response_error_message(response)) + return handle_200_response(response.json()) @app.route("/partial_correlations", methods=["POST"]) def partial_correlations(): form = request.form - traits_list = tuple(filter( - lambda trt: trt is not None, - (parse_trait(tstr) for tstr in form.get("traits_list", "").split("|||")))) + traits = tuple( + parse_trait(trait) for trait in + form.get("trait_list").split(";;;")) - args_dict = dict(zip( - ("step", "primary_trait", "control_traits", "target_traits", - "traits_list", "method"), - process_step( - form.get("step", None), - parse_trait(form.get("primary_trait", "")), - sequence_of_traits( - form.getlist("control_traits[]") or - form.get("control_traits", "").split("|||")), - sequence_of_traits( - form.getlist("target_traits[]") or - form.get("target_traits", "").split("|||")), - sequence_of_traits(form.get("traits_list", "").split("|||")), - form.get("method")))) + if form.get("submit") == "Run Partial Correlations": + args = errors(__build_args(form, traits)) + if len(args.get("errors", [])) == 0: + post_data = { + **args, + "primary_trait": args["primary_trait"][0] + } + return handle_response(requests.post( + url=f"{GN_SERVER_URL}api/correlation/partial", + json=json.dumps(post_data))) - conn = MySQLdb.Connect( - db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - target_dbs = target_databases( - conn, args_dict["step"], args_dict["primary_trait"], 0) + for error in args["errors"]: + flash(error, "alert-danger") - if args_dict["step"] == "run-correlation": - args_dict = { - **args_dict, "target_db": form.get("target_db"), - "criteria": int(form.get("criteria", 500))} + with database_connection() as conn: + target_dbs = target_databases(conn, traits, threshold=0) + return render_template( + "partial_correlations/pcorrs_select_operations.html", + trait_list_str=form.get("trait_list"), + traits=traits, + target_dbs=target_dbs) - corr_results = pcorrelations(conn, args_dict) +def process_pcorrs_command_output(result): + if result["status"] == "success": + def __format_number(num): + if num is None or math.isnan(num): + return "" + if abs(num) <= 1.04E-4: + return f"{num:.2e}" + return f"{num:.5f}" - return render_template( - "partial_correlations.html", **args_dict, target_dbs=target_dbs, - corr_results=corr_results, part_corr_url=f"{GN_SERVER_URL}api/correlation/partial") + return render_template( + "partial_correlations/pcorrs_results_presentation.html", + primary=result["results"]["primary_trait"], + controls=result["results"]["control_traits"], + correlations=result["results"]["correlations"], + dataset_type=result["results"]["dataset_type"], + method=result["results"]["method"], + format_number=__format_number) + if result["status"] == "error": + return render_error( + "The partial correlations computation failed with an error") + +@app.route("/partial_correlations/<command_id>", methods=["GET"]) +def poll_partial_correlation_results(command_id): + response = requests.get( + url=f"{GN_SERVER_URL}api/async_commands/state/{command_id}") + if response.status_code == 200: + data = response.json() + if data["status"] == "error": + return render_error(response["result"]) + if data["status"] == "success": + return process_pcorrs_command_output(json.loads(data["result"])) + return render_template( + "partial_correlations/pcorrs_poll_results.html", + command_id = command_id) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index cf2905c9..858ca56d 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -148,7 +148,7 @@ class SearchResultPage: trait_dict['name'] = trait_dict['display_name'] = str(result[0]) trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait_dict['name'], trait_dict['dataset'])) permissions = check_resource_availability(self.dataset, trait_dict['display_name']) - if "view" not in permissions['data']: + if not any(x in permissions['data'] for x in ["view", "edit"]): continue if result[10]: @@ -203,8 +203,8 @@ class SearchResultPage: for i, trait in enumerate(trait_list): for key in trait.keys(): if key == "authors": - authors_string = ",".join(str(trait[key]).split(",")[:6]) + ", et al." - self.max_widths[key] = max(len(authors_string), self.max_widths[key]) if key in self.max_widths else len(str(trait[key])) + authors_string = ",".join(str(trait[key]).split(",")[:2]) + ", et al." + self.max_widths[key] = max(len(authors_string), self.max_widths[key]) if key in self.max_widths else len(str(authors_string)) else: self.max_widths[key] = max(len(str(trait[key])), self.max_widths[key]) if key in self.max_widths else len(str(trait[key])) @@ -360,7 +360,8 @@ def get_aliases(symbol_list, species): filtered_aliases = [] response = requests.get( - GN2_BASE_URL + "/gn3/gene/aliases2/" + symbols_string) + GN2_BASE_URL + "gn3/gene/aliases/" + symbols_string) + if response: alias_lists = json.loads(response.content) seen = set() diff --git a/wqflask/wqflask/snp_browser/snp_browser.py b/wqflask/wqflask/snp_browser/snp_browser.py index c4d0e135..43bb55b5 100644 --- a/wqflask/wqflask/snp_browser/snp_browser.py +++ b/wqflask/wqflask/snp_browser/snp_browser.py @@ -3,12 +3,11 @@ from flask import Flask, g, url_for import string from PIL import (Image) -from utility.logger import getLogger -logger = getLogger(__name__) - from base import species from base import webqtlConfig +from wqflask.database import database_connection + class SnpBrowser: @@ -660,32 +659,27 @@ class SnpBrowser: x_scale = plot_width / (self.end_mb - self.start_mb) # draw clickable image map at some point - n_click = 80.0 click_step = plot_width / n_click click_mb_step = (self.end_mb - self.start_mb) / n_click - # for i in range(n_click): - # href = url_for('snp_browser', first_run="false", chosen_strains_mouse=self.chosen_strains_mouse, chosen_strains_rat=self.chosen_strains_rat, variant=self.variant_type, species=self.species_name, gene_name=self.gene_name, chr=self.chr, start_mb=self.start_mb, end_mb=self.end_mb, limit_strains=self.limit_strains, domain=self.domain, function=self.function, criteria=self.criteria, score=self.score, diff_alleles=self.diff_alleles) - def get_browser_sample_lists(species_id=1): strain_lists = {} mouse_strain_list = [] - query = "SHOW COLUMNS FROM SnpPattern;" - results = g.db.execute(query).fetchall() - for result in results[1:]: - mouse_strain_list.append(result[0]) - rat_strain_list = [] - query = "SHOW COLUMNS FROM RatSnpPattern;" - results = g.db.execute(query).fetchall() - for result in results[2:]: - rat_strain_list.append(result[0]) - - strain_lists['mouse'] = mouse_strain_list - strain_lists['rat'] = rat_strain_list - + with database_connection() as conn: + with conn.cursor() as cursor: + cursor.execute("SHOW COLUMNS FROM SnpPattern") + _mouse_snp_pattern = cursor.fetchall() + cursor.execute("SHOW COLUMNS FROM RatSnpPattern") + _rats_snp_pattern = cursor.fetchall() + for result in _mouse_snp_pattern[1:]: + mouse_strain_list.append(result[0]) + for result in _rats_snp_pattern[2:]: + rat_strain_list.append(result[0]) + strain_lists['mouse'] = mouse_strain_list + strain_lists['rat'] = rat_strain_list return strain_lists @@ -891,64 +885,51 @@ def get_effect_info(effect_list): def get_gene_id(species_id, gene_name): - query = """ - SELECT - geneId - FROM - GeneList - WHERE - SpeciesId = %s AND geneSymbol = '%s' - """ % (species_id, gene_name) - - result = g.db.execute(query).fetchone() - - if len(result) > 0: - return result - else: - return "" + query = ("SELECT geneId FROM GeneList WHERE " + "SpeciesId = %s AND geneSymbol = %s") + + with database_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query, (species_id, gene_name)) + if (result := cursor.fetchone()): + return result[0] + return "" def get_gene_id_name_dict(species_id, gene_name_list): gene_id_name_dict = {} if len(gene_name_list) == 0: return "" - gene_name_str_list = ["'" + gene_name + \ - "'" for gene_name in gene_name_list] - gene_name_str = ",".join(gene_name_str_list) - - query = """ - SELECT - geneId, geneSymbol - FROM - GeneList - WHERE - SpeciesId = %s AND geneSymbol in (%s) - """ % (species_id, gene_name_str) - - results = g.db.execute(query).fetchall() - - if len(results) > 0: - for item in results: - gene_id_name_dict[item[1]] = item[0] - + query = ("SELECT geneId, geneSymbol FROM " + "GeneList WHERE SpeciesId = %s AND " + f"geneSymbol in ({', '.join(['%s'] * len(gene_name_list))})") + with database_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query, (species_id, *gene_name_list)) + results = cursor.fetchall() + if results: + for item in results: + gene_id_name_dict[item[1]] = item[0] return gene_id_name_dict -def check_if_in_gene(species_id, chr, mb): - if species_id != 0: # ZS: Check if this is necessary - query = """SELECT geneId, geneSymbol - FROM GeneList - WHERE SpeciesId = {0} AND chromosome = '{1}' AND - (txStart < {2} AND txEnd > {2}); """.format(species_id, chr, mb) - else: - query = """SELECT geneId,geneSymbol - FROM GeneList - WHERE chromosome = '{0}' AND - (txStart < {1} AND txEnd > {1}); """.format(chr, mb) - - result = g.db.execute(query).fetchone() - - if result: - return [result[0], result[1]] - else: - return "" +def check_if_in_gene(species_id, chr_, mb): + with database_connection() as conn: + with conn.cursor() as cursor: + if species_id != 0: # ZS: Check if this is necessary + cursor.execute( + "SELECT geneId, geneSymbol " + "FROM GeneList WHERE " + "SpeciesId = %s AND chromosome = %s " + "AND (txStart < %s AND txEnd > %s)", + (species_id, chr_, mb, mb)) + else: + cursor.execute( + "SELECT geneId,geneSymbol " + "FROM GeneList WHERE " + "chromosome = %s AND " + "(txStart < %s AND txEnd > %s)", + (chr_, mb, mb)) + if (result := cursor.fetchone()): + return [result[0], result[1]] + return "" diff --git a/wqflask/wqflask/static/new/css/partial_correlations.css b/wqflask/wqflask/static/new/css/partial_correlations.css index 8ff5eae7..84a0877f 100644 --- a/wqflask/wqflask/static/new/css/partial_correlations.css +++ b/wqflask/wqflask/static/new/css/partial_correlations.css @@ -30,11 +30,6 @@ tr:nth-of-type(2n) { background: #F9F9F9; } -thead tr { - background: #336699; - line-height: 1.5em; -} - .with-trait { margin-left: 0.7em; position: relative; diff --git a/wqflask/wqflask/static/new/javascript/partial_correlations.js b/wqflask/wqflask/static/new/javascript/partial_correlations.js index b3a89c5e..5de1204c 100644 --- a/wqflask/wqflask/static/new/javascript/partial_correlations.js +++ b/wqflask/wqflask/static/new/javascript/partial_correlations.js @@ -1,300 +1,26 @@ -/** - * This is, hopefully, a short-term stop-gap measure to get the system working - * and to get some feedback, even as better optimisation is done in the - * background to get better response/performance for the partial correlation - * computations - */ - -function key_value(keys, values) { - if(!(keys.length == values.length)) { - Error("The 'keys' and 'values' objects MUST be the same length"); - return null; - } - return values.reduce(function(accumulator, item, index) { - accumulator[keys[index]] = item; - return accumulator; - }, {}); -} - -function trait(trait_str) { - return key_value( - ["name", "dataset", "symbol", "description", "location", "mean_expr", - "max_lrs", "data_hmac"], - trait_str.split(":::")); -} - -function primary_trait() { - trait_string = document.querySelector( - "#partial-correlations-form input[name=primary_trait]").value; - return trait(trait_string); -} - -function control_traits() { - return document.querySelector( - "#partial-correlations-form input[name=control_traits]").value.split( - "|||").map(trait).filter(trait => !(trait === null)); -} - -function correlation_method() { - return document.querySelector( - "#partial-correlations-form select[name=method]").value; -} - -function criteria() { - return document.querySelector( - "#partial-correlations-form select[name=criteria]").value; -} - -function target_db() { - return document.querySelector( - "#partial-correlations-form select[name=target_db]").value; -} - -function partial_corr_request_data() { - return { - "primary_trait": primary_trait(), - "control_traits": control_traits(), - "method": correlation_method(), - "criteria": criteria(), - "target_db": target_db() - } -} - -function rho_or_r(method) { - if (method === "spearman") { - return "rho"; - } - return "r"; -} - -function format_number(num) { - if(num === null) { - return NaN; - } - if(Math.abs(num) <= 1.04e-4) { - return num.toExponential(2); - } - return num.toFixed(5); -} - -function display_publish_results(primary, controls, correlations, method) { - table = document.getElementById("part-corr-results-publish"); - table.setAttribute("style", "display: block;"); - table_body = document.querySelector("#part-corr-results-publish tbody"); - template_row = document.querySelector( - "#part-corr-results-publish tr.template-publish-results-row"); - correlations.forEach(function(item, index, arr) { - new_row = template_row.cloneNode(true); - new_row.setAttribute("class", "results-row"); - new_row.querySelector( - 'td[data-column-heading="Record"]').innerHTML = item["trait_name"]; - new_row.querySelector( - 'td[data-column-heading="Phenotype"]').innerHTML = ( - item["post_publication_description"]); - new_row.querySelector( - 'td[data-column-heading="Authors"]').innerHTML = item["authors"]; - new_row.querySelector( - 'td[data-column-heading="Year"]').innerHTML = item["year"]; - new_row.querySelector( - 'td[data-column-heading="N"]').innerHTML = item["noverlap"]; - new_row.querySelector( - `td[data-column-heading="Partial ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["partial_corr"]); - new_row.querySelector( - `td[data-column-heading="p(partial ${rho_or_r(method)})"]` - ).innerHTML = format_number(item["partial_corr_p_value"]); - new_row.querySelector( - `td[data-column-heading="${rho_or_r(method)}"]` - ).innerHTML = format_number(item["corr"]); - new_row.querySelector( - `td[data-column-heading="p(${rho_or_r(method)})"]` - ).innerHTML = format_number(item["corr_p_value"]); - new_row.querySelector( - `td[data-column-heading="delta ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["delta"]); - table_body.appendChild(new_row); - }); - table_body.removeChild(template_row); -} - -function display_geno_results(primary, controls, correlations, method) { - table = document.getElementById("part-corr-results-geno"); - table.setAttribute("style", "display: block;"); - table_body = document.querySelector("#part-corr-results-geno tbody"); - template_row = document.querySelector( - "#part-corr-results-geno tr.template-geno-results-row"); - correlations.forEach(function(item, index, arr) { - new_row = template_row.cloneNode(true); - new_row.setAttribute("class", "results-row"); - new_row.querySelector( - 'td[data-column-heading="Locus"]').innerHTML = item["trait_name"]; - new_row.querySelector( - 'td[data-column-heading="Chr"]').innerHTML = item["chr"]; - new_row.querySelector( - 'td[data-column-heading="Megabase"]').innerHTML = item["mb"]; - new_row.querySelector( - 'td[data-column-heading="N"]').innerHTML = item["noverlap"]; - new_row.querySelector( - `td[data-column-heading="Partial ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["partial_corr"]); - new_row.querySelector( - `td[data-column-heading="p(partial ${rho_or_r(method)})"]` - ).innerHTML = format_number(item["partial_corr_p_value"]); - new_row.querySelector( - `td[data-column-heading="${rho_or_r(method)}"]` - ).innerHTML = format_number(item["corr"]); - new_row.querySelector( - `td[data-column-heading="p(${rho_or_r(method)})"]` - ).innerHTML = format_number(item["corr_p_value"]); - new_row.querySelector( - `td[data-column-heading="delta ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["delta"]); - table_body.appendChild(new_row); - }); - table_body.removeChild(template_row); -} - -function display_probeset_results(primary, controls, correlations, method) { - table = document.getElementById("part-corr-results-probeset"); - table.setAttribute("style", "display: block;"); - table_body = document.querySelector("#part-corr-results-probeset tbody"); - template_row = document.querySelector( - "#part-corr-results-probeset tr.template-probeset-results-row"); - correlations.forEach(function(item, index, arr) { - new_row = template_row.cloneNode(true); - new_row.setAttribute("class", "results-row"); - new_row.querySelector( - 'td[data-column-heading="Record"]').innerHTML = item["trait_name"]; - new_row.querySelector( - 'td[data-column-heading="Gene ID"]').innerHTML = item["geneid"]; - new_row.querySelector( - 'td[data-column-heading="Homologene ID"]').innerHTML = item["homologeneid"]; - new_row.querySelector( - 'td[data-column-heading="Symbol"]').innerHTML = item["symbol"]; - new_row.querySelector( - 'td[data-column-heading="Description"]').innerHTML = item["description"]; - new_row.querySelector( - 'td[data-column-heading="Chr"]').innerHTML = item["chr"]; - new_row.querySelector( - 'td[data-column-heading="Megabase"]').innerHTML = item["mb"]; - new_row.querySelector( - 'td[data-column-heading="Mean Expr"]').innerHTML = item["mean_expr"]; - new_row.querySelector( - 'td[data-column-heading="N"]').innerHTML = item["noverlap"]; - new_row.querySelector( - `td[data-column-heading="Sample Partial ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["partial_corr"] || NaN); - new_row.querySelector( - `td[data-column-heading="Sample p(partial ${rho_or_r(method)})"]` - ).innerHTML = format_number(item["partial_corr_p_value"] || NaN); - new_row.querySelector( - `td[data-column-heading="Sample ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["corr"] || NaN); - new_row.querySelector( - `td[data-column-heading="Sample p(${rho_or_r(method)})"]` - ).innerHTML = format_number(item["corr_p_value"] || NaN); - new_row.querySelector( - `td[data-column-heading="delta ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["delta"] || NaN); - new_row.querySelector( - `td[data-column-heading="Lit Corr"]` - ).innerHTML = format_number(item["l_corr"] || NaN); - new_row.querySelector( - `td[data-column-heading="Tissue ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["tissue_corr"] || NaN); - new_row.querySelector( - `td[data-column-heading="Tissue p(${rho_or_r(method)})"]` - ).innerHTML = format_number(item["tissue_p_value"] || NaN); - table_body.appendChild(new_row); - }); - template_row.setAttribute("display", "none"); - /*table_body.removeChild(template_row);*/ -} - -function replace_r_with_rho(method) { - /* Mostly utility: Replace `r` with `rho` in the appropriate places */ - pattern = /\br\b/; - if(method == "spearman") { - results_div = document.getElementById("partial-correlation-results"); - headers = results_div.getElementsByTagName("th"); - for(let header of headers) { - header.innerHTML = header.innerHTML.replace(pattern, "rho"); - } - - cells = results_div.getElementsByTagName("td"); - for(let cell of cells) { - cell.setAttribute( - "data-column-heading", - cell.getAttribute( - "data-column-heading").replace(pattern, "rho")); - } - } -} - -function display_partial_corr_results(data, status, xhr) { - progress_indicator = document.getElementById( - "partial-correlations-progress-indicator").style.display = "none"; - console.log(data); - - replace_r_with_rho(data["results"]["method"]); - - display_functions = { - "Publish": display_publish_results, - "Geno": display_geno_results, - "ProbeSet": display_probeset_results +function selected_traits() { + traits = $("#trait_table input:checked").map(function() { + return $(this).attr("data-trait-info"); + }).get(); + if (traits.length == 0){ + num_traits = $("#trait_table input").length + if (num_traits <= 100){ + traits = $("#trait_table input").map(function() { + return $(this).attr("data-trait-info"); + }).get(); + } } - - display_functions[data["results"]["dataset_type"]]( - data["results"]["primary_traits"], - data["results"]["control_traits"], - data["results"]["correlations"], - data["results"]["method"]); -} - -function display_partial_corr_error(xhr, status, error) { - document.getElementById( - "partial-correlations-progress-indicator").style.display = "none"; - error_element = document.getElementById("part-corr-error"); - panel = document.createElement("div"); - panel.setAttribute("class", "panel panel-danger"); - error_element.appendChild(panel); - - panel_header = document.createElement("div"); - panel_header.setAttribute("class", "panel-heading"); - panel_header.textContent = "Error: " + xhr.status; - panel.appendChild(panel_header); - - panel_body = document.createElement("div"); - panel_body.setAttribute("class", "panel-body"); - panel_body.textContent = xhr.statusText; - panel.appendChild(panel_body); - console.log(xhr) -} - -function send_data_and_process_results( - remote_url, request_data, success_fn, error_fn, indicator_id) { - document.getElementById(indicator_id).style.display = "block"; - $.ajax({ - type: "POST", - url: remote_url, - contentType: "application/json", - data: JSON.stringify(request_data), - dataType: "JSON", - success: success_fn, - error: error_fn - }); -} - -$("#partial-correlations-form").submit(function(e) { - e.preventDefault(); -}); - -$("#run-partial-corr-button").click(function(evt) { - send_data_and_process_results( - document.getElementById( - "run-partial-corr-button").getAttribute("data-url"), - partial_corr_request_data(), - display_partial_corr_results, - display_partial_corr_error, - "partial-correlations-progress-indicator"); + return traits +} + +$("#partial-correlations").on("click", function() { + // Submit the form to the `partial_correlations` endpoint + url = $(this).data("url") + traits = selected_traits(); + $("#trait_list").val(traits.reduce(function(acc, str) { + return acc.concat(";;;".concat(str)); + })); + $("input[name=tool_used]").val("Partial Correlation") + $("input[name=form_url]").val(url) + return submit_special(url) }) diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index 432393a7..6f1a9680 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -33,48 +33,45 @@ </form> </div> - - <form id="partial-correlations-form" - method="POST" - action="{{url_for('partial_correlations')}}"> - <input type="hidden" - id="collection_uuid" - value="{{uc.uc_id}}" /> - <input type="hidden" - name="traits_list" - value="{% for this_trait in trait_obs %}{{ this_trait.name }}:::{{ this_trait.dataset.name }}:::{{this_trait.symbol}}:::{{this_trait.description_display}}:::{{this_trait.location_repr}}:::{{this_trait.mean}}:::{{this_trait['LRS_location_repr']}}:::{{data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset.name))}}|||{% endfor %}" /> - <button id="run-partial-correlations" - class="btn btn-primary" - title="Run partial correlations" - type="submit"> - Partial Correlations - </button> - </form> + <div style="display: flex;"> + <form id="heatmaps_form"> + <button id="clustered-heatmap" + class="btn btn-primary" + data-url="{{heatmap_data_url}}" + title="Generate heatmap from this collection" style="margin-top: 10px; margin-bottom: 10px;"> + Generate Heatmap + </button> + <br> + <div id="heatmap-options" style="display: none;"> + <div style="margin-bottom: 10px;"> + <b>Heatmap Orientation: </b> + <br> + Vertical + <input id="heatmap-orient-vertical" + type="radio" + name="vertical" + value="true" checked="checked"/> + Horizontal + <input id="heatmap-orient-horizontal" + type="radio" + name="vertical" + value="false" /> + </div> + <div style="margin-bottom: 10px;"> + <button id="clear-heatmap" + class="btn btn-danger" + title="Clear Heatmap"> + Clear Heatmap + </button> + </div> + </div> + </form> + + </div> <div> + <div id="clustered-heatmap-image-area"></div> <br /> - <form id="heatmaps_form"> - <fieldset> - <legend>Heatmap Orientation</legend> - <label for="heatmap-orient-vertical">Vertical</label> - <input id="heatmap-orient-vertical" - type="radio" - name="vertical" - value="true" /> - <label for="heatmap-orient-horizontal">Horizontal</label> - <input id="heatmap-orient-horizontal" - type="radio" - name="vertical" - value="false" /> - </fieldset> - <button id="clustered-heatmap" - class="btn btn-primary" - data-url="{{heatmap_data_url}}" - title="Generate heatmap from this collection"> - Generate Heatmap - </button> - </form> - <div class="collection-table-options"> <form id="export_form" method="POST" action="/export_traits_csv"> <button class="btn btn-default" id="select_all" type="button"><span class="glyphicon glyphicon-ok"></span> Select All</button> @@ -91,8 +88,6 @@ <button id="delete" class="btn btn-danger submit_special" data-url="/collections/delete" type="button" title="Delete this collection" > Delete Collection</button> </form> </div> - <div id="clustered-heatmap-image-area"> - </div> <div style="margin-top: 10px; margin-bottom: 5px;"> <b>Show/Hide Columns:</b> </div> @@ -117,7 +112,13 @@ <tbody> {% for this_trait in trait_obs %} <TR id="trait:{{ this_trait.name }}:{{ this_trait.dataset.name }}"> - <TD align="center" style="padding: 0px;"><INPUT TYPE="checkbox" NAME="searchResult" class="checkbox trait_checkbox" VALUE="{{ data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset.name)) }}"></TD> + <TD align="center" style="padding: 0px;"> + <input type="checkbox" + name="searchResult" + class="checkbox trait_checkbox" + value="{{data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset.name))}}" + data-trait-info="{{trait_info_str(this_trait)}}"> + </TD> <TD data-export="{{ loop.index }}" align="right">{{ loop.index }}</TD> <TD title="{{ this_trait.dataset.fullname }}" data-export="{{ this_trait.dataset.fullname }}">{{ this_trait.dataset.fullname }}</TD> <TD data-export="{{ this_trait.name }}"> @@ -182,6 +183,9 @@ <script type="text/javascript" src="{{ url_for('js', filename='plotly/plotly.min.js') }}"></script> + <script type="text/javascript" + src="/static/new/javascript/partial_correlations.js"></script> + <script language="javascript" type="text/javascript"> $(document).ready( function () { @@ -339,6 +343,7 @@ $("#clustered-heatmap").on("click", function() { clear_heatmap_area(); + $("#heatmap-options").show(); intv = window.setInterval(generate_progress_indicator(), 300); vert_element = document.getElementById("heatmap-orient-vertical"); vert_true = vert_element == null ? false : vert_element.checked; @@ -365,6 +370,12 @@ } }); }); + + $("#clear-heatmap").on("click", function() { + clear_heatmap_area(); + $("#heatmap-options").hide(); + }); + }); </script> diff --git a/wqflask/wqflask/templates/display_diffs.html b/wqflask/wqflask/templates/display_diffs.html index e787e468..ce50c1b4 100644 --- a/wqflask/wqflask/templates/display_diffs.html +++ b/wqflask/wqflask/templates/display_diffs.html @@ -11,14 +11,14 @@ {% set additions = diff.get("Additions") %} {% set modifications = diff.get("Modifications") %} {% set deletions = diff.get("Deletions") %} - + {% set header = diff.get("Columns", "Strain Name,Value,SE,Count") %} {% if additions %} <h2>Additions Data:</h2> <div class="row"> <div class="col-md-8"> <table class="table-responsive table-hover table-striped cell-border" id="table-additions"> <thead> - <th scope="col">Added Data</</th> + <th scope="col">Added Data ({{ header }})</th> </thead> <tbody> {% for data in additions %} @@ -39,9 +39,9 @@ <div class="col-md-8"> <table class="table-responsive table-hover table-striped cell-border" id="table-modifications"> <thead> - <th scope="col">Original</</th> - <th scope="col">Current</</th> - <th scope="col">Diff</</th> + <th scope="col">Original</th> + <th scope="col">Current</th> + <th scope="col">Diff ({{ header }})</th> </thead> <tbody> {% for data in modifications %} diff --git a/wqflask/wqflask/templates/edit_phenotype.html b/wqflask/wqflask/templates/edit_phenotype.html index 0daea51d..5458247e 100644 --- a/wqflask/wqflask/templates/edit_phenotype.html +++ b/wqflask/wqflask/templates/edit_phenotype.html @@ -218,12 +218,17 @@ </div> <div style="margin-left: 13%;"> <a href="/datasets/{{ publish_xref.id_ }}/traits/{{ publish_xref.phenotype_id }}/csv?resource-id={{ resource_id }}" class="btn btn-link btn-sm"> - Sample Data(CSV Download) + Click to Download CSV Sample Data </a> </div> <div class="form-group"> <input type = "file" class="col-sm-4 control-label" name = "file" /> </div> + <div class="col-xs-6"> + <p> + Note: Current allowable sample headers are: {{ ', '.join(headers) }} + </p> + </div> <div class="controls center-block" style="width: max-content;"> <input name="inbred-set-id" class="changed" type="hidden" value="{{ publish_xref.inbred_set_id }}"/> <input name="phenotype-id" class="changed" type="hidden" value="{{ publish_xref.phenotype_id }}"/> diff --git a/wqflask/wqflask/templates/partial_correlations.html b/wqflask/wqflask/templates/partial_correlations.html deleted file mode 100644 index 4be4890e..00000000 --- a/wqflask/wqflask/templates/partial_correlations.html +++ /dev/null @@ -1,341 +0,0 @@ -{%extends "base.html"%} - -{%block title%}Partial Correlations:{%endblock%} - -{%block css%} -<link rel="stylesheet" type="text/css" href="/static/new/css/partial_correlations.css" /> -{%endblock%} - -{%block content%} -<form id="partial-correlations-form" - method="POST" - action="{{url_for('partial_correlations')}}"> - - <div id="main-form"> - {%with messages = get_flashed_messages(with_categories=true)%} - {%if messages:%} - <ul class=flashes> - {%for category, message in messages:%} - <li class="{{category}}">{{message}}</li> - {%endfor%} - </ul> - {%endif%} - {%endwith%} - - {%if step == "select-primary":%} - <p>Please select the primary trait (X)</p> - {%include "with-trait-items.html" %} - - <button type="submit" class="btn btn-primary"> - Next: Select Control Traits - </button> - {%endif%} - - - - {%if step == "select-controls":%} - - <p>Select a maximum of three (3) control traits (Z)</p> - {%include "with-trait-items.html" %} - - <button type="submit" class="btn btn-primary"> - Next: Select Target - </button> - {%endif%} - - - - {%if step == "select-targets":%} - <p>Select at least one target trait (Y)</p> - {%for trait in traits_list:%} - <div class="label-element-combo"> - <input type="checkbox" - name="target_traits[]" - value="{{trait['name']}}:::{{trait['dataset']}}:::{{trait['symbol']}}:::{{trait['description']}}:::{{trait['data_hmac']}}" - checked="checked" - id="trait_{{trait['data_hmac']}}" /> - <label for="trait_{{trait['data_hmac']}}"> - {{trait["name"]}} - {{trait["symbol"]}} - {{trait["description"]}} - </label> - </div> - {%endfor%} - <button type="submit" class="btn btn-primary"> - Next: Select Correlation Method - </button> - {%endif%} - - - - {%if step == "select-corr-method":%} - <div class="label-element-combo"> - <label for="target-db-input">Choose Database</label> - <select id="target-db-input" required="required" name="target_db"> - {%if target_dbs:%} - {%for item in target_dbs:%} - {%if "description" in item.keys():%} - <option value="{{item['value']}}">{{item['description']}}</option> - {%else:%} - {%for group, opts in item.items()%} - {%if opts | length > 0:%} - <optgroup label="{{group}} ------"> - {%for item2 in opts:%} - <option value="{{item2['value']}}">{{item2['description']}}</option> - {%endfor%} - </optgroup> - {%endif%} - {%endfor%} - {%endif%} - {%endfor%} - {%endif%} - </select> - </div> - - <div class="label-element-combo"> - <label for="corr-method-input">Compute</label> - <select id="corr-method-input" required="required" name="method"> - <option value="Genetic Correlation, Pearson's r"> - Genetic Correlation, Pearson's r</option> - <option value="Genetic Correlation, Spearman's rho"> - Genetic Correlation, Spearman's rho</option> - <option value="SGO Literature Correlation"> - SGO Literature Correlation</option> - <option value="Tissue Correlation, Pearson's r"> - Tissue Correlation, Pearson's r</option> - <option value="Tissue Correlation, Spearman's rho"> - Tissue Correlation, Spearman's rho</option> - </select> - </div> - - <div class="label-element-combo"> - <label for="criteria-input">Return</label> - <select id="criteria-input" required="required" name="criteria" size="1"> - <option value="100">top 100</option> - <option value="200">top 200</option> - <option value="500" selected="selected">top 500</option> - <option value="1000">top 1000</option> - <option value="2000">top 2000</option> - <option value="5000">top 5000</option> - <option value="10000">top 10000</option> - <option value="15000">top 15000</option> - <option value="20000">top 20000</option> - </select> - </div> - - <button id="run-partial-corr-button" - type="submit" - class="btn btn-primary" - data-url="{{part_corr_url}}"> - Run Partial Correlation - </button> - {%endif%} - - {%if step == "run-correlation":%} - <input type="hidden" name="selected_method" value="{{method}}" /> - <input type="hidden" name="selected_target_db" value="{{target_db}}" /> - <input type="hidden" name="selected_criteria" value="{{criteria}}" /> - - {{corr_results}} - {%endif%} - </div> - - <div id="form-display-area"> - <input type="hidden" name="step" id="step-indicator" value="{{step}}" /> - <input type="hidden" - name="traits_list" - value="{% for trait in traits_list %}{{trait['name']}}:::{{trait['dataset']}}:::{{trait['symbol']}}:::{{trait['description']}}:::{{trait['location']}}:::{{trait['mean_expr']}}:::{{trait['max_lrs']}}:::{{trait['data_hmac']}}|||{% endfor %}"> - - {%if primary_trait:%} - <input type="hidden" - name="primary_trait" - value="{{primary_trait['name']}}:::{{primary_trait['dataset']}}:::{{primary_trait['symbol']}}:::{{primary_trait['description']}}:::{{primary_trait['location']}}:::{{primary_trait['mean_expr']}}:::{{primary_trait['max_lrs']}}:::{{primary_trait['data_hmac']}}" - id="trait_{{primary_trait['data_hmac']}}" /> - - <div class="panel panel-info"> - <div class="panel-heading">Primary Trait (X)</div> - <div class="panel-body"> - {{primary_trait["name"]}} - - {{primary_trait["symbol"]}} - {{primary_trait["description"]}} - </div> - </div> - {%endif%} - - {%if target_traits:%} - <input type="hidden" - name="target_traits" - value="{%for trait in target_traits:%}{{trait['name']}}:::{{trait['dataset']}}:::{{trait['symbol']}}:::{{trait['description']}}:::{{trait['location']}}:::{{trait['mean_expr']}}:::{{trait['max_lrs']}}:::{{trait['data_hmac']}}|||{%endfor%}" /> - <div class="panel panel-info"> - <div class="panel-heading">Target Traits (Y)</div> - <div class="panel-body"> - <ul> - {%for trait in target_traits:%} - <li> - {{trait["name"]}} - {{trait["symbol"]}} - - {{trait["description"]}}</li> - {%endfor%} - </ul> - </div> - </div> - {%endif%} - - {%if control_traits:%} - <input type="hidden" - name="control_traits" - value="{%for trait in control_traits:%}{{trait['name']}}:::{{trait['dataset']}}:::{{trait['symbol']}}:::{{trait['description']}}:::{{trait['location']}}:::{{trait['mean_expr']}}:::{{trait['max_lrs']}}:::{{trait['data_hmac']}}|||{%endfor%}" /> - <div class="panel panel-info"> - <div class="panel-heading">Control Traits (Z)</div> - <div class="panel-body"> - <ul> - {%for trait in control_traits:%} - <li> - {{trait["name"]}} - {{trait["symbol"]}} - - {{trait["description"]}}</li> - {%endfor%} - </ul> - </div> - </div> - {%endif%} - </div> - -</form> - -<div id="partial-correlation-results"> - <span id="partial-correlations-progress-indicator" - style="display: {%if step == 'run-correlation':%}block{%else:%}none{%endif%};"> - <img src="/static/gif/waitAnima2.gif"> - </span> - <div id="part-corr-success"> - <table id="part-corr-results-publish" style="display:none;"> - <thead> - <tr> - <th>_</th> - <th>Record</th> - <th>Phenotype</th> - <th>Authors</th> - <th>Year</th> - <th>N</th> - <th>Partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> - <th>p(partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th> - <th>{%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> - <th>p({%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th> - <th>delta {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> - </tr> - </thead> - - <tbody> - <tr class="template-publish-results-row"> - <td data-column-heading="_"></td> - <td data-column-heading="Record"></td> - <td data-column-heading="Phenotype"></td> - <td data-column-heading="Authors"></td> - <td data-column-heading="Year"></td> - <td data-column-heading="N"></td> - <td data-column-heading="Partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> - </td> - <td data-column-heading="p(partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})"> - </td> - <td data-column-heading="{%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> - </td> - <td data-column-heading="p({%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})"> - </td> - <td data-column-heading="delta {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> - </td> - </tr> - </tbody> - </table> - - <table id="part-corr-results-geno" style="display:none;"> - <thead> - <tr> - <th>_</th> - <th>Locus</th> - <th>Chr</th> - <th>Megabase</th> - <th>N</th> - <th>Partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> - <th>p(partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th> - <th>{%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> - <th>p({%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th> - <th>delta {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> - </tr> - </thead> - - <tbody> - <tr class="template-geno-results-row"> - <td data-column-heading="_"></td> - <td data-column-heading="Locus"></td> - <td data-column-heading="Chr"></td> - <td data-column-heading="Megabase"></td> - <td data-column-heading="N"></td> - <td data-column-heading="Partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> - </td> - <td data-column-heading="p(partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})"> - </td> - <td data-column-heading="{%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> - </td> - <td data-column-heading="p({%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})"> - </td> - <td data-column-heading="delta {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> - </td> - </tr> - </tbody> - </table> - - <table id="part-corr-results-probeset" style="display:none;"> - <thead> - <tr> - <th>_</th> - <th>Record</th> - <th>Gene ID</th> - <th>Homologene ID</th> - <th>Symbol</th> - <th>Description</th> - <th>Chr</th> - <th>Megabase</th> - <th>Mean Expr</th> - <th>N</th> - <th>Sample Partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> - <th>Sample p(partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th> - <th>Sample {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> - <th>Sample p({%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th> - <th>delta {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> - <th>Lit Corr</th> - <th>Tissue {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> - <th>Tissue p({%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th> - </tr> - </thead> - - <tbody> - <tr class="template-probeset-results-row"> - <td data-column-heading="_"></td> - <td data-column-heading="Record"></td> - <td data-column-heading="Gene ID"></td> - <td data-column-heading="Homologene ID"></td> - <td data-column-heading="Symbol"></td> - <td data-column-heading="Description"></td> - <td data-column-heading="Chr"></td> - <td data-column-heading="Megabase"></td> - <td data-column-heading="Mean Expr"></td> - <td data-column-heading="N"></td> - <td data-column-heading="Sample Partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"></td> - <td data-column-heading="Sample p(partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})"></td> - <td data-column-heading="Sample {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"></td> - <td data-column-heading="Sample p({%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})"></td> - <td data-column-heading="delta {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"></td> - <td data-column-heading="Lit Corr"></td> - <td data-column-heading="Tissue {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"></td> - <td data-column-heading="Tissue p({%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})"></td> - </tr> - </tbody> - </table> - </div> - <div id="part-corr-error"> - </div> -</div> -{%endblock%} - -{%block js%} -{%if step == "select-corr-method":%} -<script type="text/javascript" - src="/static/new/javascript/partial_correlations.js"></script> -{%endif%} -{%endblock%} diff --git a/wqflask/wqflask/templates/partial_correlations/pcorrs_error.html b/wqflask/wqflask/templates/partial_correlations/pcorrs_error.html new file mode 100644 index 00000000..36847f57 --- /dev/null +++ b/wqflask/wqflask/templates/partial_correlations/pcorrs_error.html @@ -0,0 +1,54 @@ +{% extends "base.html" %} +{% block title %}Error: {{message}}{% endblock %} +{% block content %} +<!-- Start of body --> + +<div class="col-md-8"> +<div class="form-group has-error"> + <div class="control-label" for="inputError1"> + + <img src="/static/gif/error/{{ error_image }}"> + + <h1>ERROR</h1> + + <p> + This error is not what we wanted to see. Unfortunately errors + are part of all software systems and we need to resolve this + together. + </p> + <p> + <b>It is important to report this ERROR so we can fix it for everyone</b>. + </p> + + <p> + Report to the GeneNetwork team by recording the steps you take + to reproduce this ERROR. Next to those steps, copy-paste below + stack trace, either as + a <a href="https://github.com/genenetwork/genenetwork2/issues/new">new + issue</a> or E-mail this full page to one of the developers + directly. + </p> + </div> + + <p> + (GeneNetwork error: {{message[:128]}}) + </p> + + <p> + To check if this already a known issue, search the + <a href="https://github.com/genenetwork/genenetwork2/issues">issue + tracker</a>. + </p> + + <a href="#Stack" class="btn btn-default" data-toggle="collapse">Toggle full stack trace</a> + <div id="Stack" class="collapse"> + <pre> + GeneNetwork {{ version }} {% for line in stack %} {{ line }} + {% endfor %} + </pre> + </div> +</div> +</div> + + +{% endblock %} diff --git a/wqflask/wqflask/templates/partial_correlations/pcorrs_poll_results.html b/wqflask/wqflask/templates/partial_correlations/pcorrs_poll_results.html new file mode 100644 index 00000000..47acc294 --- /dev/null +++ b/wqflask/wqflask/templates/partial_correlations/pcorrs_poll_results.html @@ -0,0 +1,15 @@ +{%extends "base.html"%} + +{%block title%}Partial Correlations:{%endblock%} + +{%block css%} +<meta http-equiv="refresh" + content="10;URL=/partial_correlations/{{command_id}}"> +{%endblock%} + +{%block content%} + +<p>Computing partial correlations...</p> +<img src="/static/gif/waitAnima2.gif" + alt="Image indicating computation of partial correlations is ongoing" /> +{%endblock%} diff --git a/wqflask/wqflask/templates/partial_correlations/pcorrs_results_presentation.html b/wqflask/wqflask/templates/partial_correlations/pcorrs_results_presentation.html new file mode 100644 index 00000000..c109f24b --- /dev/null +++ b/wqflask/wqflask/templates/partial_correlations/pcorrs_results_presentation.html @@ -0,0 +1,249 @@ +{%extends "base.html"%} + +{%block title%}Partial Correlations:{%endblock%} + +{%block css%} +<link rel="stylesheet" type="text/css" href="/static/new/css/partial_correlations.css" /> +<link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTables/css/jquery.dataTables.css') }}" /> +<link rel="stylesheet" type="text/css" href="/static/new/css/trait_list.css" /> +<link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" /> +{%endblock%} + +{%block content%} +<p> + <strong>Primary Trait</strong><br /><br /> + <a href="{{url_for( + 'show_trait_page', + trait_id=primary['trait_name'], + dataset=primary['dataset_name'])}}" + title="Link to trait data for trait {{primary['trait_name']}}"> + {{primary["dataset_type"]}}/{{primary["trait_name"]}} + [{{primary["symbol"] }} on Chr {{primary["chr"]}} @ {{primary["mb"]}}]: + {{primary["description"]}} + </a> --- FROM: {{primary["dataset_name"]}} +</p> +<p><strong>Control Traits</strong><br /><br /> + {%for trait in controls:%} + <a href="{{url_for( + 'show_trait_page', + trait_id=trait['trait_name'], + dataset=trait['dataset_name'])}}" + title="Link to trait data for trait {{trait['trait_name']}}"> + {{trait["dataset_type"]}}/{{trait["trait_name"]}} + [{{trait["symbol"] }} on Chr {{trait["chr"]}} @ {{trait["mb"]}}]: + {{trait["description"]}} + </a> --- FROM: {{trait["dataset_name"]}}<br /> + {%endfor%} +</p> + +<div id="partial-correlation-results"> + {%if dataset_type == "Publish":%} + <table id="part-corr-results-publish" + class="table-hover table-striped cell-border" + style="float: left;"> + <thead> + <tr> + <th></th> + <th>Index</th> + <th>Record</th> + <th>Phenotype</th> + <th>Authors</th> + <th>Year</th> + <th>N</th> + <th>Partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> + <th>p(partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th> + <th>{%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> + <th>p({%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th> + <th>delta {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> + </tr> + </thead> + + <tbody> + {%for trait in correlations:%} + <tr class="results-row"> + <td></td> + <td data-column-heading="Index"></td> + <td data-column-heading="Record"> + <a href="{{url_for( + 'show_trait_page', + trait_id=trait['trait_name'], + dataset=trait['dataset_name'])}}" + title="Link to trait data for trait {{trait['trait_name']}}"> + {{trait["trait_name"]}} + </a> + </td> + <td data-column-heading="Phenotype"> + {{trait["post_publication_description"]}}</td> + <td data-column-heading="Authors">{{trait["authors"]}}</td> + <td data-column-heading="Year">{{trait["year"]}}</td> + <td data-column-heading="N">{{trait["noverlap"]}}</td> + <td data-column-heading="Partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> + {{format_number(trait.get("partial_corr"))}} + </td> + <td data-column-heading="p(partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})"> + {{format_number(trait.get("partial_corr_p_value"))}} + </td> + <td data-column-heading="{%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> + {{format_number(trait.get("corr"))}} + </td> + <td data-column-heading="p({%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})"> + {{format_number(trait.get("corr_p_value"))}} + </td> + <td data-column-heading="delta {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> + {{format_number(trait.get("delta"))}} + </td> + </tr> + {%endfor%} + </tbody> + </table> + {%endif%} + + {%if dataset_type == "Geno":%} + <table id="part-corr-results-geno" + class="table-hover table-striped cell-border" + style="float: left;"> + <thead> + <tr> + <th></th> + <th>Index</th> + <th>Locus</th> + <th>Chr</th> + <th>Megabase</th> + <th>N</th> + <th>Partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> + <th>p(partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th> + <th>{%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> + <th>p({%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th> + <th>delta {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> + </tr> + </thead> + + <tbody> + {%for trait in correlations:%} + <tr class="results-row"> + <td></td> + <td data-column-heading="Index"></td> + <td data-column-heading="Locus"> + <a href="{{url_for( + 'show_trait_page', + trait_id=trait['trait_name'], + dataset=trait['dataset_name'])}}" + title="Link to trait data for trait {{trait['trait_name']}}"> + {{trait["trait_name"]}} + </a> + </td> + <td data-column-heading="Chr">{{trait["chr"]}}</td> + <td data-column-heading="Megabase">{{trait["mb"]}}</td> + <td data-column-heading="N">{{trait["noverlap"]}}</td> + <td data-column-heading="Partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> + {{format_number(trait.get("partial_corr"))}} + </td> + <td data-column-heading="p(partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})"> + {{format_number(trait.get("partial_corr_p_value"))}} + </td> + <td data-column-heading="{%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> + {{format_number(trait.get("corr"))}} + </td> + <td data-column-heading="p({%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})"> + {{format_number(trait.get("corr_p_value"))}} + </td> + <td data-column-heading="delta {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> + {{format_number(trait.get("delta"))}} + </td> + </tr> + {%endfor%} + </tbody> + </table> + {%endif%} + + {%if dataset_type == "ProbeSet":%} + <table id="part-corr-results-probeset" + class="table-hover table-striped cell-border" + style="float: left;"> + <thead> + <tr> + <th></th> + <th>Index</th> + <th>Record</th> + <th>Gene ID</th> + <th>Homologene ID</th> + <th>Symbol</th> + <th>Description</th> + <th>Chr</th> + <th>Megabase</th> + <th>Mean Expr</th> + <th>N</th> + <th>Sample Partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> + <th>Sample p(partial {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th> + <th>Sample {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> + <th>Sample p({%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th> + <th>delta {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> + <th>Lit Corr</th> + <th>Tissue {%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%}</th> + <th>Tissue p({%if "spearman" in (method | lower):%}rho{%else:%}r{%endif%})</th> + </tr> + </thead> + + <tbody> + {%for trait in correlations:%} + <tr class="results-row"> + <td></td> + <td data-column-heading="Index"></td> + <td data-column-heading="Record"> + <a href="{{url_for( + 'show_trait_page', + trait_id=trait['trait_name'], + dataset=trait['dataset_name'])}}" + title="Link to trait data for trait {{trait['trait_name']}}"> + {{trait["trait_name"]}} + </a> + </td> + <td data-column-heading="Gene ID">{{trait["geneid"]}}</td> + <td data-column-heading="Homologene ID">{{trait["homologeneid"]}}</td> + <td data-column-heading="Symbol">{{trait["symbol"]}}</td> + <td data-column-heading="Description">{{trait["description"]}}</td> + <td data-column-heading="Chr">{{trait["chr"]}}</td> + <td data-column-heading="Megabase">{{trait["mb"]}}</td> + <td data-column-heading="Mean Expr">{{trait["mean_expr"]}}</td> + <td data-column-heading="N">{{trait["noverlap"]}}</td> + <td data-column-heading="Sample Partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> + {{format_number(trait.get("partial_corr"))}} + </td> + <td data-column-heading="Sample p(partial {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})"> + {{format_number(trait.get("partial_corr_p_value"))}} + </td> + <td data-column-heading="Sample {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> + {{format_number(trait.get("corr"))}} + </td> + <td data-column-heading="Sample p({%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})"> + {{format_number(trait.get("corr_p_value"))}} + </td> + <td data-column-heading="delta {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> + {{format_number(trait.get("delta"))}} + </td> + <td data-column-heading="Lit Corr"> + {{format_number(trait.get("l_corr"))}} + </td> + <td data-column-heading="Tissue {%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%}"> + {{format_number(trait.get("tissue_corr"))}} + </td> + <td data-column-heading="Tissue p({%if 'spearman' in (method | lower):%}rho{%else:%}r{%endif%})"> + {{format_number(trait.get("tissue_p_value"))}} + </td> + </tr> + {%endfor%} + </tbody> + </table> + {%endif%} + +</div> +{%endblock%} + +{%block js%} +{%if step == "select-corr-method":%} +<script type="text/javascript" + src="/static/new/javascript/partial_correlations.js"></script> +<script language="javascript" type="text/javascript" + src="{{ url_for('js', filename='DataTables/js/jquery.dataTables.min.js') }}"></script> +{%endif%} +{%endblock%} diff --git a/wqflask/wqflask/templates/partial_correlations/pcorrs_select_operations.html b/wqflask/wqflask/templates/partial_correlations/pcorrs_select_operations.html new file mode 100644 index 00000000..e541f31b --- /dev/null +++ b/wqflask/wqflask/templates/partial_correlations/pcorrs_select_operations.html @@ -0,0 +1,146 @@ +{%extends "base.html"%} + +{%block title%}Partial Correlations:{%endblock%} + +{%block css%} +<link rel="stylesheet" type="text/css" + href="{{url_for('css', filename='DataTables/css/jquery.dataTables.css')}}" /> +<link rel="stylesheet" type="text/css" + href="{{url_for('js', filename='DataTablesExtensions/buttonStyles/css/buttons.dataTables.min.css')}}"> +<link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" /> +<link rel="stylesheet" type="text/css" href="/static/new/css/trait_list.css" /> +<link rel="stylesheet" type="text/css" + href="/static/new/css/partial_correlations.css" /> +{%endblock%} + +{%block content%} +<form id="pcorrs-form" + method="POST" + action="{{url_for('partial_correlations')}}"> + {%with messages = get_flashed_messages(with_categories=true)%} + {%if messages:%} + <ul class=flashes> + {%for category, message in messages:%} + <li class="{{category}}">{{message}}</li> + {%endfor%} + </ul> + {%endif%} + {%endwith%} + + <input type="hidden" value="{{trait_list_str}}" name="trait_list"> + <table id="pcorrs_traits_table" class="table-hover table-striped cell-border"> + <thead> + <tr> + <th>Primary (X)</th> + <th>Controls (Z)</th> + <th>Ignored</th> + <th>Dataset</th> + <th>Trait ID</th> + <th>Symbol</th> + <th>Description</th> + <th>Location</th> + <th>Mean</th> + <th>Max LRS</th> + <th>Max LRS Location Chr and Mb</th> + </tr> + </thead> + + <tbody> + {%for trait in traits:%} + <tr> + <td> + <input type="radio" name="trait_{{trait['trait_name']}}" + id="trait_{{trait['trait_name']}}" + value="primary_{{trait['trait_name']}}"> + </td> + <td> + <input type="radio" name="trait_{{trait['trait_name']}}" + id="trait_{{trait['trait_name']}}" + value="controls_{{trait['trait_name']}}"> + </td> + <td> + <input type="radio" name="trait_{{trait['trait_name']}}" + id="trait_{{trait['trait_name']}}" + value="ignored_{{trait['trait_name']}}" checked="checked"> + </td> + <td>{{trait.get("dataset", "_")}} + <td>{{trait.get("trait_name", "_")}}</td> + <td>{{trait.get("symbol", "_")}}</td> + <td>{{trait.get("description", "_")}}</td> + <td>{{trait.get("location", "_")}}</td> + <td>{{trait.get("mean", "_")}}</td> + <td>{{trait.get("lrs", "_")}}</td> + <td>{{trait.get("lrs_location", "_")}}</td> + </tr> + {%endfor%} + </tbody> + </table> + + <div class="form-group"> + <label for="corr-method-input" class="form-label">Compute</label> + <select id="corr-method-input" required="required" name="method" + class="form-control"> + <option value="Genetic Correlation, Pearson's r"> + Genetic Correlation, Pearson's r</option> + <option value="Genetic Correlation, Spearman's rho"> + Genetic Correlation, Spearman's rho</option> + <option value="SGO Literature Correlation"> + SGO Literature Correlation</option> + <option value="Tissue Correlation, Pearson's r"> + Tissue Correlation, Pearson's r</option> + <option value="Tissue Correlation, Spearman's rho"> + Tissue Correlation, Spearman's rho</option> + </select> + </div> + + <div class="form-group"> + <label for="target-db-input" class="form-label">Choose Database</label> + <select id="target-db-input" required="required" name="target_db" + class="form-control"> + {%if target_dbs:%} + {%for item in target_dbs:%} + {%if "description" in item.keys():%} + <option value="{{item['value']}}">{{item['description']}}</option> + {%else:%} + {%for group, opts in item.items()%} + {%if opts | length > 0:%} + <optgroup label="{{group}} ------"> + {%for item2 in opts:%} + <option value="{{item2['value']}}">{{item2['description']}}</option> + {%endfor%} + </optgroup> + {%endif%} + {%endfor%} + {%endif%} + {%endfor%} + {%endif%} + </select> + </div> + + <div class="form-group"> + <label for="criteria-input" class="form-label">Return</label> + <select id="criteria-input" required="required" name="criteria" size="1" + class="form-control"> + <option value="100">top 100</option> + <option value="200">top 200</option> + <option value="500" selected="selected">top 500</option> + <option value="1000">top 1000</option> + <option value="2000">top 2000</option> + <option value="5000">top 5000</option> + <option value="10000">top 10000</option> + <option value="15000">top 15000</option> + <option value="20000">top 20000</option> + </select> + </div> + + <button type="submit" class="btn btn-primary" name="submit" + value="Run Partial Correlations"> + Run Partial Correlations + </button> +</form> +{%endblock%} + +{%block js%} +<script type="text/javascript" + src="/static/new/javascript/partial_correlations.js"></script> +{%endblock%} diff --git a/wqflask/wqflask/templates/pca_scree_plot.html b/wqflask/wqflask/templates/pca_scree_plot.html new file mode 100644 index 00000000..41036333 --- /dev/null +++ b/wqflask/wqflask/templates/pca_scree_plot.html @@ -0,0 +1,94 @@ +<!DOCTYPE html> +<html> + +<head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title></title> +</head> + +<body> + <div> + pca scree plot + <div id="scree_plot" style="width:700px;height:600px;"></div> + </div> +</body> +<script type="text/javascript" src="{{ url_for('js', filename='plotly/plotly.min.js') }}"></script> +<script type="text/javascript"> +js_data = { { js_data | safe } } + + +let { x_coord, y_coord } = js_data["scree_data"] + +const layout = { + + title: { + text: "<b>Scree Plot</b>", + font: { + "size": 24, + "family": "Arial", + "color": "#FF0000" + } + }, + + yaxis: { + title: { + text: "Percent of total variance %", + font: { + "size": 18, + "color": "" + + } + } + }, + + xaxis: { + title: { + text: "PCA components", + font: { + "size": 18, + "color": "" + + } + } + }, + +} + +const data = [{ + x: x_coord, + y: y_coord, + marker: { + + color: 'rgb(17, 157, 255)', + size: 5, + line: { + color: 'rgb(255, 0, 0)', + width: 3 + } + + } +}] + + +let custom_configs = (filename, download_format, modebar = true) => { + + return { + displayModeBar: modebar, + scrollZoom: false, + toImageButtonOptions: { + download_format, + filename, + height: 600, + width: 700, + scale: 1 + } + } + +} + +Plotly.newPlot(document.getElementById(), data, layout, + custom_configs(file_name = "scree_plot", download_format = "svg")); +</script> + +</html> diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 70c10946..b9859229 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -131,7 +131,7 @@ {% endif %} </div> {% endif %} - <div id="table_container" style="width: {% if dataset.type == 'Geno' %}375{% else %}100%{% endif %}px;"> + <div id="table_container" style="width: {% if dataset.type == 'Geno' %}375px;{% else %}100%; min-width: 1400px;{% endif %}"> <table class="table-hover table-striped cell-border" id='trait_table' style="float: left;"> <tbody> <td colspan="100%" align="center"><br><b><font size="15">Loading...</font></b><br></td> @@ -313,8 +313,8 @@ { 'title': "Authors", 'type': "natural", - {% if (max_widths.authors * 7) < 500 %} - 'width': "{{ max_widths.authors * 7 }}px", + {% if (max_widths.authors * 5) < 500 %} + 'width': "{{ max_widths.authors * 5 }}px", {% else %} 'width': "500px", {% endif %} @@ -322,8 +322,8 @@ 'targets': 5, 'render': function(data, type, row, meta) { author_list = data.authors.split(",") - if (author_list.length >= 6) { - author_string = author_list.slice(0, 6).join(",") + ", et al." + if (author_list.length >= 2) { + author_string = author_list.slice(0, 2).join(",") + ", et al." } else{ author_string = data.authors } diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index 4e9ea0fb..1250d728 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -242,9 +242,6 @@ {% if this_trait.dataset.type == 'ProbeSet' %} <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('/datasets/traits/{{ this_trait.name }}?resource-id={{ resource_id }}', '_blank')">Edit</button> {% endif %} - {% if admin_status.get('metadata', DataRole.VIEW) > DataRole.VIEW %} - <button type="button" id="edit_resource" class="btn btn-success" title="Edit Privileges" onclick="window.open('/resource-management/resources/{{ resource_id }}', '_blank')">Edit Privileges</button> - {% endif %} {% endif %} </div> </div> diff --git a/wqflask/wqflask/templates/tool_buttons.html b/wqflask/wqflask/templates/tool_buttons.html index 3ee5be19..c3927495 100644 --- a/wqflask/wqflask/templates/tool_buttons.html +++ b/wqflask/wqflask/templates/tool_buttons.html @@ -33,3 +33,10 @@ <button id="comp_bar_chart" class="btn btn-primary submit_special" data-url="/comparison_bar_chart" title="Comparison Bar Chart" > Comparison Bar Chart </button> + +<button id="partial-correlations" + class="btn btn-primary submit_special" + data-url="{{url_for('partial_correlations')}}" + title="Run partial correlations with the selected traits"> + Partial Correlations +</button> diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 3bf17da6..59dba309 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1,6 +1,4 @@ """Main routing table for GN2""" - -import MySQLdb import array import base64 import csv @@ -123,6 +121,7 @@ def shutdown_session(exception=None): db = getattr(g, '_database', None) if db is not None: db_session.remove() + g.db.dispose() g.db = None |