From 2e4ca1f6732d14c0ce0d953f1c08ecb000a94747 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 30 Apr 2020 13:44:52 -0500 Subject: GN2 install checklist --- doc/README.org | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/doc/README.org b/doc/README.org index 828ed2cd..b0f1c56b 100644 --- a/doc/README.org +++ b/doc/README.org @@ -658,24 +658,24 @@ The following derivations would be built: Let's see how fast we can deploy a second copy of GN2. -- [-] Base install - + [X] First install a Debian server with GNU Guix on board - + [X] Get Guix build going - - [X] Build the correct version of Guix - - [X] Check out the correct gn-stable version of guix-bioinformatics http://git.genenetwork.org/pjotrp/guix-bioinformatics - - [X] guix package -i genenetwork2 -p /usr/local/guix-profiles/gn2-stable - + [X] Create a gn2 user and home with space - + [X] Install redis (currently debian) - - [X] add to systemd - - [X] update redis.cnf - - [X] update database - + [X] Install mariadb (currently debian mariadb-server) - - [X] add to systemd - - [X] system stop mysql - - [X] update mysql.cnf - - [X] update database (see gn-services/services/mariadb.md) - - [X] check tables +- [ ] Base install + + [ ] First install a Debian server with GNU Guix on board + + [ ] Get Guix build going + - [ ] Build the correct version of Guix + - [ ] Check out the correct gn-stable version of guix-bioinformatics http://git.genenetwork.org/pjotrp/guix-bioinformatics + - [ ] guix package -i genenetwork2 -p /usr/local/guix-profiles/gn2-stable + + [ ] Create a gn2 user and home with space + + [ ] Install redis (currently debian) + - [ ] add to systemd + - [ ] update redis.cnf + - [ ] update database + + [ ] Install mariadb (currently debian mariadb-server) + - [ ] add to systemd + - [ ] system stop mysql + - [ ] update mysql.cnf + - [ ] update database (see gn-services/services/mariadb.md) + - [ ] check tables + [ ] run gn2 (rust-qtlreaper not working) - + [X] update nginx + + [ ] update nginx + [ ] install genenetwork3 - [ ] add to systemd -- cgit v1.2.3 From 9b1dd2f67d51de2d6cdb687ee332597ba010a0ac Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 25 Jul 2020 11:54:10 +0100 Subject: Removing bimbam check. We'll get rid of bimbam anyway. --- wqflask/base/webqtlConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py index 3d86bc22..862ac881 100644 --- a/wqflask/base/webqtlConfig.py +++ b/wqflask/base/webqtlConfig.py @@ -85,7 +85,7 @@ assert_writable_dir(GENERATED_TEXT_DIR) # Flat file directories GENODIR = flat_files('genotype')+'/' assert_dir(GENODIR) -assert_dir(GENODIR+'bimbam') # for gemma +# assert_dir(GENODIR+'bimbam') # for gemma # JSON genotypes are OBSOLETE JSON_GENODIR = flat_files('genotype/json')+'/' -- cgit v1.2.3 From 869e3a2d7f1ef0a44cc90612a46b02d7b01bab88 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 25 Jul 2020 13:35:02 +0100 Subject: Animated gifs --- wqflask/wqflask/static/gif/error/m001.gif | Bin 0 -> 273531 bytes wqflask/wqflask/static/gif/error/m002.gif | Bin 0 -> 1799777 bytes wqflask/wqflask/static/gif/error/m003.gif | Bin 0 -> 2238947 bytes wqflask/wqflask/static/gif/error/m004.gif | Bin 0 -> 2090274 bytes wqflask/wqflask/static/gif/error/m005.gif | Bin 0 -> 947565 bytes wqflask/wqflask/static/gif/error/m006.gif | Bin 0 -> 121116 bytes wqflask/wqflask/static/gif/error/m007.gif | Bin 0 -> 41670 bytes wqflask/wqflask/static/gif/error/m008.gif | Bin 0 -> 732023 bytes wqflask/wqflask/static/gif/error/mouse-wheel.gif | Bin 0 -> 2001764 bytes 9 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 wqflask/wqflask/static/gif/error/m001.gif create mode 100644 wqflask/wqflask/static/gif/error/m002.gif create mode 100644 wqflask/wqflask/static/gif/error/m003.gif create mode 100644 wqflask/wqflask/static/gif/error/m004.gif create mode 100644 wqflask/wqflask/static/gif/error/m005.gif create mode 100644 wqflask/wqflask/static/gif/error/m006.gif create mode 100644 wqflask/wqflask/static/gif/error/m007.gif create mode 100644 wqflask/wqflask/static/gif/error/m008.gif create mode 100644 wqflask/wqflask/static/gif/error/mouse-wheel.gif diff --git a/wqflask/wqflask/static/gif/error/m001.gif b/wqflask/wqflask/static/gif/error/m001.gif new file mode 100644 index 00000000..81c8ba26 Binary files /dev/null and b/wqflask/wqflask/static/gif/error/m001.gif differ diff --git a/wqflask/wqflask/static/gif/error/m002.gif b/wqflask/wqflask/static/gif/error/m002.gif new file mode 100644 index 00000000..7232c58b Binary files /dev/null and b/wqflask/wqflask/static/gif/error/m002.gif differ diff --git a/wqflask/wqflask/static/gif/error/m003.gif b/wqflask/wqflask/static/gif/error/m003.gif new file mode 100644 index 00000000..2384ceb6 Binary files /dev/null and b/wqflask/wqflask/static/gif/error/m003.gif differ diff --git a/wqflask/wqflask/static/gif/error/m004.gif b/wqflask/wqflask/static/gif/error/m004.gif new file mode 100644 index 00000000..d77c708d Binary files /dev/null and b/wqflask/wqflask/static/gif/error/m004.gif differ diff --git a/wqflask/wqflask/static/gif/error/m005.gif b/wqflask/wqflask/static/gif/error/m005.gif new file mode 100644 index 00000000..1b2de7ec Binary files /dev/null and b/wqflask/wqflask/static/gif/error/m005.gif differ diff --git a/wqflask/wqflask/static/gif/error/m006.gif b/wqflask/wqflask/static/gif/error/m006.gif new file mode 100644 index 00000000..f354a4cc Binary files /dev/null and b/wqflask/wqflask/static/gif/error/m006.gif differ diff --git a/wqflask/wqflask/static/gif/error/m007.gif b/wqflask/wqflask/static/gif/error/m007.gif new file mode 100644 index 00000000..ba2eeb37 Binary files /dev/null and b/wqflask/wqflask/static/gif/error/m007.gif differ diff --git a/wqflask/wqflask/static/gif/error/m008.gif b/wqflask/wqflask/static/gif/error/m008.gif new file mode 100644 index 00000000..4cdec5cb Binary files /dev/null and b/wqflask/wqflask/static/gif/error/m008.gif differ diff --git a/wqflask/wqflask/static/gif/error/mouse-wheel.gif b/wqflask/wqflask/static/gif/error/mouse-wheel.gif new file mode 100644 index 00000000..164220e7 Binary files /dev/null and b/wqflask/wqflask/static/gif/error/mouse-wheel.gif differ -- cgit v1.2.3 From 91e52482071ce90acf3607597de2ae79e298fd67 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 25 Jul 2020 16:25:14 +0100 Subject: The variable is never defined, so let's default to "none" --- wqflask/utility/authentication_tools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py index ed7462d1..9a2a5ccd 100644 --- a/wqflask/utility/authentication_tools.py +++ b/wqflask/utility/authentication_tools.py @@ -45,7 +45,7 @@ def check_resource_availability(dataset, trait_id=None): def add_new_resource(dataset, trait_id=None): resource_ob = { - 'owner_id' : webqtlConfig.DEFAULT_OWNER_ID, + 'owner_id' : "none", # webqtlConfig.DEFAULT_OWNER_ID, 'default_mask': webqtlConfig.DEFAULT_PRIVILEGES, 'group_masks' : {} } @@ -84,6 +84,7 @@ def check_admin(resource_id=None): try: response = json.loads(requests.get(the_url).content)['admin'] except: + logger.debug(resource_info) response = resource_info['default_mask']['admin'] if 'edit-admins' in response: @@ -124,4 +125,4 @@ def check_owner_or_admin(dataset=None, trait_id=None, resource_id=None): else: return check_admin(resource_id) - return "not-admin" \ No newline at end of file + return "not-admin" -- cgit v1.2.3 From 918dcce5f42fcbe9327eb30bac06a69000a1fc8b Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 25 Jul 2020 16:47:24 +0100 Subject: Updating docs and show how to include a local Python module --- README.md | 21 +-- bin/genenetwork2 | 4 +- doc/README.org | 482 ++++++---------------------------------------------- doc/development.org | 55 ++++++ 4 files changed, 114 insertions(+), 448 deletions(-) diff --git a/README.md b/README.md index 47519118..a2a4625a 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,8 @@ developers). See the [installation docs](doc/README.org). ## Run -Once installed GN2 can be run online through a browser interface +Once having installed GN2 it can be run through a browser +interface ```sh genenetwork2 @@ -35,18 +36,12 @@ For full examples (you may need to set a number of environment variables), including running scripts and a Python REPL, also see the startup script [./bin/genenetwork2](https://github.com/genenetwork/genenetwork2/blob/testing/bin/genenetwork2). -Also Mysql and Elasticsearch need to be running, see +Also mariadb and redis need to be running, see [INSTALL](./doc/README.org). ## Testing -We are building 'Mechanical Rob' automated testing using Python -[requests](https://github.com/genenetwork/genenetwork2/tree/master/test/lib) -which can be run with something like - -```sh -env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py -a http://localhost:5003 -``` +We are building 'Mechanical Rob' automated testing using Python. The GN2_PROFILE is the Guix profile that contains all dependencies. The ./bin/genenetwork2 script sets up the environment @@ -73,10 +68,10 @@ Contribute to GN2 source code by forking the [github repository](https://github.com/genenetwork/genenetwork2/) with git and sending us pull requests. -For development GN2 has a -[mailing list](http://listserv.uthsc.edu/mailman/listinfo/genenetwork-dev) -and an active IRC channel #genenetwork on freenode.net with a -[web interface](http://webchat.freenode.net/). +For development GN2 has a [mailing +list](http://listserv.uthsc.edu/mailman/listinfo/genenetwork-dev) and +an active IRC channel #genenetwork on freenode.net with a [web +interface](http://webchat.freenode.net/). ## License diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 3ae08e0a..dd6db7d6 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -121,7 +121,7 @@ if [ -z $GN2_PROFILE ]; then read -p "PRESS [ENTER] TO CONTINUE..." else export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages" # never inject another PYTHONPATH!! + export PYTHONPATH="$PYTHON_GN_PATH:$GN2_PROFILE/lib/python2.7/site-packages" # never inject another PYTHONPATH!! export R_LIBS_SITE=$GN2_PROFILE/site-library export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" @@ -154,7 +154,7 @@ if [ ! -d $R_LIBS_SITE ] ; then fi # We may change this one: -export PYTHONPATH=$GN2_BASE_DIR/wqflask:$PYTHONPATH +export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$PYTHONPATH # Our UNIX TMPDIR defaults to /tmp - change this on a shared server if [ -z $TMPDIR ]; then diff --git a/doc/README.org b/doc/README.org index c1b1c2b6..46df03c7 100644 --- a/doc/README.org +++ b/doc/README.org @@ -4,9 +4,12 @@ - [[#introduction][Introduction]] - [[#install][Install]] - [[#running-gn2][Running GN2]] + - [[#run-gn-proxy][Run gn-proxy]] + - [[#run-redis][Run Redis]] - [[#run-mariadb-server][Run MariaDB server]] - [[#install-mariadb-with-gnu-guix][Install MariaDB with GNU GUIx]] - [[#load-the-small-database-in-mysql][Load the small database in MySQL]] + - [[#get-genotype-files][Get genotype files]] - [[#gn2-dependency-graph][GN2 Dependency Graph]] - [[#working-with-the-gn2-source-code][Working with the GN2 source code]] - [[#read-more][Read more]] @@ -16,7 +19,6 @@ - [[#cant-run-a-module][Can't run a module]] - [[#rpy2-error-show-now-found][Rpy2 error 'show' now found]] - [[#mysql-cant-connect-server-through-socket-error][Mysql can't connect server through socket ERROR]] - - [[#irc-session][IRC session]] - [[#notes][NOTES]] - [[#deploying-gn2-official][Deploying GN2 official]] @@ -57,28 +59,39 @@ or you can set environment variables to override individual parameters, e.g. the debug and logging switches can be particularly useful when developing GN2. -* Running Redis +* Run gn-proxy -Install redis. Make sure you add the setting: +GeneNetwork requires a separate gn-proxy server which handles +authorisation and access control. For instructions see the [[https://github.com/genenetwork/gn-proxy][README]]. -: appendonly yes +* Run Redis + +Redis part of GN2 deployment and will be started by the ./bin/genenetwork2 +startup script. * Run MariaDB server ** Install MariaDB with GNU GUIx -/Note: we moved to MariaDB/ - These are the steps you can take to install a fresh installation of mariadb (which comes as part of the GNU Guix genenetwork2 install). -As root configure and run +As root configure the Guix profile + +: . ~/opt/genenetwork2/etc/profile + +and run for example #+BEGIN_SRC bash adduser mariadb && addgroup mariadb -mysqld --datadir=/home/mariadb/database --initialize-insecure -mkdir -p /var/run/mariadbd -chown mariadb.mariadb /var/run/mariadbd -mysqld -u mariadb --datadir=/home/mariadb/database/mariadb --explicit_defaults_for_timestamp -P 12048" +mkdir -p /export2/mariadb/database +chown mariadb.mariadb -R /export2/mariadb/ +mkdir -p /var/run/mysqld +chown mariadb.mariadb /var/run/mysqld +su mariadb +mysql --version + mysql Ver 15.1 Distrib 10.1.45-MariaDB, for Linux (x86_64) using readline 5.1 +mysql_install_db --user=mariadb --datadir=/export2/mariadb/database +mysqld -u mariadb --datadir=/exportdb/mariadb/database/mariadb --explicit_defaults_for_timestamp -P 12048" #+END_SRC If you want to run as root you may have to set @@ -120,21 +133,22 @@ material. Download one database from -[[http://files.genenetwork.org/raw_database/]] +http://ipfs.genenetwork.org/ipfs/QmRUmYu6ogxEdzZeE8PuXMGCDa8M3y2uFcfo4zqQRbpxtk -[[https://s3.amazonaws.com/genenetwork2/db_webqtl_s.zip]] +After installation unzip the database binary in the MySQL directory -Check the md5sum. - -After installation inflate the database binary in the MySQL directory - -: cd ~/mysql -: chown -R mysql:mysql db_webqtl_s/ -: chmod 700 db_webqtl_s/ -: chmod 660 db_webqtl_s/* +#+BEGIN_SRC sh +cd ~/mysql +p7zip -d db_webqtl_s.7z +chown -R mysql:mysql db_webqtl_s/ +chmod 700 db_webqtl_s/ +chmod 660 db_webqtl_s/* +#+END_SRC restart MySQL service (mysqld). Login as root +: mysql_upgrade -u root --force + : myslq -u root and @@ -151,7 +165,7 @@ and Set permissions and match password in your settings file below: -: mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'mysql_password'; +: mysql> grant all privileges on db_webqtl_s.* to gn2@"localhost" identified by 'webqtl'; You may need to change "localhost" to whatever domain you are connecting from (mysql will give an error). @@ -163,6 +177,17 @@ configuration (see below). Note for the plant database you can rename it to db_webqtl_s, or change the settings in etc/default_settings.py to match your path. +* Get genotype files + +The script looks for genotype files. You can find them in +http://ipfs.genenetwork.org/ipfs/QmXQy3DAUWJuYxubLHLkPMNCEVq1oV7844xWG2d1GSPFPL + +#+BEGIN_SRC sh +mkdir -p $HOME/genotype_files +cd $HOME/genotype_files + +#+END_SRC + * GN2 Dependency Graph Graph of all runtime dependencies as installed by GNU Guix. @@ -249,415 +274,6 @@ if that works run genenetwork after setting SQL_URI to something like : export SQL_URI=mysql://gn2:mysql_password@127.0.0.1/db_webqtl_s - -* IRC session - -Here an IRC session where we installed GN2 from scratch using GNU Guix -and a download of the test database. - -#+begin_src - time to get binary install sorted :) [07:03] - Guix is designed for distributed installation servers - we have one on guix.genenetwork.org - it contains all the prebuild packages - for GN - okay [07:04] - let's step back however [07:05] - I presume the environment is set with all guix package --search-paths - right? - yep - set to the ones in ~/.guix-profile/ - good, and you are in gn-deploy-guix repo [07:06] - yep [07:07] - git log shows - -Author: David Thompson -Date: Sun Mar 27 21:20:19 2016 -0400 - - yes - env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix - package -A genenetwork2 [07:08] - shows - -genenetwork2 2.0-a8fcff4 out ../guix-bioinformatics/gn/packages/genenetwork.scm:144:2 -genenetwork2-database-small 1.0 out ../guix-bioinformatics/gn/packages/genenetwork.scm:270:4 -genenetwork2-files-small 1.0 out ../guix-bioinformatics/gn/packages/genenetwork.scm:228:4 - - yeah [07:09] - OK, we are in sync. This means we should be able to install the exact - same software - I need to start up my guix daemon - I usually run it in a screen - screen -S guix-daemon - hah, I don't have screen installed yet [07:11] - comes with guix ;) [07:12] - no worries, you can run it any way you want - $HOME/.guix-profile/bin/guix-daemon --build-users-group=guixbuild - then something's weird, because it says I don't have it - oh, you need to install it first [07:13] - guix package -A screen - screen 4.3.1 out gnu/packages/screen.scm:34:2 - but you can skip this install, for now - alright [07:14] - env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix - package -i genenetwork2 --dry-run - substitute: updating list of substitutes from - 'https://mirror.hydra.gnu.org'... 79.1% - you see that? - followed by [07:15] -substitute: updating list of substitutes from -'https://hydra.gnu.org'... 100.0% -The following derivations would be built: - /gnu/store/rk7nw0rjqqsha958m649wrykadx6mmhl-profile.drv - -/gnu/store/7b0qjybvfx8syzvfs7p5rdablwhbkbvs-module-import-compiled.drv - /gnu/store/cy9zahbbf23d3cqyy404lk9f50z192kp-module-import.drv - /gnu/store/ibdn603i8grf0jziy5gjsly34wx82lmk-gtk-icon-themes.drv - - which should have the same HASH values /gnu/store/7b0qjybvf... etc. - [07:16] - profile has a different hash - but the next ones? - they're the same - not sure why profile differs. Do you see the contact with - mirror.hydra.org? [07:17] - yeah - OK, that means you set the key correctly for that one :) - alright we are at the same state now. You can see most packages need - to be rebuild because they are no longer cached as binaries on hydra - [07:18] - things move fast... - hehe - let me also do the same on my laptop - which I have staged before - [07:19] - btw, to set the path I often do [07:20] - export - PATH="/home/wrk/.guix-profile/bin:/home/wrk/.guix-profile/sbin":$PATH - to keep things like 'screen' from Debian - Once past building guix itself that is normally OK [07:21] - ah, okay - will do that - the guix build requires certain versions of tools, so you don't want - to mix foreign tools in [07:23] - makes sense [07:24] - On my laptop I am trying the main updating list of substitutes from - 'http://hydra.gnu.org'... 10.5% [07:27] - it is a bit slow, but let's see if there is a difference with the - mirror - you can see there are two servers here. Actually with recent daemons, - if the mirror fails it will try the main server [07:28] - I documented the use of a caching server here [07:29] - https://github.com/pjotrp/guix-notes/blob/master/REPRODUCIBLE.org - this is exactly what we are doing now - alrighty [07:35] - To see if a remote server has a guix server running it should respond - [07:36] - lynx http://guix.genenetwork.org:8080 --dump - Resource not found: / - - you see that? - yes [07:37] - good. The main hydra server is too slow. So on my laptop I forced - using the mirror with [07:38] - env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env guix - package -i genenetwork2 --dry-run - --substitute-urls="http://mirror.hydra.gnu.org" - - the list looks the same to me [07:40] - me too - note that some packages will be built and some downloaded, right? - [07:41] - yes - atlas is actually a binary on my system [07:43] - I mean in that list - so, it should not build. Same as yours? - yeah, atlas and r-gtable are the ones to be downloaded - You should not have seen that error ;) - we should try and install it this way, try [07:44] - env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix - package -i genenetwork2 --cores=4 --max-jobs=4 --keep-going [07:46] - set CPUs and max-jobs to something sensible - Does your VM have multiple cores? - note you can always press Ctrl-C during install - it doesn't, I'll reboot it and give it another core [07:47] - Hey [07:48] - I'm here - Will be stepping away for some breakfast - Can you do the same as us - Can you see the irc log - Alright - Yes, I can - Please email me a copy in five minutes - user01: so when I use the GN server [07:56] - env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix - package -i genenetwork2 --dry-run - --substitute-urls=http://guix.genenetwork.org:8080 - I don't need to build anything [07:57] - (this won't work for you, yet) - to get it to work you need to 'trust' it [07:58] - but, first get the build going - I'll have a coffee while you and get building - yeah it's doing its thing now [08:01] - cool [08:02] - in a separate terminal you can try and install with the gn mirror - [08:05] - I'll send you the public key and you can paste it as said - https://github.com/pjotrp/guix-notes/blob/master/REPRODUCIBLE.org - [08:06] - alright - should be in the E-mail [08:09] - getting it working it kinda nasty since the server gives no feedback - it works when you see no more in the build list ;) [08:11] - btw, you can install software in parallel. Guix does that. - even the same packages - so keep building ;) - try and do this with Debian... - coffee for me [08:12] - the first build failed [08:15] - OK, Dennis fixed that one yesterday [08:27] - the problem is that sometime source tarballs disappear [08:28] - R is notorious for that - haha, that's inconvenient.. - well, it is good that Guix catches them - but we do not cache sources - binaries are cached - to some degree - so we don't have to rebuild - those [08:29] - time to use the guix cache at guix.genenetwork.org - try and install the key (it is in the E-mail) - and see what this lists [08:31] - env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix - package -i genenetwork2 - --substitute-urls=http://guix.genenetwork.org --dry-run - should be all binary installs - it's not.. [08:32] - if I remove --substitute-urls, the list changes, does that mean I - have the key set up correctly at least? [08:33] - dunno [08:35] - how many packages does it want to build? - should be zero - four - Ah, that is OK - those are default profile things - genenetwork2 is among the ones to be downloaded so [08:36] - remove --dry-run - yeah, good sign :) - we'll still hit a snag, but run it - should be fast - doing it [08:37] - it worked! [08:38] - I think [08:39] - heh [08:40] - you mean it is finished? - yep - type genenetwork2 - complains about not being able to connect to the database [08:41] - last snag :) - no database - well, we succeeded in installing a same-byte install of a very - complex system :) [08:42] - (always take time to congratulate yourself) - now we need to install mysql - hehe :) - this can be done throug guix or through debian [08:43] - the latter is a bit easier here, so let's do that - fun note: you can mix debian and guix - Follow instructions on [08:44] - - https://github.com/genenetwork/genenetwork2/tree/staging/doc#run-mysql-server - apt-get install mysql-common [08:45] - may do it - You can also install with guix, but I need to document that - btw your internet must be fast :) [08:46] - hehe it is ;) - when the database is installed [08:48] - be sure to set the password as instructed [08:50] - when mysql is set the genenetwork2 command should fire up the web - server on localhost:5003 [08:58] - btw my internet is way slower :) [09:00] - I'm back [09:04] - fixed router firmware upgrade problem - unbricking - tssk [09:07] - I'll never leave routers to update themselves again [09:08] - self-brick highway - Resuming [09:09] - auto-updates are evil - always switch them off - user02: can you install genenetwork like user has done? [09:10] - pretty well documented here now :) - Yes I can [09:11] - Already installed key - user02: you are getting binary packages only now? [09:13] - That's the sanest way to go now - seriously - everything should be pre-built from guix.genenetwork.org - you are downloading? - yes [09:15] - cool. Maybe an idea to set up a server - for your own use - Stuck at downloading preprocesscore - should not [09:24] - what does env GUIX_PACKAGE_PATH=../guix-bioinformatics/ - ./pre-inst-env guix package -i genenetwork2 - --substitute-urls="http://guix.genenetwork.org" --dry-run - [09:25] - say for r-prepocesscore - download or build? - mine says download [09:26] - it only lists the derivatives to be built - nothing else happens [09:27] - OK, so there is a problem - your key may not be working - everything should be listed as 'to be download' [09:28] - Hmm - Ah - I know where I messed up - where? - I did add the key - However - (I am documenting) - I did not tell guix to trust it - yes - and there is another potential problem - Remember the documentation on installing guix? - You have to tell guix to trust the default key [09:29] - Right? - So in this case - read the IRC log - That step is mandatory - user01: how are you doing? - user02: - https://github.com/pjotrp/guix-notes/blob/master/REPRODUCIBLE.org#using-gnu-guix-archive - [09:30] - a little bit left on the db download - user02: you should see no more building - user02: another issue may be that you updated r-preprocesscore - package in guix-buinformatics [09:32] - all downstream packages will want to rebuild - no, not really - It's not even installed - checkout a branch of the the old version - make sure we are in synch - should be at - /gnu/store/y1f3r2xs3fhyadd46nd2aqbr2p9qv2ra-r-biocpreprocesscore-1.32.0 - [09:33] - - pjotrp: Possibly we should use the archive utility of Guix to do - deployment to avoid such out-of-sync differences :) [09:34] - maybe. I did not get archive to update profiles properly [09:37] - Also it is good that they get to understand guix - this way - carved in stone, eh [09:38] - Yeah, all good [09:39] - My mistake was skipping the guix archive part - Can we begin with the install? - It's telling me of derivatives that will be downloaded [09:40] - So we're good - Here goes - yeeha [09:42] - pjotrp, where is this guix.genenetwork.org located at? - Tennessee - It's...it's....sloooooooowwwwwwwwwwwwww - not from Europe - is it downloading at all? - It should be extended - Yes...like at 100KB/s [09:43] - tear-jerker - Verizon problems - who's the host? - I am getting 500Kb/s - UT - Guix's servers can run off more than one server, right? - I'd like to host that particular server here - For speed - yes - Sooner or later - It will be a necessity [09:45] - exactly what I am doing - this is our server - guix.genenetwork.org:8080 - All done installing [09:46] - what? - Now the databases - what do you mean by slow exactly? - Yes, it's installed - can you run genenetwork2 - setting variables - If I try running it now, it will fail as I don't have the DBs [09:47] - cool - you had a lot of prebuilt packages already - OK, follow the instructions I wrote above - now everything seems to be working for me :) - OK - user01: excellent! - you see a webserver? - yep, can connect to localhost:5003 [09:48] - So now you are running a guix copy of GN2 - you can see where it lives with `which genenetwork2` or ls -l - ~/.guix-profile/bin/genenetwork2 [09:49] - - /gnu/store/1kma5xszvzsvmbb4k699h7gvdncw901i-genenetwork2-2.0-a8fcff4/bin/genenetwork2 - it is a script - written by guix, open it [09:50] - inside it points to paths and our script at - - /gnu/store/1kma5xszvzsvmbb4k699h7gvdncw901i-genenetwork2-2.0-a8fcff4/bin/.genenetwork2-real - if you open that you can see how the webserver is started [09:51] - next step is to run a recent version of GN2 - okay [09:52] - See - https://github.com/genenetwork/genenetwork2/tree/staging/doc#run-your-own-copy-of-gn2 - but do not checkout that genetwork2_diet - we reverted to the main tree - clone git@github.com:genenetwork/genenetwork2.git [09:53] - instead and checkout the staging branch - that is effectively my branch [09:54] - when that is done you should be able to fire up the webserver from - there [09:55] - using ./bin/genenetwork2 - now installing DBs - Downloading - annoyingly the source tree is ~700Mb [09:56] - Can it also be done by installing the guix package - genenetwork2-database-small? - I changed it in the diet version to 8Mb, but I had to revert - I need to make my VM bigger... - user02: not ready [09:57] - ok - user01: sorry - user01: you could mount a local dir inside the VM for development - that would allow you to use MAC tools for editing - just an idea - yeah, I figure I'll do something like that - do you use emacs? [09:58] - yep - that can also run on remote files over ssh - that's an alternative - kudos for using emacs :), wdyt user03 - 79 minutes to go downloading the db - user02: sorry about that [09:59] - it is 2GB - user, you can also mount the directory via sshfs - Mac OSX runs OpenSSH - user02: sopa - You can therefore mount a directory outside the VM to the VM via - sshfs [10:00] - yes, 3 options now - That way, you can set up a VM only for it's logic - Apps + the OS it runs [10:01] - For data, let it reside on physical host accessible via sshfs - Use this Arch wiki reference: - https://wiki.archlinux.org/index.php/SSHFS - I edited that last somewhere in 2015, may have been updated since - then - alright, cool! [10:04] - user01: you are almost done [10:06] - I wrote an elixir package for guix :) - env GUIX_PACKAGE_PATH=../guix-bioinformatics/ ./pre-inst-env guix - package -A elixir - --substitute-urls="http://guix.genenetwork.org" [10:08] - elixir 1.2.3 out - ../guix-bioinformatics/gn/packages/elixir.scm:31:2 - - I am building it on guix.genenetwork.org right now [10:09] - nice [10:10] -#+end_src - * NOTES ** Deploying GN2 official @@ -671,7 +287,7 @@ Let's see how fast we can deploy a second copy of GN2. - [ ] Check out the correct gn-stable version of guix-bioinformatics http://git.genenetwork.org/pjotrp/guix-bioinformatics - [ ] guix package -i genenetwork2 -p /usr/local/guix-profiles/gn2-stable + [ ] Create a gn2 user and home with space - + [ ] Install redis (currently debian) + + [ ] Install redis - [ ] add to systemd - [ ] update redis.cnf - [ ] update database @@ -681,7 +297,7 @@ Let's see how fast we can deploy a second copy of GN2. - [ ] update mysql.cnf - [ ] update database (see gn-services/services/mariadb.md) - [ ] check tables - + [ ] run gn2 (rust-qtlreaper not working) + + [ ] run gn2 + [ ] update nginx + [ ] install genenetwork3 - [ ] add to systemd diff --git a/doc/development.org b/doc/development.org index 5e6e318b..e65ccd58 100644 --- a/doc/development.org +++ b/doc/development.org @@ -41,3 +41,58 @@ JS_GN_PATH (3) is for development purposes. By default is is set to $HOME/genenetwork/javascript. Say you are working on an updated version of a JS module not yet in (1) you can simply check out that module in that path and it should show up. + +* Python modules + +Python modules are automatically found in the Guix profile. + +For development purposes it may be useful to try some Python package. +Obviously this is only a temporary measure and when you decide to +include the package it should be packaged in [[http://git.genenetwork.org/guix-bioinformatics/guix-bioinformatics][our GNU Guix software +stack]]! + +To add packages you need to make sure the correct Python is used (currently +Python 2.7) to install a package. E.g.. + +#+BEGIN_SRC sh +python --version + Python 2.7.16 +pip --version + pip 18.1 from /usr/lib/python2.7/dist-packages/pip (python 2.7) +#+END_SRC + +You can install a Python package locally with pip, e.g. + +#+BEGIN_SRC sh +pip install hjson +#+END_SRC + +This installed in ~$HOME/.local/lib/python2.7/site-packages~. To add +the search path for GeneNetwork use the environment variable + +#+BEGIN_SRC sh +export PYTHON_GN_PATH=$HOME/.local/lib/python2.7/site-packages +#+END_SRC + +Now you should be able to do + +#+BEGIN_SRC python +import hjson +#+END_SRC + +In fact you can kick off a Python shell with something like + +#+BEGIN_SRC python +env SERVER_PORT=5013 WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \ + SQL_URI=mysql://gn2:webqtl@localhost/db_webqtl_s \ + GN2_PROFILE=~/opt/genenetwork2 \ + ./bin/genenetwork2 ./etc/default_settings.py -c +Python 2.7.17 (default, Jan 1 1970, 00:00:01) +[GCC 7.5.0] on linux2 +Type "help", "copyright", "credits" or "license" for more information. +>>> import hjson +#+END_SRC + +It should now also work in GN2. + +* TODO External tools -- cgit v1.2.3 From 5a63137af7e383bbae867c4385c42cacf0d78b8e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sun, 26 Jul 2020 23:04:29 +0300 Subject: Delete tests/unittests folder * tests/unittests: Delete it. All unittests were moved to wqflask/tests in 3d6483e8 --- test/unittest/__init__.py | 0 test/unittest/base/__init__.py | 0 test/unittest/base/test_general_object.py | 21 --------------------- 3 files changed, 21 deletions(-) delete mode 100644 test/unittest/__init__.py delete mode 100644 test/unittest/base/__init__.py delete mode 100644 test/unittest/base/test_general_object.py diff --git a/test/unittest/__init__.py b/test/unittest/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/unittest/base/__init__.py b/test/unittest/base/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/unittest/base/test_general_object.py b/test/unittest/base/test_general_object.py deleted file mode 100644 index 699cb079..00000000 --- a/test/unittest/base/test_general_object.py +++ /dev/null @@ -1,21 +0,0 @@ -import unittest - -from base.GeneralObject import GeneralObject - - -class TestGeneralObjectTests(unittest.TestCase): - """ - Test the GeneralObject base class - """ - - def test_object_contents(self): - """Test whether base contents are stored properly""" - test_obj = GeneralObject("a", "b", "c") - self.assertEqual("abc", ''.join(test_obj.contents)) - - def test_object_dict(self): - """Test whether the base class is printed properly""" - test_obj = GeneralObject("a", name="test", value=1) - self.assertEqual(str(test_obj), "value = 1\nname = test\n") - self.assertEqual( - repr(test_obj), "value = 1\nname = test\ncontents = ['a']\n") -- cgit v1.2.3 From c1143ffc544abf9c4f12d6676d2584ef1d1cb95e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sun, 26 Jul 2020 23:18:54 +0300 Subject: Document how to run tests * README.md: - Add details on how to run unit tests - Update link that points to Mechanical Rob --- README.md | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 47519118..45aca1b2 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,13 @@ Also Mysql and Elasticsearch need to be running, see ## Testing +To have tests pass, the redis and mariadb instance should be running, because of +asserts sprinkled in the code base(these will be removed in due time). + +#### Mechanical Rob We are building 'Mechanical Rob' automated testing using Python -[requests](https://github.com/genenetwork/genenetwork2/tree/master/test/lib) -which can be run with something like +[requests](https://github.com/genenetwork/genenetwork2/tree/testing/test/requests) +which can be run with: ```sh env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py -a http://localhost:5003 @@ -54,6 +58,32 @@ and executes test-website.py in a Python interpreter. The -a switch says to run all tests and the URL points to the running GN2 http server. +#### Unit tests + +To run unittests, first `cd` into the genenetwork2 directory: + +```sh +# You can use the coverage tool to run the tests +# You could omit the -v which makes the output verbose +runcmd coverage run -m unittest discover -v + +# Alternatively, you could run the unittests using: +runpython -m unittest discover -v + +# To generate a report in wqflask/coverage_html_report/: +runcmd coverage html +``` + +The `runcmd` and `runpython` are shell aliases defined in the following way: + +```sh +alias runpython="env GN2_PROFILE=~/opt/gn-latest TMPDIR=/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ ./bin/genenetwork2 + +alias runcmd="time env GN2_PROFILE=~/opt/gn-latest TMPDIR=//tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ ./bin/genenetwork2 ./etc/default_settings.py -cli" +``` + +Replace some of the env variables as per your use case. + ## Documentation User documentation can be found -- cgit v1.2.3 From e7c02e570635f6aebe2e2596bfb2e6231a7569e8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Jul 2020 00:31:12 +0300 Subject: Document how to run tests * doc/testing.org: - Update how Mechanical Robs works. Python replaced Ruby as the testing tool - Add section that describes how to run unit tests --- doc/testing.org | 66 +++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/doc/testing.org b/doc/testing.org index 1d5cc8b8..d5ab117d 100644 --- a/doc/testing.org +++ b/doc/testing.org @@ -1,43 +1,67 @@ #+TITLE: Testing GN2 * Table of Contents :TOC: - - [[#introduction][Introduction]] - - [[#run-tests][Run tests]] - - [[#setup][Setup]] - - [[#running][Running]] +- [[#introduction][Introduction]] +- [[#run-tests][Run tests]] + - [[#setup][Setup]] + - [[#running][Running]] * Introduction -For integration testing we currently use the brilliant Ruby Mechanize -gem against the small database; a setup we call mechanical Rob because -it emulates someone clicking through the website and checking results. +For integration testing, we currently use [[https://github.com/genenetwork/genenetwork2/tree/testing/test/requests][Mechanica Rob]] against the +small [[https://github.com/genenetwork/genenetwork2/blob/testing/doc/database.org][database]]; a setup we call Mechanical Rob because it emulates +someone clicking through the website and checking results. -These scripts invoke calls to a running webserver and test the -response. If a page changes or is broken tests will break and we are -informed. In principle, Mechanical Rob is run before code merges are -committed to the main server. +These scripts invoke calls to a running webserver and test the response. +If a page changes or breaks, tests will fail. In principle, Mechanical +Rob runs before code merges get committed to the main server. -In the future we may move to Python mechanize - it'll be easy to mix -the Ruby and Python versions. +For unit tests, we use python's =unittest= framework. Coverage reports +get generated using [[https://coverage.readthedocs.io/en/coverage-5.2.1/][coverage.py]] which you could also use to run +unit tests. When adding new functionality, it is advisable to add +unit tests. * Run tests ** Setup -Mechanize is not yet included in Guix deployment. +Everything required for testing is already package with guix: +: ./pre-ins-env guix package -i genenetwork2 -p ~/opt/genenetwork2 ** Running -Run the tests from the root of the genenetwork2 source tree as, for -example, +Run the tests from the root of the genenetwork2 source tree as. Ensure +that Redis and Mariadb are running. -: ./bin/test-website http://localhost:5003/ (default) +To run Mechanical Rob: +: time env GN2_PROFILE=~/opt/genenetwork2 TMPDIR=~/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ ./bin/genenetwork2 ./etc/default_settings.py -c ~/projects/genenetwork2/test/requests/test-website.py -a http://localhost:5004 -If you are using the small deployment database you can use +Use these aliases for the following examples. -: ./bin/test-website --skip -n +#+begin_src sh +alias runpython="env GN2_PROFILE=~/opt/gn-latest TMPDIR=/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ ./bin/genenetwork2 -To run individual tests on localhost you can do +alias runcmd="time env GN2_PROFILE=~/opt/gn-latest TMPDIR=//tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ ./bin/genenetwork2 ./etc/default_settings.py -cli" +#+end_src -: ruby -Itest -Itest/lib test/lib/mapping.rb --name="/Mapping/" +You could use them in your =.bashrc= or =.zshrc= file. + +To run unit tests: + +: runpython -m unittest discover -v + +Or alternatively using the coverage tool: + +: runcmd coverage run -m unittest discover -v + +To generate a html coverage report in =wqflask/coverage_html_report/= + +: runcmd coverage html + +To output the report to =STDOUT=: + +: runcmd coverage report + +All the configs for running the coverage tool are in +=wqflask/.coveragerc= -- cgit v1.2.3 From d63e7554d6dfce4e80c2570667a0fa371235beb7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Jul 2020 14:31:31 +0300 Subject: Apply py-lint * wqflask/tests/base/test_data_set.py: Apply pylint --- wqflask/tests/base/test_data_set.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/wqflask/tests/base/test_data_set.py b/wqflask/tests/base/test_data_set.py index 44a54c7e..74799e49 100644 --- a/wqflask/tests/base/test_data_set.py +++ b/wqflask/tests/base/test_data_set.py @@ -1,3 +1,5 @@ +"""Tests for wqflask/base/data_set.py""" + import unittest import mock @@ -5,8 +7,10 @@ from wqflask import app from base.data_set import DatasetType - + class TestDataSetTypes(unittest.TestCase): + """Tests for the DataSetType class""" + def setUp(self): self.app_context = app.app_context() self.app_context.push() @@ -16,10 +20,14 @@ class TestDataSetTypes(unittest.TestCase): @mock.patch('base.data_set.g') def test_data_set_type(self, db_mock): + """Test that DatasetType returns correctly if the Redis Instance is not empty + and the name variable exists in the dictionary + + """ with app.app_context(): db_mock.get = mock.Mock() - r = mock.Mock() - r.get.return_value = """ + redis_mock = mock.Mock() + redis_mock.get.return_value = """ { "AD-cases-controls-MyersGeno": "Geno", "AD-cases-controls-MyersPublish": "Publish", @@ -32,4 +40,6 @@ class TestDataSetTypes(unittest.TestCase): "B139_K_1206_R": "ProbeSet" } """ - self.assertEqual(DatasetType(r)("All Phenotypes"), "Publish") + self.assertEqual(DatasetType(redis_mock) + ("All Phenotypes"), "Publish") + -- cgit v1.2.3 From 5cad720187b3c53b6d64c64d45be4bc020eed52d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Jul 2020 14:32:26 +0300 Subject: Add test case for empty redis instance for DatasetType * wqflask/tests/base/test_data_set.py(tests): Check correct results are returned when Redis is empty * wqflask/tests/base/data.py(tests): New file. Adds json test data. --- wqflask/tests/base/data.py | 110 ++++++++++++++++++++++++++++++++++++ wqflask/tests/base/test_data_set.py | 16 +++++- 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 wqflask/tests/base/data.py diff --git a/wqflask/tests/base/data.py b/wqflask/tests/base/data.py new file mode 100644 index 00000000..06a5a989 --- /dev/null +++ b/wqflask/tests/base/data.py @@ -0,0 +1,110 @@ +gen_menu_json = """ +{ + "datasets": { + "human": { + "HLC": { + "Liver mRNA": [ + [ + "320", + "HLC_0311", + "GSE9588 Human Liver Normal (Mar11) Both Sexes" + ] + ], + "Phenotypes": [ + [ + "635", + "HLCPublish", + "HLC Published Phenotypes" + ] + ] + } + }, + "mouse": { + "BXD": { + "Genotypes": [ + [ + "600", + "BXDGeno", + "BXD Genotypes" + ] + ], + "Hippocampus mRNA": [ + [ + "112", + "HC_M2_0606_P", + "Hippocampus Consortium M430v2 (Jun06) PDNN" + ] + ], + "Phenotypes": [ + [ + "602", + "BXDPublish", + "BXD Published Phenotypes" + ] + ] + } + } + }, + "groups": { + "human": [ + [ + "HLC", + "Liver: Normal Gene Expression with Genotypes (Merck)", + "Family:None" + ] + ], + "mouse": [ + [ + "BXD", + "BXD", + "Family:None" + ] + ] + }, + "species": [ + [ + "human", + "Human" + ], + [ + "mouse", + "Mouse" + ] + ], + "types": { + "human": { + "HLC": [ + [ + "Phenotypes", + "Traits and Cofactors", + "Phenotypes" + ], + [ + "Liver mRNA", + "Liver mRNA", + "Molecular Trait Datasets" + ] + ] + }, + "mouse": { + "BXD": [ + [ + "Phenotypes", + "Traits and Cofactors", + "Phenotypes" + ], + [ + "Genotypes", + "DNA Markers and SNPs", + "Genotypes" + ], + [ + "Hippocampus mRNA", + "Hippocampus mRNA", + "Molecular Trait Datasets" + ] + ] + } + } +} +""" diff --git a/wqflask/tests/base/test_data_set.py b/wqflask/tests/base/test_data_set.py index 74799e49..835d786a 100644 --- a/wqflask/tests/base/test_data_set.py +++ b/wqflask/tests/base/test_data_set.py @@ -4,7 +4,7 @@ import unittest import mock from wqflask import app - +from data import gen_menu_json from base.data_set import DatasetType @@ -43,3 +43,17 @@ class TestDataSetTypes(unittest.TestCase): self.assertEqual(DatasetType(redis_mock) ("All Phenotypes"), "Publish") + @mock.patch('base.data_set.requests.get') + def test_data_set_type_with_empty_redis(self, request_mock): + """Test that DatasetType returns correctly if the Redis Instance is empty and + the name variable exists in the dictionary + + """ + with app.app_context(): + request_mock.return_value.content = gen_menu_json + redis_mock = mock.Mock() + redis_mock.get.return_value = None + data_set = DatasetType(redis_mock) + self.assertEqual(data_set("BXDGeno"), "Geno") + self.assertEqual(data_set("BXDPublish"), "Publish") + self.assertEqual(data_set("HLC_0311"), "ProbeSet") -- cgit v1.2.3 From 957e05d33eb423df99181a99e7c25891810a21f7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Jul 2020 15:06:02 +0300 Subject: Check that Redis is called correctly * wqflask/tests/base/test_data_set.py: assert that `set` and `get` are called correctly --- wqflask/tests/base/test_data_set.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/tests/base/test_data_set.py b/wqflask/tests/base/test_data_set.py index 835d786a..3ac1b6d6 100644 --- a/wqflask/tests/base/test_data_set.py +++ b/wqflask/tests/base/test_data_set.py @@ -42,6 +42,7 @@ class TestDataSetTypes(unittest.TestCase): """ self.assertEqual(DatasetType(redis_mock) ("All Phenotypes"), "Publish") + redis_mock.get.assert_called_once_with("dataset_structure") @mock.patch('base.data_set.requests.get') def test_data_set_type_with_empty_redis(self, request_mock): @@ -57,3 +58,6 @@ class TestDataSetTypes(unittest.TestCase): self.assertEqual(data_set("BXDGeno"), "Geno") self.assertEqual(data_set("BXDPublish"), "Publish") self.assertEqual(data_set("HLC_0311"), "ProbeSet") + redis_mock.set.assert_called_once_with( + "dataset_structure", + '{"BXDGeno": "Geno", "BXDPublish": "Publish", "HLCPublish": "Publish", "HLC_0311": "ProbeSet", "HC_M2_0606_P": "ProbeSet"}') -- cgit v1.2.3 From 441e2dfa2772794673b69cc72f8561fc026d077b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Jul 2020 18:35:45 +0300 Subject: Use the correct redis instance inside object * wqflask/base/data_set.py (DatasetType): Use object's redis instance --- wqflask/base/data_set.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 5d562871..bce94aa8 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -93,7 +93,7 @@ Publish or ProbeSet. E.g. """ self.redis_instance = redis_instance self.datasets = {} - data = redis_instance.get("dataset_structure") + data = self.redis_instance.get("dataset_structure") if data: self.datasets = json.loads(data) else: # ZS: I don't think this should ever run unless Redis is emptied @@ -115,7 +115,7 @@ Publish or ProbeSet. E.g. except: pass - redis_instance.set("dataset_structure", json.dumps(self.datasets)) + self.redis_instance.set("dataset_structure", json.dumps(self.datasets)) # Set LOG_LEVEL_DEBUG=5 to see the following: logger.debugf(5, "datasets", self.datasets) -- cgit v1.2.3 From 79b8cf45c1c40b9c20278762b6e8f587a2820b43 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Jul 2020 19:10:36 +0300 Subject: Test that non-existent keys in Dataset are set correctly * wqflask/tests/base/test_data_set.py: Add more tests. --- wqflask/tests/base/test_data_set.py | 103 +++++++++++++++++++++++++++++++----- 1 file changed, 90 insertions(+), 13 deletions(-) diff --git a/wqflask/tests/base/test_data_set.py b/wqflask/tests/base/test_data_set.py index 3ac1b6d6..94780a5d 100644 --- a/wqflask/tests/base/test_data_set.py +++ b/wqflask/tests/base/test_data_set.py @@ -12,6 +12,19 @@ class TestDataSetTypes(unittest.TestCase): """Tests for the DataSetType class""" def setUp(self): + self.test_dataset = """ + { + "AD-cases-controls-MyersGeno": "Geno", + "AD-cases-controls-MyersPublish": "Publish", + "AKXDGeno": "Geno", + "AXBXAGeno": "Geno", + "AXBXAPublish": "Publish", + "Aging-Brain-UCIPublish": "Publish", + "All Phenotypes": "Publish", + "B139_K_1206_M": "ProbeSet", + "B139_K_1206_R": "ProbeSet" + } + """ self.app_context = app.app_context() self.app_context.push() @@ -27,19 +40,7 @@ class TestDataSetTypes(unittest.TestCase): with app.app_context(): db_mock.get = mock.Mock() redis_mock = mock.Mock() - redis_mock.get.return_value = """ - { - "AD-cases-controls-MyersGeno": "Geno", - "AD-cases-controls-MyersPublish": "Publish", - "AKXDGeno": "Geno", - "AXBXAGeno": "Geno", - "AXBXAPublish": "Publish", - "Aging-Brain-UCIPublish": "Publish", - "All Phenotypes": "Publish", - "B139_K_1206_M": "ProbeSet", - "B139_K_1206_R": "ProbeSet" - } - """ + redis_mock.get.return_value = self.test_dataset self.assertEqual(DatasetType(redis_mock) ("All Phenotypes"), "Publish") redis_mock.get.assert_called_once_with("dataset_structure") @@ -61,3 +62,79 @@ class TestDataSetTypes(unittest.TestCase): redis_mock.set.assert_called_once_with( "dataset_structure", '{"BXDGeno": "Geno", "BXDPublish": "Publish", "HLCPublish": "Publish", "HLC_0311": "ProbeSet", "HC_M2_0606_P": "ProbeSet"}') + + @mock.patch('base.data_set.g') + def test_set_dataset_key_mrna(self, db_mock): + with app.app_context(): + db_mock.db.execute.return_value = [1, 2, 3] + redis_mock = mock.Mock() + redis_mock.get.return_value = self.test_dataset + data_set = DatasetType(redis_mock) + data_set.set_dataset_key("mrna_expr", "Test") + self.assertEqual(data_set("Test"), "ProbeSet") + redis_mock.set.assert_called_once_with( + "dataset_structure", + '{"Aging-Brain-UCIPublish": "Publish", "AKXDGeno": "Geno", "B139_K_1206_M": "ProbeSet", "AD-cases-controls-MyersGeno": "Geno", "AD-cases-controls-MyersPublish": "Publish", "All Phenotypes": "Publish", "Test": "ProbeSet", "AXBXAPublish": "Publish", "B139_K_1206_R": "ProbeSet", "AXBXAGeno": "Geno"}') + expected_db_call = """""" + db_mock.db.execute.assert_called_with( + ("SELECT ProbeSetFreeze.Id FROM ProbeSetFreeze " + + "WHERE ProbeSetFreeze.Name = \"Test\" ") + ) + + @mock.patch('base.data_set.g') + def test_set_dataset_key_pheno(self, db_mock): + with app.app_context(): + db_mock.db.execute.return_value = [1, 2, 3] + redis_mock = mock.Mock() + redis_mock.get.return_value = self.test_dataset + data_set = DatasetType(redis_mock) + data_set.set_dataset_key("pheno", "Test") + self.assertEqual(data_set("Test"), "Publish") + redis_mock.set.assert_called_once_with( + "dataset_structure", + '{"Aging-Brain-UCIPublish": "Publish", "AKXDGeno": "Geno", "B139_K_1206_M": "ProbeSet", "AD-cases-controls-MyersGeno": "Geno", "AD-cases-controls-MyersPublish": "Publish", "All Phenotypes": "Publish", "Test": "Publish", "AXBXAPublish": "Publish", "B139_K_1206_R": "ProbeSet", "AXBXAGeno": "Geno"}') + expected_db_call = """""" + db_mock.db.execute.assert_called_with( + ("SELECT InfoFiles.GN_AccesionId " + + "FROM InfoFiles, PublishFreeze, InbredSet " + + "WHERE InbredSet.Name = 'Test' AND " + "PublishFreeze.InbredSetId = InbredSet.Id AND " + + "InfoFiles.InfoPageName = PublishFreeze.Name") + ) + + @mock.patch('base.data_set.g') + def test_set_dataset_other_pheno(self, db_mock): + with app.app_context(): + db_mock.db.execute.return_value = [1, 2, 3] + redis_mock = mock.Mock() + redis_mock.get.return_value = self.test_dataset + data_set = DatasetType(redis_mock) + data_set.set_dataset_key("other_pheno", "Test") + self.assertEqual(data_set("Test"), "Publish") + redis_mock.set.assert_called_once_with( + "dataset_structure", + '{"Aging-Brain-UCIPublish": "Publish", "AKXDGeno": "Geno", "B139_K_1206_M": "ProbeSet", "AD-cases-controls-MyersGeno": "Geno", "AD-cases-controls-MyersPublish": "Publish", "All Phenotypes": "Publish", "Test": "Publish", "AXBXAPublish": "Publish", "B139_K_1206_R": "ProbeSet", "AXBXAGeno": "Geno"}') + expected_db_call = """""" + db_mock.db.execute.assert_called_with( + ("SELECT PublishFreeze.Name " + + "FROM PublishFreeze, InbredSet " + + "WHERE InbredSet.Name = 'Test' AND " + "PublishFreeze.InbredSetId = InbredSet.Id") + ) + + @mock.patch('base.data_set.g') + def test_set_dataset_geno(self, db_mock): + with app.app_context(): + db_mock.db.execute.return_value = [1, 2, 3] + redis_mock = mock.Mock() + redis_mock.get.return_value = self.test_dataset + data_set = DatasetType(redis_mock) + data_set.set_dataset_key("geno", "Test") + self.assertEqual(data_set("Test"), "Geno") + redis_mock.set.assert_called_once_with( + "dataset_structure", + '{"Aging-Brain-UCIPublish": "Publish", "AKXDGeno": "Geno", "B139_K_1206_M": "ProbeSet", "AD-cases-controls-MyersGeno": "Geno", "AD-cases-controls-MyersPublish": "Publish", "All Phenotypes": "Publish", "Test": "Geno", "AXBXAPublish": "Publish", "B139_K_1206_R": "ProbeSet", "AXBXAGeno": "Geno"}') + expected_db_call = """""" + db_mock.db.execute.assert_called_with( + ("SELECT GenoFreeze.Id FROM GenoFreeze WHERE GenoFreeze.Name = \"Test\" ") + ) -- cgit v1.2.3 From fea8444fb7b0224fd44711853904822ee8b43f4f Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Jul 2020 19:11:11 +0300 Subject: Abstract away redundant code into set_dataset_key method * wqflask/base/data_set.py (Dataset): - Add set_dataset_key - Remove __call__ method --- wqflask/base/data_set.py | 109 ++++++++++++++++++++--------------------------- 1 file changed, 47 insertions(+), 62 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index bce94aa8..cfba9104 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -117,71 +117,56 @@ Publish or ProbeSet. E.g. self.redis_instance.set("dataset_structure", json.dumps(self.datasets)) - # Set LOG_LEVEL_DEBUG=5 to see the following: - logger.debugf(5, "datasets", self.datasets) + def set_dataset_key(self, t, name): + """If name is not in the object's dataset dictionary, set it, and update + dataset_structure in Redis + + args: + t: Type of dataset structure which can be: 'mrna_expr', 'pheno', + 'other_pheno', 'geno' + name: The name of the key to inserted in the datasets dictionary + + """ + sql_query_mapping = { + 'mrna_expr': ("""SELECT ProbeSetFreeze.Id FROM """ + + """ProbeSetFreeze WHERE ProbeSetFreeze.Name = "{}" """), + 'pheno': ("""SELECT InfoFiles.GN_AccesionId """ + + """FROM InfoFiles, PublishFreeze, InbredSet """ + + """WHERE InbredSet.Name = '{}' AND """ + + """PublishFreeze.InbredSetId = InbredSet.Id AND """ + + """InfoFiles.InfoPageName = PublishFreeze.Name"""), + 'other_pheno': ("""SELECT PublishFreeze.Name """ + + """FROM PublishFreeze, InbredSet """ + + """WHERE InbredSet.Name = '{}' AND """ + + """PublishFreeze.InbredSetId = InbredSet.Id"""), + 'geno': ("""SELECT GenoFreeze.Id FROM GenoFreeze WHERE """ + + """GenoFreeze.Name = "{}" """) + } + + dataset_name_mapping = { + "mrna_expr": "ProbeSet", + "pheno": "Publish", + "other_pheno": "Publish", + "geno": "Geno", + } + + if t in ['pheno', 'other_pheno']: + name = name.replace("Publish", "") + if bool(len(g.db.execute(sql_query_mapping[t].format(name)))): + self.datasets[name] = dataset_name_mapping[t] + self.redis_instance.set("dataset_structure", json.dumps(self.datasets)) + return True + + return None def __call__(self, name): + if name not in self.datasets: - mrna_expr_query = """ - SELECT - ProbeSetFreeze.Id - FROM - ProbeSetFreeze - WHERE - ProbeSetFreeze.Name = "{0}" - """.format(name) - - results = g.db.execute(mrna_expr_query).fetchall() - if len(results): - self.datasets[name] = "ProbeSet" - redis_instance.set("dataset_structure", json.dumps(self.datasets)) - return self.datasets[name] - - group_name = name.replace("Publish", "") - - pheno_query = """SELECT InfoFiles.GN_AccesionId - FROM InfoFiles, PublishFreeze, InbredSet - WHERE InbredSet.Name = '{0}' AND - PublishFreeze.InbredSetId = InbredSet.Id AND - InfoFiles.InfoPageName = PublishFreeze.Name""".format(group_name) - - results = g.db.execute(pheno_query).fetchall() - if len(results): - self.datasets[name] = "Publish" - redis_instance.set("dataset_structure", json.dumps(self.datasets)) - return self.datasets[name] - - # ZS: For when there isn't an InfoFiles ID; not sure if this and the preceding query are both necessary - other_pheno_query = """SELECT PublishFreeze.Name - FROM PublishFreeze, InbredSet - WHERE InbredSet.Name = '{}' AND - PublishFreeze.InbredSetId = InbredSet.Id""".format(group_name) - - results = g.db.execute(other_pheno_query).fetchall() - if len(results): - self.datasets[name] = "Publish" - redis_instance.set("dataset_structure", json.dumps(self.datasets)) - return self.datasets[name] - - geno_query = """ - SELECT - GenoFreeze.Id - FROM - GenoFreeze - WHERE - GenoFreeze.Name = "{0}" - """.format(name) - - results = g.db.execute(geno_query).fetchall() - if len(results): - self.datasets[name] = "Geno" - self.redis_instance.set("dataset_structure", json.dumps(self.datasets)) - return self.datasets[name] - - # ZS: It shouldn't ever reach this - return None - else: - return self.datasets[name] + for t in ["mrna_expr", "pheno", "other_pheno", "geno"]: + # This has side-effects, with the end result being a truth-y value + if(self.set_dataset_key(t, name)): + break + return self.datasets.get(name, None) # Return None if name has not been set # Do the intensive work at startup one time only -- cgit v1.2.3 From 7732204662bf395eb8ed55b6d26fd208998c1067 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Jul 2020 19:48:16 +0300 Subject: Add unittests for WebqtlCaseData * wqflask/tests/base/test_webqtl_case_data.py: Add it --- wqflask/tests/base/test_webqtl_case_data.py | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 wqflask/tests/base/test_webqtl_case_data.py diff --git a/wqflask/tests/base/test_webqtl_case_data.py b/wqflask/tests/base/test_webqtl_case_data.py new file mode 100644 index 00000000..8e8ba482 --- /dev/null +++ b/wqflask/tests/base/test_webqtl_case_data.py @@ -0,0 +1,39 @@ +"""Tests for wqflask/base/webqtlCaseData.py""" +import unittest + +from wqflask import app # Required because of utility.tools in webqtlCaseData.py +from base.webqtlCaseData import webqtlCaseData + +class TestWebqtlCaseData(unittest.TestCase): + """Tests for WebqtlCaseData class""" + + def setUp(self): + self.w = webqtlCaseData(name="Test", + value=0, + variance=0.0, + num_cases=10, + name2="Test2") + + def test_webqtl_case_data_repr(self): + self.assertEqual( + repr(self.w), + " value=0.000 variance=0.000 ndata=10 name=Test name2=Test2" + ) + + def test_class_outlier(self): + self.assertEqual(self.w.class_outlier, "") + + def test_display_value(self): + self.assertEqual(self.w.display_value, "0.000") + self.w.value = None + self.assertEqual(self.w.display_value, "x") + + def test_display_variance(self): + self.assertEqual(self.w.display_variance, "0.000") + self.w.variance = None + self.assertEqual(self.w.display_variance, "x") + + def test_display_num_cases(self): + self.assertEqual(self.w.display_num_cases, "10") + self.w.num_cases = None + self.assertEqual(self.w.display_num_cases, "x") -- cgit v1.2.3 From c1e22edf5f08b6d9c79a6608b8faf434578e246d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 27 Jul 2020 21:00:24 +0300 Subject: Clean up webqtlCaseData class * wqflask/base/webqtlCaseData.py (webqtlCaseData): - Remove obsolete 'Object' from Class inheritance - Replace 'str' with 'case_data_string' variable- it collides with python in-builts - Use pythonic 'is Not None' form - Remove redundancy in 'if' forms - Update copyright header --- wqflask/base/webqtlCaseData.py | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/wqflask/base/webqtlCaseData.py b/wqflask/base/webqtlCaseData.py index d8487f01..2844cedd 100644 --- a/wqflask/base/webqtlCaseData.py +++ b/wqflask/base/webqtlCaseData.py @@ -19,8 +19,7 @@ # This module is used by GeneNetwork project (www.genenetwork.org) # # Created by GeneNetwork Core Team 2010/08/10 -# -# Last updated by GeneNetwork Core Team 2010/10/20 + from utility.logger import getLogger logger = getLogger(__name__) @@ -29,7 +28,7 @@ import utility.tools utility.tools.show_settings() -class webqtlCaseData(object): +class webqtlCaseData: """one case data in one trait""" def __init__(self, name, value=None, variance=None, num_cases=None, name2=None): @@ -43,44 +42,40 @@ class webqtlCaseData(object): self.outlier = None # Not set to True/False until later def __repr__(self): - str = " " - if self.value != None: - str += "value=%2.3f" % self.value - if self.variance != None: - str += " variance=%2.3f" % self.variance + case_data_string = " " + if self.value is not None: + case_data_string += "value=%2.3f" % self.value + if self.variance is not None: + case_data_string += " variance=%2.3f" % self.variance if self.num_cases: - str += " ndata=%s" % self.num_cases + case_data_string += " ndata=%s" % self.num_cases if self.name: - str += " name=%s" % self.name + case_data_string += " name=%s" % self.name if self.name2: - str += " name2=%s" % self.name2 - return str + case_data_string += " name2=%s" % self.name2 + return case_data_string @property def class_outlier(self): """Template helper""" if self.outlier: return "outlier" - else: - return "" + return "" @property def display_value(self): - if self.value != None: + if self.value is not None: return "%2.3f" % self.value - else: - return "x" + return "x" @property def display_variance(self): - if self.variance != None: + if self.variance is not None: return "%2.3f" % self.variance - else: - return "x" + return "x" @property def display_num_cases(self): - if self.num_cases != None: + if self.num_cases is not None: return "%s" % self.num_cases - else: - return "x" + return "x" -- cgit v1.2.3