diff options
| -rw-r--r-- | doc/Architecture.org | 44 | ||||
| -rw-r--r-- | doc/README.org | 1 | ||||
| -rw-r--r-- | etc/default_settings.py | 23 | ||||
| -rwxr-xr-x | wqflask/maintenance/gen_select_dataset.py | 11 | ||||
| -rw-r--r-- | wqflask/utility/tools.py | 29 | ||||
| -rw-r--r-- | wqflask/wqflask/search_results.py | 2 | ||||
| -rwxr-xr-x | wqflask/wqflask/static/new/javascript/dataset_menu_structure.json | 2 | ||||
| -rwxr-xr-x | wqflask/wqflask/static/new/javascript/dataset_select_menu.coffee | 129 | ||||
| -rw-r--r-- | wqflask/wqflask/static/new/javascript/dataset_select_menu.js | 91 | ||||
| -rw-r--r-- | wqflask/wqflask/views.py | 20 |
10 files changed, 150 insertions, 202 deletions
diff --git a/doc/Architecture.org b/doc/Architecture.org index c263d3b9..e8ffe69b 100644 --- a/doc/Architecture.org +++ b/doc/Architecture.org @@ -1,5 +1,13 @@ * GeneNetwork Architecture +#+TITLE: Installing GeneNetwork services + +* Table of Contents :TOC: + - [[#genenetwork-architecture][GeneNetwork Architecture]] + - [[#introduction][Introduction]] + - [[#webserver][Webserver]] + - [[#gnserver-rest][GnServer (REST)]] + ** Introduction This document describes the architecture of GN2. Because GN2 is @@ -7,9 +15,33 @@ evolving, only a high-level overview is given here. ** Webserver -The webserver is built on [[http://flask.pocoo.org/][Python flask]] and this GN2 source code can be -found on [[https://github.com/genenetwork/genenetwork2/tree/master/wqflask/wqflask][github]] in the wqflask directory. The routing tables are -defined in [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/views.py][views.py]]. For example the main page is loaded from a -template named [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/templates/index_page.htm][index_page.html]] in the [[https://github.com/genenetwork/genenetwork2/tree/master/wqflask/wqflask/templates][templates]] directory. In the -template you can find get the form gets filled by a Javascript -routine defined in [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/static/new/javascript/dataset_select_menu.js][data_select_menu.js]]. +The main [[https://github.com/genenetwork/genenetwork2][GN2 webserver]] is built on [[http://flask.pocoo.org/][Python flask]] and this GN2 source +code can be found on [[https://github.com/genenetwork/genenetwork2/tree/master/wqflask/wqflask][github]] in the wqflask directory. The routing +tables are defined in [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/views.py][views.py]]. For example the main page is loaded +from a template named [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/templates/index_page.htm][index_page.html]] in the [[https://github.com/genenetwork/genenetwork2/tree/master/wqflask/wqflask/templates][templates]] directory. In +the template you can find get the form gets filled by a Javascript +routine defined in [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/static/new/javascript/dataset_select_menu.js][data_select_menu.js]] which picks up a static JSON +file for the menu. This static file is generated with +[[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/maintenance/gen_select_dataset.py][gen_select_dataset.py]]. Note that this JSON data is served by +gn_server in the latest version, see [[#gnserver-rest][GnServer (REST)]]. + +When you hit a search with, for example, +'http://localhost:5003/search?species=mouse&group=BXD&type=Hippocampus+mRNA&dataset=HC_M2_0606_P&search_terms_or=&search_terms_and=MEAN%3D%2815+16%29+LRS%3D%2823+46%29+&FormID=searchResult' +it has the menu items as parameters. According to the routing table, +the search is executed and Redis caching is used (we'll probably +change that to gn_server). The logic is in search_result.py which +invokes database functions in +wqflask/dbFunction/webqtlDatabaseFunction.py, for example. The +receiving template lives at [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/templates/search_result_page.html][search_result_page.html]]. + +** GnServer (REST) + +The [[https://github.com/genenetwork/gn_server][GnServer REST API]] is built on high performance [[http://elixir-lang.org/][Elixir]] with [[https://github.com/falood/maru][Maru]]. +Mainly the GnServer serves JSON requests, for example to fetch data +from the database. To get the menu data in YAML you can do something like + +: curl localhost:8880/int/menu/main.json|ruby extra/json2yaml.rb + +(json2yaml.rb is in the gn_server repo). + + diff --git a/doc/README.org b/doc/README.org index 3754dbbd..b3c78f29 100644 --- a/doc/README.org +++ b/doc/README.org @@ -1,4 +1,3 @@ - #+TITLE: Installing GeneNetwork services * Table of Contents :TOC: diff --git a/etc/default_settings.py b/etc/default_settings.py index 0cf40265..bc464861 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -1,7 +1,18 @@ +# Default settings file defines a single Flask process for the Python +# webserver running in developer mode with limited console +# output. Copy this file and run it from ./bin/genenetwork2 configfile +# +# Note that these settings are fetched in ./wqflask/utilities/tools.py +# which has support for overriding them through environment variables, +# e.g. +# +# env LOG_SQL=True USE_REDIS=False ./bin/genenetwork2 + import os import sys -LOGFILE = "/tmp/genenetwork2.log" +HOME=os.environ['HOME'] +LOGFILE = HOME+"/genenetwork2.log" # This is needed because Flask turns key errors into a # 400 bad request response with no exception/log @@ -22,8 +33,16 @@ SQLALCHEMY_POOL_RECYCLE = 3600 SERVER_PORT = 5003 SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc' +# Behavioural settings (defaults) note that logger and log levels can +# be overridden at the module level +WEBSERVER_MODE = 'DEV' # Python webserver mode (DEBUG|DEV|PROD) +LOGGING = 'WARNING' # Logger mode (DEBUG|INFO|WARNING|ERROR|CRITICAL) +DEBUG_LOG_LEVEL = 1 # Debug log level (0-5) +LOG_SQL = False # Log SQL/backend calls +USE_REDIS = True # REDIS caching (note that redis will be phased out) + # Path overrides for Genenetwork -GENENETWORK_FILES = os.environ['HOME']+"/gn2_data" +GENENETWORK_FILES = HOME+"/gn2_data" PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read()) PLINK_COMMAND = str.strip(os.popen("which plink2").read()) GEMMA_COMMAND = str.strip(os.popen("which gemma").read()) diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py index b1459bf3..f91d3d88 100755 --- a/wqflask/maintenance/gen_select_dataset.py +++ b/wqflask/maintenance/gen_select_dataset.py @@ -5,6 +5,7 @@ It needs to be run manually when database has been changed. """ + # Copyright (C) University of Tennessee Health Science Center, Memphis, TN. # # This program is free software: you can redistribute it and/or modify it @@ -36,11 +37,11 @@ from __future__ import print_function, division #print("cdict is:", cdict) import sys -import zach_settings # no hard code paths! +# import zach_settings # no hard code paths! -import MySQLdb +# import MySQLdb -import simplejson as json +# import simplejson as json import urlparse @@ -54,8 +55,8 @@ from pprint import pformat as pf #conn = Engine.connect() - - +print('ERROR: This conversion is now OBSOLETE as the menu gets built from the database in Javascript using GN_SERVER instead!') +sys.exit() def parse_db_uri(db_uri): diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 11441d7a..6e9f6d4a 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -7,12 +7,12 @@ from wqflask import app def get_setting(command_id,guess=None): """Resolve a setting from the environment or the global settings in - app.config, with get_valid_path is a function checking whether the + app.config, with valid_path is a function checking whether the path points to an expected directory and returns the full path to the binary command guess = os.environ.get('HOME')+'/pylmm' - get_setting('PYLMM_PATH',guess) + valid_path(get_setting('PYLMM_PATH',guess)) first tries the environment variable in +id+, next gets the Flask app setting for the same +id+ and finally does an educated @@ -31,13 +31,13 @@ def get_setting(command_id,guess=None): """ def value(command): if command: - sys.stderr.write("Found path "+command+"\n") + # sys.stderr.write("Found "+command+"\n") return command else: return None # ---- Check whether environment exists - sys.stderr.write("Looking for "+command_id+"\n") + # sys.stderr.write("Looking for "+command_id+"\n") command = value(os.environ.get(command_id)) if not command: # ---- Check whether setting exists in app @@ -45,7 +45,8 @@ def get_setting(command_id,guess=None): if not command: command = value(guess) if not command: - raise Exception(command_id+' path unknown or faulty (update settings.py?). '+command_id+' should point to the path') + raise Exception(command_id+' setting unknown or faulty (update settings.py?).') + sys.stderr.write("Set "+command_id+"="+str(command)+"\n") return command def valid_bin(bin): @@ -128,10 +129,16 @@ def locate_ignore_error(name, subdir=None): def tempdir(): return valid_path(get_setting("TEMPDIR","/tmp")) - # Cached values -PYLMM_COMMAND = pylmm_command() -GEMMA_COMMAND = gemma_command() -PLINK_COMMAND = plink_command() -FLAT_FILES = flat_files() -TEMPDIR = tempdir() +WEBSERVER_MODE = get_setting('WEBSERVER_MODE') +LOGGING = get_setting('LOGGING') +DEBUG_LOG_LEVEL = get_setting('DEBUG_LOG_LEVEL') +LOG_SQL = get_setting('LOG_SQL') in [True,'TRUE','True','true'] +USE_REDIS = get_setting('USE_REDIS') in [True,'TRUE','True','true'] + +PYLMM_COMMAND = pylmm_command() +GEMMA_COMMAND = gemma_command() +PLINK_COMMAND = plink_command() +FLAT_FILES = flat_files() +TEMPDIR = tempdir() + diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 4c4e1ef2..f04881a6 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -31,7 +31,7 @@ from base.data_set import create_dataset from base.trait import GeneralTrait from wqflask import parser from wqflask import do_search -from utility import webqtlUtil +from utility import webqtlUtil,tools from dbFunction import webqtlDatabaseFunction from utility import formatting diff --git a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json index 12a30e84..4ff90ca8 100755 --- a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json +++ b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json @@ -4827,4 +4827,4 @@ ] } } -} \ No newline at end of file +} diff --git a/wqflask/wqflask/static/new/javascript/dataset_select_menu.coffee b/wqflask/wqflask/static/new/javascript/dataset_select_menu.coffee deleted file mode 100755 index 83264da3..00000000 --- a/wqflask/wqflask/static/new/javascript/dataset_select_menu.coffee +++ /dev/null @@ -1,129 +0,0 @@ -$ -> - - ## Handle menu changes - - process_json = (data) -> - window.jdata = data - populate_species() - apply_default() - - $.ajax '/static/new/javascript/dataset_menu_structure.json', - dataType: 'json' - success: process_json - - populate_species = -> - species_list = @jdata.species - redo_dropdown($('#species'), species_list) - populate_group() - window.populate_species = populate_species - - populate_group = -> - console.log("in populate group") - species = $('#species').val() - group_list = @jdata.groups[species] - redo_dropdown($('#group'), group_list) - populate_type() - window.populate_group = populate_group - - populate_type = -> - species = $('#species').val() - group = $('#group').val() - type_list = @jdata.types[species][group] - redo_dropdown($('#type'), type_list) - populate_dataset() - window.populate_type = populate_type - - populate_dataset = -> - species = $('#species').val() - group = $('#group').val() - type = $('#type').val() - console.log("sgt:", species, group, type) - dataset_list = @jdata.datasets[species][group][type] - console.log("pop_dataset:", dataset_list) - redo_dropdown($('#dataset'), dataset_list) - window.populate_dataset = populate_dataset - - redo_dropdown = (dropdown, items) -> - console.log("in redo:", dropdown, items) - dropdown.empty() - for item in items - dropdown.append($("<option />").val(item[0]).text(item[1])) - - $('#species').change => - populate_group() - - $('#group').change => - populate_type() - - $('#type').change => - populate_dataset() - - ## Info buttons - - open_window = (url, name) -> - options = "menubar=1,toolbar=1,location=1,resizable=1,status=1,scrollbars=1,directories=1,width=900" - open(url, name, options).focus() - - # Link to info on selected group; use of "Cross" - # in the url is outdated and should be changed to group - group_info = -> - species = $('#species').val() - group = $('#group').val() - url = "/" + species + "Cross.html#" + group - open_window(url, "Group Info") - - $('#group_info').click(group_info) - - # Link to dataset info - dataset_info = -> - dataset = $('#dataset').val() - url = "/webqtl/main.py?FormID=sharinginfo&InfoPageName=" + dataset - open_window(url, "Dataset Info") - - $('#dataset_info').click(dataset_info) - - - ## Handle setting new default drop downs - - make_default = -> - holder = {} - for item in ['species', 'group', 'type', 'dataset'] - holder[item] = $("##{item}").val() - jholder = JSON.stringify(holder) - $.cookie('search_defaults', jholder, - expires: 365) - - apply_default = -> - defaults = $.cookie('search_defaults') - if defaults - # defaults are stored as a JSON string in a cookie - defaults = $.parseJSON(defaults) - else - # If user hasn't set a default we use this - # (Most of GN's data is from BXD mice) - defaults = - species: "mouse" - group: "BXD" - type: "Hippocampus mRNA" - dataset: "HC_M2_0606_P" - - for item in [['species', 'group'] - ['group', 'type'] - ['type', 'dataset'], - ['dataset', null]] - $("##{item[0]}").val(defaults[item[0]]) - - if item[1] - populate_function = "populate_" + item[1] - console.log("Calling:", populate_function) - window[populate_function]() - - check_search_term = -> - search_term = $('#tfor').val() - console.log("search_term:", search_term) - if (search_term == "") - alert("Please enter one or more search terms or search equations.") - return false - - $("#make_default").click(make_default) - $("#btsearch").click(check_search_term) \ No newline at end of file diff --git a/wqflask/wqflask/static/new/javascript/dataset_select_menu.js b/wqflask/wqflask/static/new/javascript/dataset_select_menu.js index 1fe4cf75..d4110e11 100644 --- a/wqflask/wqflask/static/new/javascript/dataset_select_menu.js +++ b/wqflask/wqflask/static/new/javascript/dataset_select_menu.js @@ -1,54 +1,67 @@ -// Generated by CoffeeScript 1.8.0 $(function() { - var apply_default, check_search_term, dataset_info, group_info, make_default, open_window, populate_dataset, populate_group, populate_species, populate_type, process_json, redo_dropdown; - process_json = function(data) { - window.jdata = data; + var gndata; // loaded once for all to use + process_json = function(data) { + populate_species(); + return apply_default(); + }; + $.getJSON("http://localhost:8880/int/menu/main.json", + function(data) { + gndata = data; + console.log("***** GOT DATA from GN_SERVER ****"); + console.log(gndata); populate_species(); - return apply_default(); - }; - $.ajax('/static/new/javascript/dataset_menu_structure.json', { - dataType: 'json', - success: process_json + }).error(function() { + console.log("ERROR: GN_SERVER not responding"); + alert("ERROR: GN_SERVER internal REST API is not responding"); }); - populate_species = function() { - var species_list; - species_list = this.jdata.species; + + var populate_species = function() { + var species_list = Object.keys(gndata.menu).map(function(species) { + var mitem = gndata.menu[species].menu + // console.log("Species menu:",species,mitem) + return [species,mitem]; + }); redo_dropdown($('#species'), species_list); return populate_group(); }; window.populate_species = populate_species; - populate_group = function() { - var group_list, species; - console.log("in populate group"); - species = $('#species').val(); - group_list = this.jdata.groups[species]; - redo_dropdown($('#group'), group_list); + + var populate_group = function() { + var species = $('#species').val(); + var groups = gndata.groups[species].map(function(item) { + console.log("group:",item); + return item.slice(1,3); + }) + redo_dropdown($('#group'), groups); return populate_type(); }; window.populate_group = populate_group; - populate_type = function() { - var group, species, type_list; - console.log("in populate type"); - species = $('#species').val(); - group = $('#group').val(); - type_list = this.jdata.types[species][group]; + + var populate_type = function() { + var species = $('#species').val(); + var group = $('#group').val(); + var type_list = gndata.menu[species].types[group].map(function(item) { + return [item,item]; + }); + redo_dropdown($('#type'), type_list); return populate_dataset(); }; window.populate_type = populate_type; - populate_dataset = function() { - var dataset_list, group, species, type; - console.log("in populate dataset"); - species = $('#species').val(); - group = $('#group').val(); - type = $('#type').val(); - console.log("sgt:", species, group, type); - dataset_list = this.jdata.datasets[species][group][type]; - console.log("pop_dataset:", dataset_list); + + var populate_dataset = function() { + var species = $('#species').val(); + var group = $('#group').val(); + var type = $('#type').val(); + var dataset_list = gndata.datasets[species][group][type].map(function(item) { + return item.slice(1,3); + }) + return redo_dropdown($('#dataset'), dataset_list); }; window.populate_dataset = populate_dataset; - redo_dropdown = function(dropdown, items) { + + var redo_dropdown = function(dropdown, items) { var item, _i, _len, _results; console.log("in redo:", dropdown, items); dropdown.empty(); @@ -83,7 +96,7 @@ $(function() { options = "menubar=1,toolbar=1,location=1,resizable=1,status=1,scrollbars=1,directories=1,width=900"; return open(url, name, options).focus(); }; - group_info = function() { + var group_info = function() { var group, species, url; species = $('#species').val(); group = $('#group').val(); @@ -91,14 +104,14 @@ $(function() { return open_window(url, "Group Info"); }; $('#group_info').click(group_info); - dataset_info = function() { + var dataset_info = function() { var dataset, url; accession_id = $('#dataset option:selected').data("id"); url = "http://genenetwork.org/webqtl/main.py?FormID=sharinginfo&GN_AccessionId=" + accession_id; return open_window(url, "Dataset Info"); }; $('#dataset_info').click(dataset_info); - make_default = function() { + var make_default = function() { var holder, item, jholder, _i, _len, _ref; alert("The current settings are now your default.") holder = {}; @@ -112,7 +125,7 @@ $(function() { expires: 365 }); }; - apply_default = function() { + var apply_default = function() { var defaults, item, populate_function, _i, _len, _ref, _results; defaults = $.cookie('search_defaults'); if (defaults) { @@ -140,7 +153,7 @@ $(function() { } return _results; }; - check_search_term = function() { + var check_search_term = function() { var or_search_term, and_search_term; or_search_term = $('#or_search').val(); and_search_term = $('#and_search').val(); diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index f88a9cdb..ba250d4f 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -52,7 +52,7 @@ from wqflask.wgcna import wgcna_analysis from wqflask.ctl import ctl_analysis from utility import temp_data -from utility.tools import TEMPDIR +from utility.tools import TEMPDIR,USE_REDIS from base import webqtlFormData from base.webqtlConfig import GENERATED_IMAGE_DIR @@ -138,11 +138,16 @@ def search_page(): else: key = "search_results:v1:" + json.dumps(request.args, sort_keys=True) print("key is:", pf(key)) - with Bench("Loading cache"): - result = Redis.get(key) + if USE_REDIS: + with Bench("Trying Redis cache"): + result = Redis.get(key) + else: + print("Skipping Redis cache (USE_REDIS=False)") + result = None if result: - print("Cache hit!!!") + print("Cache hit on search results!!!") + print("USE_REDIS=",USE_REDIS) with Bench("Loading results"): result = pickle.loads(result) else: @@ -152,8 +157,9 @@ def search_page(): result = the_search.__dict__ print("result: ", pf(result)) - Redis.set(key, pickle.dumps(result, pickle.HIGHEST_PROTOCOL)) - Redis.expire(key, 60*60) + if USE_REDIS: + Redis.set(key, pickle.dumps(result, pickle.HIGHEST_PROTOCOL)) + Redis.expire(key, 60*60) if result['search_term_exists']: return render_template("search_result_page.html", **result) @@ -179,7 +185,7 @@ def gsearch_updating(): # return render_template("gsearch_gene_updating.html", **result) # elif type == "phenotype": # return render_template("gsearch_pheno.html", **result) - + @app.route("/docedit") def docedit(): doc = docs.Docs(request.args['entry']) |