aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--doc/Architecture.org44
-rw-r--r--doc/README.org1
-rw-r--r--etc/default_settings.py23
-rwxr-xr-xwqflask/maintenance/gen_select_dataset.py11
-rw-r--r--wqflask/utility/tools.py29
-rw-r--r--wqflask/wqflask/search_results.py2
-rwxr-xr-xwqflask/wqflask/static/new/javascript/dataset_menu_structure.json2
-rwxr-xr-xwqflask/wqflask/static/new/javascript/dataset_select_menu.coffee129
-rw-r--r--wqflask/wqflask/static/new/javascript/dataset_select_menu.js91
-rw-r--r--wqflask/wqflask/views.py20
10 files changed, 150 insertions, 202 deletions
diff --git a/doc/Architecture.org b/doc/Architecture.org
index c263d3b9..e8ffe69b 100644
--- a/doc/Architecture.org
+++ b/doc/Architecture.org
@@ -1,5 +1,13 @@
* GeneNetwork Architecture
+#+TITLE: Installing GeneNetwork services
+
+* Table of Contents :TOC:
+ - [[#genenetwork-architecture][GeneNetwork Architecture]]
+ - [[#introduction][Introduction]]
+ - [[#webserver][Webserver]]
+ - [[#gnserver-rest][GnServer (REST)]]
+
** Introduction
This document describes the architecture of GN2. Because GN2 is
@@ -7,9 +15,33 @@ evolving, only a high-level overview is given here.
** Webserver
-The webserver is built on [[http://flask.pocoo.org/][Python flask]] and this GN2 source code can be
-found on [[https://github.com/genenetwork/genenetwork2/tree/master/wqflask/wqflask][github]] in the wqflask directory. The routing tables are
-defined in [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/views.py][views.py]]. For example the main page is loaded from a
-template named [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/templates/index_page.htm][index_page.html]] in the [[https://github.com/genenetwork/genenetwork2/tree/master/wqflask/wqflask/templates][templates]] directory. In the
-template you can find get the form gets filled by a Javascript
-routine defined in [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/static/new/javascript/dataset_select_menu.js][data_select_menu.js]].
+The main [[https://github.com/genenetwork/genenetwork2][GN2 webserver]] is built on [[http://flask.pocoo.org/][Python flask]] and this GN2 source
+code can be found on [[https://github.com/genenetwork/genenetwork2/tree/master/wqflask/wqflask][github]] in the wqflask directory. The routing
+tables are defined in [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/views.py][views.py]]. For example the main page is loaded
+from a template named [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/templates/index_page.htm][index_page.html]] in the [[https://github.com/genenetwork/genenetwork2/tree/master/wqflask/wqflask/templates][templates]] directory. In
+the template you can find get the form gets filled by a Javascript
+routine defined in [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/static/new/javascript/dataset_select_menu.js][data_select_menu.js]] which picks up a static JSON
+file for the menu. This static file is generated with
+[[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/maintenance/gen_select_dataset.py][gen_select_dataset.py]]. Note that this JSON data is served by
+gn_server in the latest version, see [[#gnserver-rest][GnServer (REST)]].
+
+When you hit a search with, for example,
+'http://localhost:5003/search?species=mouse&group=BXD&type=Hippocampus+mRNA&dataset=HC_M2_0606_P&search_terms_or=&search_terms_and=MEAN%3D%2815+16%29+LRS%3D%2823+46%29+&FormID=searchResult'
+it has the menu items as parameters. According to the routing table,
+the search is executed and Redis caching is used (we'll probably
+change that to gn_server). The logic is in search_result.py which
+invokes database functions in
+wqflask/dbFunction/webqtlDatabaseFunction.py, for example. The
+receiving template lives at [[https://github.com/genenetwork/genenetwork2/blob/master/wqflask/wqflask/templates/search_result_page.html][search_result_page.html]].
+
+** GnServer (REST)
+
+The [[https://github.com/genenetwork/gn_server][GnServer REST API]] is built on high performance [[http://elixir-lang.org/][Elixir]] with [[https://github.com/falood/maru][Maru]].
+Mainly the GnServer serves JSON requests, for example to fetch data
+from the database. To get the menu data in YAML you can do something like
+
+: curl localhost:8880/int/menu/main.json|ruby extra/json2yaml.rb
+
+(json2yaml.rb is in the gn_server repo).
+
+
diff --git a/doc/README.org b/doc/README.org
index 3754dbbd..b3c78f29 100644
--- a/doc/README.org
+++ b/doc/README.org
@@ -1,4 +1,3 @@
-
#+TITLE: Installing GeneNetwork services
* Table of Contents :TOC:
diff --git a/etc/default_settings.py b/etc/default_settings.py
index 0cf40265..bc464861 100644
--- a/etc/default_settings.py
+++ b/etc/default_settings.py
@@ -1,7 +1,18 @@
+# Default settings file defines a single Flask process for the Python
+# webserver running in developer mode with limited console
+# output. Copy this file and run it from ./bin/genenetwork2 configfile
+#
+# Note that these settings are fetched in ./wqflask/utilities/tools.py
+# which has support for overriding them through environment variables,
+# e.g.
+#
+# env LOG_SQL=True USE_REDIS=False ./bin/genenetwork2
+
import os
import sys
-LOGFILE = "/tmp/genenetwork2.log"
+HOME=os.environ['HOME']
+LOGFILE = HOME+"/genenetwork2.log"
# This is needed because Flask turns key errors into a
# 400 bad request response with no exception/log
@@ -22,8 +33,16 @@ SQLALCHEMY_POOL_RECYCLE = 3600
SERVER_PORT = 5003
SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc'
+# Behavioural settings (defaults) note that logger and log levels can
+# be overridden at the module level
+WEBSERVER_MODE = 'DEV' # Python webserver mode (DEBUG|DEV|PROD)
+LOGGING = 'WARNING' # Logger mode (DEBUG|INFO|WARNING|ERROR|CRITICAL)
+DEBUG_LOG_LEVEL = 1 # Debug log level (0-5)
+LOG_SQL = False # Log SQL/backend calls
+USE_REDIS = True # REDIS caching (note that redis will be phased out)
+
# Path overrides for Genenetwork
-GENENETWORK_FILES = os.environ['HOME']+"/gn2_data"
+GENENETWORK_FILES = HOME+"/gn2_data"
PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read())
PLINK_COMMAND = str.strip(os.popen("which plink2").read())
GEMMA_COMMAND = str.strip(os.popen("which gemma").read())
diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py
index b1459bf3..f91d3d88 100755
--- a/wqflask/maintenance/gen_select_dataset.py
+++ b/wqflask/maintenance/gen_select_dataset.py
@@ -5,6 +5,7 @@ It needs to be run manually when database has been changed.
"""
+
# Copyright (C) University of Tennessee Health Science Center, Memphis, TN.
#
# This program is free software: you can redistribute it and/or modify it
@@ -36,11 +37,11 @@ from __future__ import print_function, division
#print("cdict is:", cdict)
import sys
-import zach_settings # no hard code paths!
+# import zach_settings # no hard code paths!
-import MySQLdb
+# import MySQLdb
-import simplejson as json
+# import simplejson as json
import urlparse
@@ -54,8 +55,8 @@ from pprint import pformat as pf
#conn = Engine.connect()
-
-
+print('ERROR: This conversion is now OBSOLETE as the menu gets built from the database in Javascript using GN_SERVER instead!')
+sys.exit()
def parse_db_uri(db_uri):
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 11441d7a..6e9f6d4a 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -7,12 +7,12 @@ from wqflask import app
def get_setting(command_id,guess=None):
"""Resolve a setting from the environment or the global settings in
- app.config, with get_valid_path is a function checking whether the
+ app.config, with valid_path is a function checking whether the
path points to an expected directory and returns the full path to
the binary command
guess = os.environ.get('HOME')+'/pylmm'
- get_setting('PYLMM_PATH',guess)
+ valid_path(get_setting('PYLMM_PATH',guess))
first tries the environment variable in +id+, next gets the Flask
app setting for the same +id+ and finally does an educated
@@ -31,13 +31,13 @@ def get_setting(command_id,guess=None):
"""
def value(command):
if command:
- sys.stderr.write("Found path "+command+"\n")
+ # sys.stderr.write("Found "+command+"\n")
return command
else:
return None
# ---- Check whether environment exists
- sys.stderr.write("Looking for "+command_id+"\n")
+ # sys.stderr.write("Looking for "+command_id+"\n")
command = value(os.environ.get(command_id))
if not command:
# ---- Check whether setting exists in app
@@ -45,7 +45,8 @@ def get_setting(command_id,guess=None):
if not command:
command = value(guess)
if not command:
- raise Exception(command_id+' path unknown or faulty (update settings.py?). '+command_id+' should point to the path')
+ raise Exception(command_id+' setting unknown or faulty (update settings.py?).')
+ sys.stderr.write("Set "+command_id+"="+str(command)+"\n")
return command
def valid_bin(bin):
@@ -128,10 +129,16 @@ def locate_ignore_error(name, subdir=None):
def tempdir():
return valid_path(get_setting("TEMPDIR","/tmp"))
-
# Cached values
-PYLMM_COMMAND = pylmm_command()
-GEMMA_COMMAND = gemma_command()
-PLINK_COMMAND = plink_command()
-FLAT_FILES = flat_files()
-TEMPDIR = tempdir()
+WEBSERVER_MODE = get_setting('WEBSERVER_MODE')
+LOGGING = get_setting('LOGGING')
+DEBUG_LOG_LEVEL = get_setting('DEBUG_LOG_LEVEL')
+LOG_SQL = get_setting('LOG_SQL') in [True,'TRUE','True','true']
+USE_REDIS = get_setting('USE_REDIS') in [True,'TRUE','True','true']
+
+PYLMM_COMMAND = pylmm_command()
+GEMMA_COMMAND = gemma_command()
+PLINK_COMMAND = plink_command()
+FLAT_FILES = flat_files()
+TEMPDIR = tempdir()
+
diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py
index 4c4e1ef2..f04881a6 100644
--- a/wqflask/wqflask/search_results.py
+++ b/wqflask/wqflask/search_results.py
@@ -31,7 +31,7 @@ from base.data_set import create_dataset
from base.trait import GeneralTrait
from wqflask import parser
from wqflask import do_search
-from utility import webqtlUtil
+from utility import webqtlUtil,tools
from dbFunction import webqtlDatabaseFunction
from utility import formatting
diff --git a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json
index 12a30e84..4ff90ca8 100755
--- a/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json
+++ b/wqflask/wqflask/static/new/javascript/dataset_menu_structure.json
@@ -4827,4 +4827,4 @@
]
}
}
-} \ No newline at end of file
+}
diff --git a/wqflask/wqflask/static/new/javascript/dataset_select_menu.coffee b/wqflask/wqflask/static/new/javascript/dataset_select_menu.coffee
deleted file mode 100755
index 83264da3..00000000
--- a/wqflask/wqflask/static/new/javascript/dataset_select_menu.coffee
+++ /dev/null
@@ -1,129 +0,0 @@
-$ ->
-
- ## Handle menu changes
-
- process_json = (data) ->
- window.jdata = data
- populate_species()
- apply_default()
-
- $.ajax '/static/new/javascript/dataset_menu_structure.json',
- dataType: 'json'
- success: process_json
-
- populate_species = ->
- species_list = @jdata.species
- redo_dropdown($('#species'), species_list)
- populate_group()
- window.populate_species = populate_species
-
- populate_group = ->
- console.log("in populate group")
- species = $('#species').val()
- group_list = @jdata.groups[species]
- redo_dropdown($('#group'), group_list)
- populate_type()
- window.populate_group = populate_group
-
- populate_type = ->
- species = $('#species').val()
- group = $('#group').val()
- type_list = @jdata.types[species][group]
- redo_dropdown($('#type'), type_list)
- populate_dataset()
- window.populate_type = populate_type
-
- populate_dataset = ->
- species = $('#species').val()
- group = $('#group').val()
- type = $('#type').val()
- console.log("sgt:", species, group, type)
- dataset_list = @jdata.datasets[species][group][type]
- console.log("pop_dataset:", dataset_list)
- redo_dropdown($('#dataset'), dataset_list)
- window.populate_dataset = populate_dataset
-
- redo_dropdown = (dropdown, items) ->
- console.log("in redo:", dropdown, items)
- dropdown.empty()
- for item in items
- dropdown.append($("<option />").val(item[0]).text(item[1]))
-
- $('#species').change =>
- populate_group()
-
- $('#group').change =>
- populate_type()
-
- $('#type').change =>
- populate_dataset()
-
- ## Info buttons
-
- open_window = (url, name) ->
- options = "menubar=1,toolbar=1,location=1,resizable=1,status=1,scrollbars=1,directories=1,width=900"
- open(url, name, options).focus()
-
- # Link to info on selected group; use of "Cross"
- # in the url is outdated and should be changed to group
- group_info = ->
- species = $('#species').val()
- group = $('#group').val()
- url = "/" + species + "Cross.html#" + group
- open_window(url, "Group Info")
-
- $('#group_info').click(group_info)
-
- # Link to dataset info
- dataset_info = ->
- dataset = $('#dataset').val()
- url = "/webqtl/main.py?FormID=sharinginfo&InfoPageName=" + dataset
- open_window(url, "Dataset Info")
-
- $('#dataset_info').click(dataset_info)
-
-
- ## Handle setting new default drop downs
-
- make_default = ->
- holder = {}
- for item in ['species', 'group', 'type', 'dataset']
- holder[item] = $("##{item}").val()
- jholder = JSON.stringify(holder)
- $.cookie('search_defaults', jholder,
- expires: 365)
-
- apply_default = ->
- defaults = $.cookie('search_defaults')
- if defaults
- # defaults are stored as a JSON string in a cookie
- defaults = $.parseJSON(defaults)
- else
- # If user hasn't set a default we use this
- # (Most of GN's data is from BXD mice)
- defaults =
- species: "mouse"
- group: "BXD"
- type: "Hippocampus mRNA"
- dataset: "HC_M2_0606_P"
-
- for item in [['species', 'group']
- ['group', 'type']
- ['type', 'dataset'],
- ['dataset', null]]
- $("##{item[0]}").val(defaults[item[0]])
-
- if item[1]
- populate_function = "populate_" + item[1]
- console.log("Calling:", populate_function)
- window[populate_function]()
-
- check_search_term = ->
- search_term = $('#tfor').val()
- console.log("search_term:", search_term)
- if (search_term == "")
- alert("Please enter one or more search terms or search equations.")
- return false
-
- $("#make_default").click(make_default)
- $("#btsearch").click(check_search_term) \ No newline at end of file
diff --git a/wqflask/wqflask/static/new/javascript/dataset_select_menu.js b/wqflask/wqflask/static/new/javascript/dataset_select_menu.js
index 1fe4cf75..d4110e11 100644
--- a/wqflask/wqflask/static/new/javascript/dataset_select_menu.js
+++ b/wqflask/wqflask/static/new/javascript/dataset_select_menu.js
@@ -1,54 +1,67 @@
-// Generated by CoffeeScript 1.8.0
$(function() {
- var apply_default, check_search_term, dataset_info, group_info, make_default, open_window, populate_dataset, populate_group, populate_species, populate_type, process_json, redo_dropdown;
- process_json = function(data) {
- window.jdata = data;
+ var gndata; // loaded once for all to use
+ process_json = function(data) {
+ populate_species();
+ return apply_default();
+ };
+ $.getJSON("http://localhost:8880/int/menu/main.json",
+ function(data) {
+ gndata = data;
+ console.log("***** GOT DATA from GN_SERVER ****");
+ console.log(gndata);
populate_species();
- return apply_default();
- };
- $.ajax('/static/new/javascript/dataset_menu_structure.json', {
- dataType: 'json',
- success: process_json
+ }).error(function() {
+ console.log("ERROR: GN_SERVER not responding");
+ alert("ERROR: GN_SERVER internal REST API is not responding");
});
- populate_species = function() {
- var species_list;
- species_list = this.jdata.species;
+
+ var populate_species = function() {
+ var species_list = Object.keys(gndata.menu).map(function(species) {
+ var mitem = gndata.menu[species].menu
+ // console.log("Species menu:",species,mitem)
+ return [species,mitem];
+ });
redo_dropdown($('#species'), species_list);
return populate_group();
};
window.populate_species = populate_species;
- populate_group = function() {
- var group_list, species;
- console.log("in populate group");
- species = $('#species').val();
- group_list = this.jdata.groups[species];
- redo_dropdown($('#group'), group_list);
+
+ var populate_group = function() {
+ var species = $('#species').val();
+ var groups = gndata.groups[species].map(function(item) {
+ console.log("group:",item);
+ return item.slice(1,3);
+ })
+ redo_dropdown($('#group'), groups);
return populate_type();
};
window.populate_group = populate_group;
- populate_type = function() {
- var group, species, type_list;
- console.log("in populate type");
- species = $('#species').val();
- group = $('#group').val();
- type_list = this.jdata.types[species][group];
+
+ var populate_type = function() {
+ var species = $('#species').val();
+ var group = $('#group').val();
+ var type_list = gndata.menu[species].types[group].map(function(item) {
+ return [item,item];
+ });
+
redo_dropdown($('#type'), type_list);
return populate_dataset();
};
window.populate_type = populate_type;
- populate_dataset = function() {
- var dataset_list, group, species, type;
- console.log("in populate dataset");
- species = $('#species').val();
- group = $('#group').val();
- type = $('#type').val();
- console.log("sgt:", species, group, type);
- dataset_list = this.jdata.datasets[species][group][type];
- console.log("pop_dataset:", dataset_list);
+
+ var populate_dataset = function() {
+ var species = $('#species').val();
+ var group = $('#group').val();
+ var type = $('#type').val();
+ var dataset_list = gndata.datasets[species][group][type].map(function(item) {
+ return item.slice(1,3);
+ })
+
return redo_dropdown($('#dataset'), dataset_list);
};
window.populate_dataset = populate_dataset;
- redo_dropdown = function(dropdown, items) {
+
+ var redo_dropdown = function(dropdown, items) {
var item, _i, _len, _results;
console.log("in redo:", dropdown, items);
dropdown.empty();
@@ -83,7 +96,7 @@ $(function() {
options = "menubar=1,toolbar=1,location=1,resizable=1,status=1,scrollbars=1,directories=1,width=900";
return open(url, name, options).focus();
};
- group_info = function() {
+ var group_info = function() {
var group, species, url;
species = $('#species').val();
group = $('#group').val();
@@ -91,14 +104,14 @@ $(function() {
return open_window(url, "Group Info");
};
$('#group_info').click(group_info);
- dataset_info = function() {
+ var dataset_info = function() {
var dataset, url;
accession_id = $('#dataset option:selected').data("id");
url = "http://genenetwork.org/webqtl/main.py?FormID=sharinginfo&GN_AccessionId=" + accession_id;
return open_window(url, "Dataset Info");
};
$('#dataset_info').click(dataset_info);
- make_default = function() {
+ var make_default = function() {
var holder, item, jholder, _i, _len, _ref;
alert("The current settings are now your default.")
holder = {};
@@ -112,7 +125,7 @@ $(function() {
expires: 365
});
};
- apply_default = function() {
+ var apply_default = function() {
var defaults, item, populate_function, _i, _len, _ref, _results;
defaults = $.cookie('search_defaults');
if (defaults) {
@@ -140,7 +153,7 @@ $(function() {
}
return _results;
};
- check_search_term = function() {
+ var check_search_term = function() {
var or_search_term, and_search_term;
or_search_term = $('#or_search').val();
and_search_term = $('#and_search').val();
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index f88a9cdb..ba250d4f 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -52,7 +52,7 @@ from wqflask.wgcna import wgcna_analysis
from wqflask.ctl import ctl_analysis
from utility import temp_data
-from utility.tools import TEMPDIR
+from utility.tools import TEMPDIR,USE_REDIS
from base import webqtlFormData
from base.webqtlConfig import GENERATED_IMAGE_DIR
@@ -138,11 +138,16 @@ def search_page():
else:
key = "search_results:v1:" + json.dumps(request.args, sort_keys=True)
print("key is:", pf(key))
- with Bench("Loading cache"):
- result = Redis.get(key)
+ if USE_REDIS:
+ with Bench("Trying Redis cache"):
+ result = Redis.get(key)
+ else:
+ print("Skipping Redis cache (USE_REDIS=False)")
+ result = None
if result:
- print("Cache hit!!!")
+ print("Cache hit on search results!!!")
+ print("USE_REDIS=",USE_REDIS)
with Bench("Loading results"):
result = pickle.loads(result)
else:
@@ -152,8 +157,9 @@ def search_page():
result = the_search.__dict__
print("result: ", pf(result))
- Redis.set(key, pickle.dumps(result, pickle.HIGHEST_PROTOCOL))
- Redis.expire(key, 60*60)
+ if USE_REDIS:
+ Redis.set(key, pickle.dumps(result, pickle.HIGHEST_PROTOCOL))
+ Redis.expire(key, 60*60)
if result['search_term_exists']:
return render_template("search_result_page.html", **result)
@@ -179,7 +185,7 @@ def gsearch_updating():
# return render_template("gsearch_gene_updating.html", **result)
# elif type == "phenotype":
# return render_template("gsearch_pheno.html", **result)
-
+
@app.route("/docedit")
def docedit():
doc = docs.Docs(request.args['entry'])