From 0408cc61181eaeda95bd1f60f76dc5798e315663 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Wed, 5 Apr 2017 15:37:29 +0000
Subject: - Temporary traits can be created, viewed, and added to collections -
 Correct PCA trait data is created but can't be saved yet - Added inner
 margins by increasing xDomain and yDomain of probability plot on trait page -
 Increased X/Y-axis label font size - Turned "processes" to 0 on runserver.py
 for PROD setting, since it doesn't work with threading - Improved appearance
 of correlation page table - Added links to github to index page - Removed
 js_data from GEMMA/PLINK mapping results, since it isn't used for those -
 Removed "Tissue" from trait page for phenotype traits

---
 wqflask/runserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'wqflask/runserver.py')

diff --git a/wqflask/runserver.py b/wqflask/runserver.py
index 0342b7ad..a6ae28af 100644
--- a/wqflask/runserver.py
+++ b/wqflask/runserver.py
@@ -50,5 +50,5 @@ else: # staging/production modes
             debug=False,
             use_debugger=False,
             threaded=True,
-            processes=8,
+            processes=0,
             use_reloader=True)
-- 
cgit v1.2.3


From 3c46a58f50f339ec56027db6d2108de5b21d63ac Mon Sep 17 00:00:00 2001
From: zsloan
Date: Tue, 25 Apr 2017 16:52:47 +0000
Subject: Fixed issue with correlation result trait links if target dataset is
 different from base dataset

Improved collection list and view table appearances

Added question mark image for glossary entries in table headers

Fixed issue where extra decimal place sometimes appeared in bar chart Y axis tick values
---
 wqflask/runserver.py                               |   3 ++
 .../wqflask/static/new/images/question_mark.jpg    | Bin 0 -> 47158 bytes
 wqflask/wqflask/static/new/javascript/bar_chart.js |  11 ++++++
 wqflask/wqflask/templates/collections/list.html    |   9 ++++-
 wqflask/wqflask/templates/collections/view.html    |   8 -----
 wqflask/wqflask/templates/correlation_page.html    |   2 +-
 wqflask/wqflask/templates/search_result_page.html  |  39 +++++++++++----------
 7 files changed, 43 insertions(+), 29 deletions(-)
 create mode 100644 wqflask/wqflask/static/new/images/question_mark.jpg

(limited to 'wqflask/runserver.py')

diff --git a/wqflask/runserver.py b/wqflask/runserver.py
index a6ae28af..50805643 100644
--- a/wqflask/runserver.py
+++ b/wqflask/runserver.py
@@ -11,6 +11,9 @@ import logging
 import utility.logger
 logger = utility.logger.getLogger(__name__ )
 
+import signal
+signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
 BLUE  = '\033[94m'
 GREEN = '\033[92m'
 BOLD  = '\033[1m'
diff --git a/wqflask/wqflask/static/new/images/question_mark.jpg b/wqflask/wqflask/static/new/images/question_mark.jpg
new file mode 100644
index 00000000..82df7e81
Binary files /dev/null and b/wqflask/wqflask/static/new/images/question_mark.jpg differ
diff --git a/wqflask/wqflask/static/new/javascript/bar_chart.js b/wqflask/wqflask/static/new/javascript/bar_chart.js
index 7ec35148..d8540580 100644
--- a/wqflask/wqflask/static/new/javascript/bar_chart.js
+++ b/wqflask/wqflask/static/new/javascript/bar_chart.js
@@ -253,6 +253,17 @@
             })())
           ]);
           console.log("values: ", values);
+
+          decimal_exists = "False";
+          for(i=0; i < values.length; i++){
+              if (values[i]['y'] % 1 != 0){
+                  decimal_exists = "True";
+                  break;
+              }
+          }
+          if (decimal_exists == "False"){
+              _this.chart.yAxis.tickFormat(d3.format('d'))
+          }
           d3.select("#bar_chart_container svg").datum([
             {
               values: values
diff --git a/wqflask/wqflask/templates/collections/list.html b/wqflask/wqflask/templates/collections/list.html
index 5a30c98c..b1284895 100644
--- a/wqflask/wqflask/templates/collections/list.html
+++ b/wqflask/wqflask/templates/collections/list.html
@@ -51,7 +51,7 @@
             <tbody>
             {% for uc in collections %}
                 <tr class="collection_line">
-                    <td style="padding-right: 0px;"><INPUT TYPE="checkbox" NAME="collection" class="checkbox trait_checkbox" VALUE="{{ uc.id }}"></td>
+                    <td style="padding-left: 8px; padding-right: 0px; padding-top: 4px; align: center;"><INPUT TYPE="checkbox" NAME="collection" class="checkbox trait_checkbox" VALUE="{{ uc.id }}"></td>
                     <td align="right">{{ loop.index }}
                     {% if g.user_session.user_ob %}
                     <td><a class="collection_name" href="{{ url_for('view_collection', uc_id=uc.id) }}">{{ uc.name }}</a></td>
@@ -89,6 +89,13 @@
     {% endif %}
     <script>
             $('#trait_table').dataTable( {
+                "drawCallback": function( settings ) {
+                     $('#trait_table tr').click(function(event) {
+                         if (event.target.type !== 'checkbox') {
+                             $(':checkbox', this).trigger('click');
+                         }
+                     });
+                },
                 "columns": [
                     { "type": "natural", "width": "3%" },
                     { "type": "natural", "width": "8%" },
diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html
index 094bc273..cae03b4a 100644
--- a/wqflask/wqflask/templates/collections/view.html
+++ b/wqflask/wqflask/templates/collections/view.html
@@ -181,14 +181,6 @@
                     {
                         extend: 'columnsToggle',
                         columns: ':not(:first-child)',
-                        columnText: function ( dt, idx, title ) {
-                            console.log("TITLE:", title)
-                            if (title == "Additive Effect  ?"){
-                                return "Additive Effect"
-                            } else {
-                                return title;
-                            }
-                        },
                         postfixButtons: [ 'colvisRestore' ]
                     }
                 ],
diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html
index 3bde7dc7..f5fe2120 100644
--- a/wqflask/wqflask/templates/correlation_page.html
+++ b/wqflask/wqflask/templates/correlation_page.html
@@ -104,7 +104,7 @@
                         <td>
                             <a href="{{ url_for('show_trait_page',
                                     trait_id = trait.name,
-                                    dataset = dataset.name
+                                    dataset = trait.dataset.name
                                     )}}">
                                 {{ trait.name }}
                             </a>
diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html
index d4ec18e3..93e24cc7 100644
--- a/wqflask/wqflask/templates/search_result_page.html
+++ b/wqflask/wqflask/templates/search_result_page.html
@@ -77,16 +77,16 @@
           {% endif %}
 -->
 
-          <div id="table_container" style="width: {% if dataset.type == 'ProbeSet' %}1300{% elif dataset.type == 'Publish' %}1200{% elif dataset.type == 'Geno' %}500{% endif %}px;">
+          <div id="table_container" style="width: {% if dataset.type == 'ProbeSet' %}1300{% elif dataset.type == 'Publish' %}1300{% elif dataset.type == 'Geno' %}400{% endif %}px;">
             <table class="display dataTable nowrap" id='trait_table' style="float: left;">
                 <thead>
                     <tr>
                         <th></th>
                     {% for header in header_fields %}
                         {% if header == 'Max LRS' %}
-                        <th data-export="Max LRS">Max LRS</th>
+                        <th data-export="Max LRS">Max LRS <a href="http://genenetwork.org//glossary.html#LRS" target="_blank"><img style="width: 15px; height: 15px;" src="/static/new/images/question_mark.jpg"></a></th>
                         {% elif header == 'Additive Effect' %}
-                        <th data-export="Additive Effect">Additive <a href="http://genenetwork.org//glossary.html#A" target="_blank"><sup style="color:#f00"> ?</sup></a></th>
+                        <th data-export="Additive Effect">Additive Effect <a href="http://genenetwork.org//glossary.html#A" target="_blank"><img style="width: 15px; height: 15px;" src="/static/new/images/question_mark.jpg"></a></th>
                         {% else %}
                         <th data-export="{{header}}">{{header}}</th>
                         {% endif %}
@@ -99,9 +99,9 @@
                         <th></th>
                     {% for header in header_fields %}
                         {% if header == 'Max LRS' %}
-                        <th data-export="Max LRS">Max LRS</th>
+                        <th data-export="Max LRS">Max LRS <a href="http://genenetwork.org//glossary.html#LRS" target="_blank"><img style="width: 15px; height: 15px;" src="/static/new/images/question_mark.jpg"></a></th>
                         {% elif header == 'Additive Effect' %}
-                        <th data-export="Additive Effect">Additive Effect<a href="http://genenetwork.org//glossary.html#A" target="_blank"><sup style="color:#f00"> ?</sup></a></th>
+                        <th data-export="Additive Effect">Additive Effect <a href="http://genenetwork.org//glossary.html#A" target="_blank"><img style="width: 15px; height: 15px;" src="/static/new/images/question_mark.jpg"></a></th>
                         {% else %}
                         <th data-export="{{header}}">{{header}}</th>
                         {% endif %}
@@ -252,6 +252,7 @@
                      $('.trait_checkbox:checkbox').on("change", change_buttons);
                 },
                 "createdRow": function ( row, data, index ) {
+                    $('td', row).eq(0).attr('style', 'padding-left: 8px; padding-right: 0px; padding-top: 4px; align: center;');
                     $('td', row).eq(1).attr('align', 'right');
                     $('td', row).eq(1).attr('data-export', index+1);
                     $('td', row).eq(2).attr('data-export', $('td', row).eq(2).text());
@@ -281,12 +282,12 @@
                     { "type": "natural", "width": "2%" },
                     { "type": "natural", "width": "5%" },
                     { "type": "natural", "width": "6%" },
+                    { "type": "natural" },
                     { "type": "natural", "width": "30%" },
-                    { "type": "natural", "width": "25%" },
                     { "type": "natural", "width": "5%" },
                     { "type": "natural", "width": "6%" },
                     { "type": "natural", "width": "10%" },
-                    { "type": "natural", "width": "9%" }
+                    { "type": "natural", "width": "8%" }
                 ],
                 "columnDefs": [ {
                     "targets": 0,
@@ -295,8 +296,7 @@
                 "order": [[1, "asc" ]],
                 buttons: [
                     {
-                        extend: 'colvis',
-                        text: 'Show or Hide Columns',
+                        extend: 'columnsToggle',
                         columns: ':not(:first-child)',
                         postfixButtons: [ 'colvisRestore' ]
                     }
@@ -306,10 +306,10 @@
                 "autoWidth": false,
                 "deferRender": true,
                 "bSortClasses": false,
-                "scrollY": "600px",
-                "scrollCollapse": true,
-                "scroller": true,
-                "paging": false
+                "scrollCollapse": false,
+                "scroller": false,
+                "paging": false,
+                "orderClasses": true
             } );
             {% elif dataset.type == 'Geno' %}
             $('#trait_table').DataTable( {
@@ -322,6 +322,7 @@
                      $('.trait_checkbox:checkbox').on("change", change_buttons);
                 },
                 "createdRow": function ( row, data, index ) {
+                    $('td', row).eq(0).attr('style', 'padding-left: 8px; padding-right: 0px; padding-top: 4px; align: center;');
                     $('td', row).eq(1).attr('align', 'right');
                     $('td', row).eq(1).attr('data-export', index+1);
                     $('td', row).eq(2).attr('data-export', $('td', row).eq(2).text());
@@ -329,10 +330,10 @@
                 },
                 "data": json_trait_list,
                 "columns": [
-                    { "type": "natural", "width": "8%" },
+                    { "type": "natural", "width": "5%" },
                     { "type": "natural", "width": "12%" },
-                    { "type": "natural", "width": "40%" },
-                    { "type": "natural", "width": "40%"}
+                    { "type": "natural" },
+                    { "type": "natural", "width": "30%"}
                 ],
                 "columnDefs": [ {
                     "targets": 0,
@@ -341,8 +342,7 @@
                 "order": [[1, "asc" ]],
                 buttons: [
                     {
-                        extend: 'colvis',
-                        text: 'Show or Hide Columns',
+                        extend: 'columnsToggle',
                         columns: ':not(:first-child)',
                         postfixButtons: [ 'colvisRestore' ]
                     }
@@ -355,7 +355,8 @@
                 "scrollY": "600px",
                 "scrollCollapse": true,
                 "scroller": true,
-                "paging": false
+                "paging": false,
+                "orderClasses": true
             } );
             {% endif %}
 
-- 
cgit v1.2.3


From 837ce8b9c779b1da6e4390220ccd353b289c1fca Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 20 Dec 2017 09:11:31 -0600
Subject: Bring in some earlier work - mostly documentation and startup
 handling

---
 VERSION                   |   2 +-
 bin/genenetwork2          |   9 +++
 doc/README.org            |  26 ++++++--
 doc/database.org          | 165 ++++++++++++++++++++++++++++++++++++++++++++--
 etc/default_settings.py   |  11 +++-
 wqflask/base/data_set.py  |   2 +-
 wqflask/runserver.py      |  12 +++-
 wqflask/utility/logger.py |   2 +-
 wqflask/utility/tools.py  |  27 ++++++--
 9 files changed, 233 insertions(+), 23 deletions(-)

(limited to 'wqflask/runserver.py')

diff --git a/VERSION b/VERSION
index 3e0b7cab..6a5fe6e8 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.10-pre4
+2.11
diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 5e791885..a7edb1c2 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -1,5 +1,11 @@
 #! /bin/bash
 #
+# Typical usage
+#
+#   env GN2_PROFILE=~/opt/genenetwork2-phewas ./bin/genenetwork2
+#
+# Where GN2_PROFILE points to the GNU Guix profile used for deployment.
+#
 # This will run the GN2 server (with default settings if none
 # supplied). Typically you need a GNU Guix profile which is set with
 # an environment variable (this profile is dictated by the
@@ -120,6 +126,9 @@ echo -n "dir $TMPDIR
 dbfilename gn2.rdb
 " | redis-server - &
 
+# Overrides for packages that are not yet public (currently r-auwerx)
+export R_LIBS_SITE=$R_LIBS_SITE:$HOME/.Rlibs/das1i1pm54dj6lbdcsw5w0sdwhccyj1a-r-3.3.2/lib/R/lib
+
 # Start the flask server running GN2
 cd $GN2_BASE_DIR/wqflask
 echo "Starting with $settings"
diff --git a/doc/README.org b/doc/README.org
index a39ef603..937a9549 100644
--- a/doc/README.org
+++ b/doc/README.org
@@ -104,11 +104,29 @@ As root configure and run
 :  mysqld --datadir=/var/mysql --initialize-insecure
 :  mkdir -p /var/run/mysqld
 :  chown mysql.mysql ~/mysql /var/run/mysqld
-:  su mysql -c mysqld --datadir=/var/mysql --explicit_defaults_for_timestamp -P 12048
+:  mysqld -u mysql --datadir=/var/mysql --explicit_defaults_for_timestamp -P 12048"
 
-/etc/my.cnf
-[mysqld]
-user=root
+If you want to run as root you may have to set
+
+: /etc/my.cnf
+: [mysqld]
+: user=root
+
+To check error output in a file on start-up run with something like
+
+: mysqld -u mysql --console  --explicit_defaults_for_timestamp  --datadir=/gnu/mysql --log-error=~/test.log
+
+Other tips are that Guix installs mysqld in your profile, so this may work
+
+: /home/user/.guix-profile/bin/mysqld -u mysql --explicit_defaults_for_timestamp  --datadir=/gnu/mysql
+
+When you get errors like:
+
+: qlalchemy.exc.IntegrityError: (_mysql_exceptions.IntegrityError) (1215, 'Cannot add foreign key constraint')
+
+you may need to set
+
+: set foreign_key_checks=0
 
 ** Load the small database in MySQL
 
diff --git a/doc/database.org b/doc/database.org
index 624174a4..5107b660 100644
--- a/doc/database.org
+++ b/doc/database.org
@@ -1,9 +1,19 @@
-- github Document reduction issue
+* Database Information
+
+WARNING: This document contains information on the GN databases which
+will change over time. The GN database is currently MySQL based and,
+while efficient, contains a number of design choices we want to grow
+'out' of. Especially with an eye on reproducibility we want to
+introduce versioning.
+
+So do not treat the information in this document as a final way of
+accessing data. It is better to use the
+[[https://github.com/genenetwork/gn_server/blob/master/doc/API.md][REST API]].
 
 * The small test database (2GB)
 
 The default install comes with a smaller database which includes a
-number of the BSD's and the Human liver dataset (GSE9588).
+number of the BXD's and the Human liver dataset (GSE9588).
 
 * GeneNetwork database
 
@@ -750,9 +760,30 @@ show indexes from ProbeSetFreeze;
 |      1 |        5 | 0.303492 |
 +--------+----------+----------+
 
-** Publication and publishdata (all pheno)
+** Publication
+
+Publication:
+
+| Id   | PubMed_ID | Abstract    | Title   | Pages   | Month | Year |
+
 
-Phenotype pubs
+** Publishdata (all pheno)
+
+One of three phenotype tables.
+
+mysql> select * from PublishData limit 5;
++---------+----------+-------+
+| Id      | StrainId | value |
++---------+----------+-------+
+| 8966353 |      349 |  29.6 |
+| 8966353 |      350 |  27.8 |
+| 8966353 |      351 |  26.6 |
+| 8966353 |      352 |  28.5 |
+| 8966353 |      353 |  24.6 |
++---------+----------+-------+
+5 rows in set (0.25 sec)
+
+See below for phenotype access.
 
 ** QuickSearch
 
@@ -1073,7 +1104,37 @@ select * from ProbeSetXRef limit 5;
 i.e., for Strain Id 1 (DataId) 1, the locus '10.095.400' has a
 phenotype value of 5.742.
 
-GeneNetwork1 already has a limited REST interface, if you do
+Interestingly ProbeData and PublishData have the same layout as
+ProbeSetData. ProbeData is only in use for Affy assays - and not used
+for computations. PublishData contains trait values. ProbeSetData.id
+matches ProbeSetXRef.DataId while PublishData.id matches
+PublishXRef.DataId.
+
+select * from PublishXRef limit 3;
++-------+-------------+-------------+---------------+---------+----------------+------------------+-----------+----------+-------------------------------------------------------+
+| Id    | InbredSetId | PhenotypeId | PublicationId | DataId  | Locus          | LRS              | additive  | Sequence | comments                                              |
++-------+-------------+-------------+---------------+---------+----------------+------------------+-----------+----------+-------------------------------------------------------+
+| 10001 |           8 |           1 |             1 | 8966353 | D2Mit5         |   10.18351644706 |  -1.20875 |        1 |                                                       |
+| 10001 |           7 |           2 |            53 | 8966813 | D7Mit25UT      | 9.85534330983917 |  -2.86875 |        1 |                                                       |
+| 10001 |           4 |           3 |            81 | 8966947 | CEL-6_57082524 | 11.7119505898121 | -23.28875 |        1 | elissa modified Abstract at Tue Jun  7 11:38:00 2005  |
++-------+-------------+-------------+---------------+---------+----------------+------------------+-----------+----------+-------------------------------------------------------+
+3 rows in set (0.00 sec)
+
+ties the trait data (PublishData) with the inbredsetid (matching
+PublishFreeze.InbredSetId), locus and publication.
+
+select * from PublishFreeze -> ;
++----+------------+--------------------------+-------------+------------+--------+-------------+-----------------+-----------------+
+| Id | Name       | FullName                 | ShortName   | CreateTime | public | InbredSetId | confidentiality | AuthorisedUsers |
++----+------------+--------------------------+-------------+------------+--------+-------------+-----------------+-----------------+
+|  1 | BXDPublish | BXD Published Phenotypes | BXDPublish  | 2004-07-17 |      2 |           1 |               0 | NULL            |
+| 18 | HLCPublish | HLC Published Phenotypes | HLC Publish | 2012-02-20 |      2 |          34 |               0 | NULL            |
++----+------------+--------------------------+-------------+------------+--------+-------------+-----------------+-----------------+
+2 rows in set (0.02 sec)
+
+which gives us the datasets.
+
+GeneNetwork1 has a limited REST interface, if you do
 
 : curl "http://robot.genenetwork.org/webqtl/main.py?cmd=get&probeset=1443823_s_at&db=HC_M2_0606_P"
 
@@ -1082,6 +1143,9 @@ we get
 : ProbeSetID      B6D2F1  C57BL/6J        DBA/2J  BXD1    BXD2    BXD5    BXD6   BXD8     BXD9    BXD11   BXD12   BXD13   BXD15   BXD16   BXD19   BXD20   BXD21  BXD22    BXD23   BXD24   BXD27   BXD28   BXD29   BXD31   BXD32   BXD33   BXD34  BXD38    BXD39   BXD40   BXD42   BXD67   BXD68   BXD43   BXD44   BXD45   BXD48  BXD50    BXD51   BXD55   BXD60   BXD61   BXD62   BXD63   BXD64   BXD65   BXD66  BXD69    BXD70   BXD73   BXD74   BXD75   BXD76   BXD77   BXD79   BXD73a  BXD83  BXD84    BXD85   BXD86   BXD87   BXD89   BXD90   BXD65b  BXD93   BXD94   A/J    AKR/J    C3H/HeJ C57BL/6ByJ      CXB1    CXB2    CXB3    CXB4    CXB5    CXB6   CXB7     CXB8    CXB9    CXB10   CXB11   CXB12   CXB13   BXD48a  129S1/SvImJ    BALB/cJ  BALB/cByJ       LG/J    NOD/ShiLtJ      PWD/PhJ BXD65a  BXD98   BXD99  CAST/EiJ KK/HlJ  WSB/EiJ NZO/HlLtJ       PWK/PhJ D2B6F1
 : 1443823_s_at    15.251  15.626  14.716  15.198  14.918  15.057  15.232  14.968 14.87    15.084  15.192  14.924  15.343  15.226  15.364  15.36   14.792  14.908 15.344   14.948  15.08   15.021  15.176  15.14   14.796  15.443  14.636  14.921 15.22    15.62   14.816  15.39   15.428  14.982  15.05   15.13   14.722  14.636 15.242   15.527  14.825  14.416  15.125  15.362  15.226  15.176  15.328  14.895 15.141   15.634  14.922  14.764  15.122  15.448  15.398  15.089  14.765  15.234 15.302   14.774  14.979  15.212  15.29   15.012  15.041  15.448  14.34   14.338 14.809   15.046  14.816  15.232  14.933  15.255  15.21   14.766  14.8    15.506 15.749   15.274  15.599  15.673  14.651  14.692  14.552  14.563  14.164  14.546 15.044   14.695  15.162  14.772  14.645  15.493  14.75   14.786  15.003  15.148 15.221
 
+(see https://github.com/genenetwork/gn_server/blob/master/doc/API.md
+for the latest REST API).
+
 getTraitData is defined in the file [[https://github.com/genenetwork/genenetwork/blob/master/web/webqtl/textUI/cmdClass.py#L134][web/webqtl/textUI/cmdClass.py]].
 probe is None, so the code at line 199 is run
 
@@ -1165,6 +1229,97 @@ select * from ProbeSetData limit 5;
 5 rows in set (0.00 sec)
 
 linked by ProbeSetXRef.dataid.
+
+*** For PublishData:
+
+List datasets for BXD (InbredSetId=1):
+
+select * from PublishXRef where InbredSetId=1 limit 3;
++-------+-------------+-------------+---------------+---------+-----------+------------------+------------------+----------+--------------------------------------------------------------------------------+
+| Id    | InbredSetId | PhenotypeId | PublicationId | DataId  | Locus     | LRS              | additive         | Sequence | comments                                                                       |
++-------+-------------+-------------+---------------+---------+-----------+------------------+------------------+----------+--------------------------------------------------------------------------------+
+| 10001 |           1 |           4 |           116 | 8967043 | rs8253516 | 13.4974914158039 | 2.39444444444444 |        1 | robwilliams modified post_publication_description at Mon Jul 30 14:58:10 2012
+ |
+| 10002 |           1 |          10 |           116 | 8967044 | rs3666069 | 22.0042692151629 | 2.08178571428572 |        1 | robwilliams modified phenotype at Thu Oct 28 21:43:28 2010
+                    |
+| 10003 |           1 |          15 |           116 | 8967045 | D18Mit4   | 15.5929163293343 | 19.0882352941176 |        1 | robwilliams modified phenotype at Mon May 23 20:52:19 2011
+                    |
++-------+-------------+-------------+---------------+---------+-----------+------------------+------------------+----------+--------------------------------------------------------------------------------+
+
+where ID is the 'record' or, effectively, dataset.
+
+select distinct(publicationid) from PublishXRef where InbredSetId=1 limit 3;
++---------------+
+| publicationid |
++---------------+
+|           116 |
+|           117 |
+|           118 |
++---------------+
+
+select distinct
+PublishXRef.id,publicationid,phenotypeid,Phenotype.post_publication_description
+from PublishXRef,Phenotype where InbredSetId=1 and
+phenotypeid=Phenotype.id limit 3;
++-------+---------------+-------------+----------------------------------------------------------------------------------------------------------------------------+
+| id    | publicationid | phenotypeid | post_publication_description                                                                                               |
++-------+---------------+-------------+----------------------------------------------------------------------------------------------------------------------------+
+| 10001 |           116 |           4 | Central nervous system, morphology: Cerebellum weight [mg]                                                                 |
+| 10002 |           116 |          10 | Central nervous system, morphology: Cerebellum weight after adjustment for covariance with brain size [mg]                 |
+| 10003 |           116 |          15 | Central nervous system, morphology: Brain weight, male and female adult average, unadjusted for body weight, age, sex [mg] |
++-------+---------------+-------------+----------------------------------------------------------------------------------------------------------------------------+
+
+The id field is the same that is used in the GN2 web interface and the
+PublicationID ties the datasets together.
+
+To list trait values:
+
+SELECT Strain.Name, PublishData.id, PublishData.value from
+(Strain,PublishData, PublishXRef) Where PublishData.StrainId =
+Strain.id limit 3;
+
++------+---------+-------+
+| Name | id      | value |
++------+---------+-------+
+| CXB1 | 8966353 |  29.6 |
+| CXB1 | 8966353 |  29.6 |
+| CXB1 | 8966353 |  29.6 |
++------+---------+-------+
+
+here id should match dataid again:
+
+SELECT Strain.Name, PublishData.id, PublishData.value from
+(Strain,PublishData, PublishXRef) Where PublishData.StrainId =
+Strain.id and PublishXRef.dataid=8967043 and
+PublishXRef.dataid=PublishData.id limit 3;
++------+---------+-------+
+| Name | id      | value |
++------+---------+-------+
+| BXD1 | 8967043 |  61.4 |
+| BXD2 | 8967043 |    49 |
+| BXD5 | 8967043 |  62.5 |
++------+---------+-------+
+
+*** Datasets
+
+The REST API aims to present a unified interface for genotype and
+phenotype data. Phenotype datasets appear in two major forms in the
+database and we want to present them as one resource.
+
+Dataset names are defined in ProbeSetFreeze.name and Published.id ->
+publication (we'll ignore the probe dataset that uses
+ProbeFreeze.name). These tables should be meshed. It looks like the
+ids are non-overlapping with the publish record IDs starting at 10,001
+(someone has been smart, though it sets the limit of probesets now to
+10,000).
+
+The datasets are organized differently in these tables. All published
+BXD data is grouped on BXDpublished with the publications as
+'datasets'. So, that is how we list them in the REST API.
+
+To fetch all the datasets we first list ProbeSetFreeze entries. Then
+we list the Published entries.
+
 ** Fetch genotype information
 
 *** SNPs
diff --git a/etc/default_settings.py b/etc/default_settings.py
index c00f6c8f..59e22f1a 100644
--- a/etc/default_settings.py
+++ b/etc/default_settings.py
@@ -2,7 +2,7 @@
 # webserver running in developer mode with limited console
 # output. Copy this file and run it from ./bin/genenetwork2 configfile
 #
-# Note that these settings are fetched in ./wqflask/utilities/tools.py
+# Note: these settings are fetched in ./wqflask/utilities/tools.py
 # which has support for overriding them through environment variables,
 # e.g.
 #
@@ -14,8 +14,12 @@
 # Note also that in the near future we will additionally fetch
 # settings from a JSON file
 #
-# Note that values for False and 0 have to be strings here - otherwise
+# Note: values for False and 0 have to be strings here - otherwise
 # Flask won't pick them up
+#
+# For GNU Guix deployment also check the paths in
+#
+#  ~/.guix-profile/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py
 
 import os
 import sys
@@ -34,7 +38,7 @@ SECURITY_RECOVERABLE = True
 SECURITY_EMAIL_SENDER = "no-reply@genenetwork.org"
 SECURITY_POST_LOGIN_VIEW = "/thank_you"
 
-SERVER_PORT = 5003
+SERVER_PORT = 5003          # running on localhost
 SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc'
 
 # ---- Behavioural settings (defaults) note that logger and log levels can
@@ -42,6 +46,7 @@ SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a
 WEBSERVER_MODE  = 'DEV'     # Python webserver mode (DEBUG|DEV|PROD)
 WEBSERVER_BRANDING = None   # Set the branding (nyi)
 WEBSERVER_DEPLOY = None     # Deployment specifics (nyi)
+WEBSERVER_URL    = "http://localhost:"+str(SERVER_PORT)+"/" # external URL
 
 LOG_LEVEL       = 'WARNING' # Logger mode (DEBUG|INFO|WARNING|ERROR|CRITICAL)
 LOG_LEVEL_DEBUG = '0'       # logger.debugf log level (0-5, 5 = show all)
diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index 6649f8af..a4eaaa2e 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -332,7 +332,7 @@ class DatasetGroup(object):
         if check_plink_gemma():
             marker_class = HumanMarkers
         else:
-            marker_class = Markers            
+            marker_class = Markers
 
         if self.genofile:
             self.markers = marker_class(self.genofile[:-5])
diff --git a/wqflask/runserver.py b/wqflask/runserver.py
index 50805643..50f134db 100644
--- a/wqflask/runserver.py
+++ b/wqflask/runserver.py
@@ -22,11 +22,19 @@ ENDC  = '\033[0m'
 import os
 app.config['SECRET_KEY'] = os.urandom(24)
 
-from utility.tools import WEBSERVER_MODE,get_setting_int
+from utility.tools import WEBSERVER_MODE,get_setting_int,get_setting,get_setting_bool
 
 port = get_setting_int("SERVER_PORT")
 
-logger.info("GN2 is running. Visit %shttp://localhost:%s/%s" % (BLUE,port,ENDC))
+print("GN2 API server URL is ["+BLUE+get_setting("GN_SERVER_URL")+ENDC+"]")
+
+if get_setting_bool("USE_GN_SERVER"):
+    import requests
+    page = requests.get(get_setting("GN_SERVER_URL"))
+    if page.status_code != 200:
+        raise Exception("API server not found!")
+
+print("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL")))
 
 werkzeug_logger = logging.getLogger('werkzeug')
 
diff --git a/wqflask/utility/logger.py b/wqflask/utility/logger.py
index bacb0aa4..128706df 100644
--- a/wqflask/utility/logger.py
+++ b/wqflask/utility/logger.py
@@ -72,7 +72,7 @@ LOG_LEVEL_DEBUG (NYI).
     def warning(self,*args):
         """Call logging.warning for multiple args"""
         self.collect(self.logger.warning,*args)
-        self.logger.warning(self.collect(*args))
+        # self.logger.warning(self.collect(*args))
 
     def error(self,*args):
         """Call logging.error for multiple args"""
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index c5685cdd..63b7149c 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -147,9 +147,14 @@ def assert_writable_dir(dir):
         fh.close()
         os.remove(fn)
     except IOError:
-        raise Exception('Unable to write test.txt to directory ' + dir )
+        raise Exception('Unable to write test.txt to directory ' + dir)
     return dir
 
+def assert_file(fn):
+    if not valid_file(fn):
+        raise Exception('Unable to find file '+fn)
+    return fn
+
 def mk_dir(dir):
     if not valid_path(dir):
         os.makedirs(dir)
@@ -174,6 +179,9 @@ def locate(name, subdir=None):
     if subdir: sys.stderr.write(subdir)
     raise Exception("Can not locate "+name+" in "+base)
 
+def locate_phewas(name, subdir=None):
+    return locate(name,'/phewas/'+subdir)
+
 def locate_ignore_error(name, subdir=None):
     """
     Locate a static flat file in the GENENETWORK_FILES environment.
@@ -243,11 +251,11 @@ JS_GUIX_PATH       = get_setting('JS_GUIX_PATH')
 JS_GN_PATH         = get_setting('JS_GN_PATH')
 # assert_dir(JS_GN_PATH)
 
-PYLMM_COMMAND         = pylmm_command()
-GEMMA_COMMAND         = gemma_command()
-GEMMA_WRAPPER_COMMAND = gemma_wrapper_command()
-PLINK_COMMAND         = plink_command()
-TEMPDIR               = tempdir() # defaults to UNIX TMPDIR
+PYLMM_COMMAND      = app_set("PYLMM_COMMAND",pylmm_command())
+GEMMA_COMMAND      = app_set("GEMMA_COMMAND",gemma_command())
+PLINK_COMMAND      = app_set("PLINK_COMMAND",plink_command())
+TEMPDIR            = tempdir() # defaults to UNIX TMPDIR
+assert_dir(TEMPDIR)
 
 # ---- Handle specific JS modules
 JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("Twitter-Post-Fetcher"))
@@ -267,3 +275,10 @@ if os.environ.get('WQFLASK_OVERRIDES'):
             else:
                 OVERRIDES[k] = cmd
             logger.debug(OVERRIDES)
+
+assert_file(PHEWAS_FILES+"/auwerx/PheWAS_pval_EMMA_norm.RData")
+assert_dir(get_setting("JS_BIODALLIANCE"))
+assert_file(get_setting("JS_BIODALLIANCE")+"/build/dalliance-all.js")
+assert_file(get_setting("JS_BIODALLIANCE")+"/build/worker-all.js")
+assert_dir(get_setting("JS_TWITTER_POST_FETCHER"))
+assert_file(get_setting("JS_TWITTER_POST_FETCHER")+"/js/twitterFetcher_min.js")
-- 
cgit v1.2.3


From b3724d60d2e50dd19e61140406aa85dff981ac16 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 29 Jan 2018 14:15:49 +0000
Subject: startup script: adapting for gunicorn

---
 bin/genenetwork2                 | 16 ++++++++++++++++
 wqflask/__init__.py              |  1 -
 wqflask/gunicorn/__init__.py     |  0
 wqflask/gunicorn/genenetwork2.py | 10 ++++++++++
 wqflask/gunicorn/wsgi.py         |  5 +++++
 wqflask/runserver.py             |  2 ++
 6 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 wqflask/gunicorn/__init__.py
 create mode 100644 wqflask/gunicorn/genenetwork2.py
 create mode 100644 wqflask/gunicorn/wsgi.py

(limited to 'wqflask/runserver.py')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index c6b08771..f32c76eb 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -22,6 +22,14 @@
 # a -c switch, e.g.
 #
 #   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ~/my_overrides.json -c ./wqflask/maintenance/gen_select_dataset.py
+#
+# For development you may want to run
+#
+#   env GN2_PROFILE=~/opt/gn-latest-guix WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2
+#
+# For production we use gunicorn. Run with
+#
+#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:8000 --workers=2 --chdir wqflask/gunicorn/ wsgi"
 
 SCRIPT=$(readlink -f "$0")
 GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT"))
@@ -123,6 +131,14 @@ if [ "$1" = '-c' ] ; then
     python $cmd
     exit 0
 fi
+if [ "$1" = '-gunicorn' ] ; then
+    cd $GN2_BASE_DIR
+    cmd=${2}
+    echo PYTHONPATH=$PYTHONPATH
+    echo RUNNING COMMAND gunicorn $cmd
+    gunicorn $cmd
+    exit 0
+fi
 
 echo "Starting the redis server:"
 echo -n "dir $TMPDIR
diff --git a/wqflask/__init__.py b/wqflask/__init__.py
index 315b709e..e69de29b 100644
--- a/wqflask/__init__.py
+++ b/wqflask/__init__.py
@@ -1 +0,0 @@
-from wqflask import app
diff --git a/wqflask/gunicorn/__init__.py b/wqflask/gunicorn/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/wqflask/gunicorn/genenetwork2.py b/wqflask/gunicorn/genenetwork2.py
new file mode 100644
index 00000000..afbdc934
--- /dev/null
+++ b/wqflask/gunicorn/genenetwork2.py
@@ -0,0 +1,10 @@
+from flask import Flask
+application = Flask(__name__)
+
+@application.route("/")
+def hello():
+    return "<h1 style='color:blue'>Hello There!</h1>"
+
+if __name__ == "__main__":
+    application.run(host='0.0.0.0')
+
diff --git a/wqflask/gunicorn/wsgi.py b/wqflask/gunicorn/wsgi.py
new file mode 100644
index 00000000..8c14ee6b
--- /dev/null
+++ b/wqflask/gunicorn/wsgi.py
@@ -0,0 +1,5 @@
+from genenetwork2 import application
+
+if __name__ == "__main__":
+    application.run()
+
diff --git a/wqflask/runserver.py b/wqflask/runserver.py
index 50f134db..e62e34f2 100644
--- a/wqflask/runserver.py
+++ b/wqflask/runserver.py
@@ -1,5 +1,7 @@
 # Starts the webserver with the ./bin/genenetwork2 command
 #
+# This uses Werkzeug WSGI, see gunicorn.py for the alternative
+#
 # Please note, running with host set externally below combined with
 # debug mode is a security risk unless you have a firewall setup, e.g.
 #
-- 
cgit v1.2.3


From 85819dbeaf922456dc205e482e7e338952945d9b Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 29 Jan 2018 15:10:09 +0000
Subject: Gunicorn works

---
 bin/genenetwork2                 | 28 +++++++++++++++++++---------
 etc/default_settings.py          |  2 +-
 wqflask/gunicorn/__init__.py     |  0
 wqflask/gunicorn/genenetwork2.py | 10 ----------
 wqflask/gunicorn/wsgi.py         |  5 -----
 wqflask/run_gunicorn.py          | 19 +++++++++++++++++++
 wqflask/runserver.py             |  2 +-
 wqflask/wsgi.py                  |  4 ++++
 8 files changed, 44 insertions(+), 26 deletions(-)
 delete mode 100644 wqflask/gunicorn/__init__.py
 delete mode 100644 wqflask/gunicorn/genenetwork2.py
 delete mode 100644 wqflask/gunicorn/wsgi.py
 create mode 100644 wqflask/run_gunicorn.py
 create mode 100644 wqflask/wsgi.py

(limited to 'wqflask/runserver.py')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index f32c76eb..f6ae3807 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -31,7 +31,7 @@
 #
 #   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:8000 --workers=2 --chdir wqflask/gunicorn/ wsgi"
 
-SCRIPT=$(readlink -f "$0")
+SCRIPT=$(realpath "$0")
 GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT"))
 GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR)
 
@@ -50,15 +50,25 @@ else
 fi
 echo GN_VERSION=$GN_VERSION
 
+if [ "$1" = "-c" -o "$1" = "-gunicorn" ]; then
+    echo "Can not use $1 switch without default settings file"
+    exit 1
+fi
 # Handle settings parameter (can be .py or .json)
-settings=$1
-ext="${settings##*.}"
-if [ -z "$settings" -o "$ext" = "json" -o "$ext" = "JSON" ]; then
-    overrides=$settings
+settings=$(realpath "$1")
+if [ ! -e $settings ]; then
     settings=$GN2_BASE_DIR/etc/default_settings.py
 else
     shift
 fi
+
+ext="${settings##*.}"
+if [ "$ext" = "json" -o "$ext" = "JSON" ]; then
+    overrides=$settings
+else
+    echo $settings
+fi
+
 if [ ! -e $settings ]; then
     echo "ERROR: can not locate settings file - pass it in the command line"
     exit 1
@@ -132,12 +142,12 @@ if [ "$1" = '-c' ] ; then
     exit 0
 fi
 if [ "$1" = '-gunicorn' ] ; then
-    cd $GN2_BASE_DIR
-    cmd=${2}
+    cd $GN2_BASE_DIR/wqflask
+    cmd=$2
     echo PYTHONPATH=$PYTHONPATH
-    echo RUNNING COMMAND gunicorn $cmd
+    echo RUNNING gunicorn $cmd
     gunicorn $cmd
-    exit 0
+    exit $?
 fi
 
 echo "Starting the redis server:"
diff --git a/etc/default_settings.py b/etc/default_settings.py
index 59e22f1a..699d21f1 100644
--- a/etc/default_settings.py
+++ b/etc/default_settings.py
@@ -27,7 +27,7 @@ import sys
 GN_VERSION = open("../etc/VERSION","r").read()
 SQL_URI = "mysql://gn2:mysql_password@localhost/db_webqtl_s"
 SQL_ALCHEMY_POOL_RECYCLE = 3600
-GN_SERVER_URL = "http://localhost:8880/"
+GN_SERVER_URL = "http://localhost:8880/" # REST API server
 
 # ---- Flask configuration (see website)
 TRAP_BAD_REQUEST_ERRORS = True
diff --git a/wqflask/gunicorn/__init__.py b/wqflask/gunicorn/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/wqflask/gunicorn/genenetwork2.py b/wqflask/gunicorn/genenetwork2.py
deleted file mode 100644
index afbdc934..00000000
--- a/wqflask/gunicorn/genenetwork2.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from flask import Flask
-application = Flask(__name__)
-
-@application.route("/")
-def hello():
-    return "<h1 style='color:blue'>Hello There!</h1>"
-
-if __name__ == "__main__":
-    application.run(host='0.0.0.0')
-
diff --git a/wqflask/gunicorn/wsgi.py b/wqflask/gunicorn/wsgi.py
deleted file mode 100644
index 8c14ee6b..00000000
--- a/wqflask/gunicorn/wsgi.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from genenetwork2 import application
-
-if __name__ == "__main__":
-    application.run()
-
diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py
new file mode 100644
index 00000000..14a2d689
--- /dev/null
+++ b/wqflask/run_gunicorn.py
@@ -0,0 +1,19 @@
+# Run with gunicorn, see ./bin/genenetwork2 for an example
+#
+# Run standalone with
+#
+#   ./bin/genenetwork2 ./etc/default_settings.py -c run_gunicorn.py
+
+# from flask import Flask
+# application = Flask(__name__)
+
+print "Starting up Gunicorn process"
+
+from wqflask import app
+
+@app.route("/gunicorn")
+def hello():
+    return "<h1 style='color:blue'>Hello There!</h1>"
+
+if __name__ == "__main__":
+    app.run(host='0.0.0.0')
diff --git a/wqflask/runserver.py b/wqflask/runserver.py
index e62e34f2..a0c76e51 100644
--- a/wqflask/runserver.py
+++ b/wqflask/runserver.py
@@ -1,6 +1,6 @@
 # Starts the webserver with the ./bin/genenetwork2 command
 #
-# This uses Werkzeug WSGI, see gunicorn.py for the alternative
+# This uses Werkzeug WSGI, see ./run_gunicorn.py for the alternative
 #
 # Please note, running with host set externally below combined with
 # debug mode is a security risk unless you have a firewall setup, e.g.
diff --git a/wqflask/wsgi.py b/wqflask/wsgi.py
new file mode 100644
index 00000000..be9c7b37
--- /dev/null
+++ b/wqflask/wsgi.py
@@ -0,0 +1,4 @@
+from run_gunicorn import app as application # expect application as a name
+
+if __name__ == "__main__":
+    application.run()
-- 
cgit v1.2.3


From 7aae77762e47c9269591e138ab25320d3ed3a85f Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Tue, 20 Mar 2018 11:02:25 +0000
Subject: Refactor startup config for gunicorn and werkzeug

---
 wqflask/run_gunicorn.py                            |  6 ++--
 wqflask/runserver.py                               | 17 ++--------
 wqflask/utility/elasticsearch_tools.py             |  5 +++
 wqflask/utility/startup_config.py                  | 39 ++++++++++++++++++++++
 wqflask/utility/tools.py                           |  4 ++-
 .../wqflask/templates/new_security/login_user.html |  8 ++++-
 6 files changed, 59 insertions(+), 20 deletions(-)
 create mode 100644 wqflask/utility/startup_config.py

(limited to 'wqflask/runserver.py')

diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py
index ebe3add5..adffdca3 100644
--- a/wqflask/run_gunicorn.py
+++ b/wqflask/run_gunicorn.py
@@ -7,12 +7,12 @@
 # from flask import Flask
 # application = Flask(__name__)
 
-print "Starting up Gunicorn process"
+print "===> Starting up Gunicorn process"
 
 from wqflask import app
+from utility.startup_config import app_config
 
-app.config['SESSION_TYPE'] = 'filesystem'
-app.config['SECRET_KEY'] = 'super secret key'
+app_config()
 
 @app.route("/gunicorn")
 def hello():
diff --git a/wqflask/runserver.py b/wqflask/runserver.py
index a0c76e51..5f41d04d 100644
--- a/wqflask/runserver.py
+++ b/wqflask/runserver.py
@@ -21,22 +21,9 @@ GREEN = '\033[92m'
 BOLD  = '\033[1m'
 ENDC  = '\033[0m'
 
-import os
-app.config['SECRET_KEY'] = os.urandom(24)
+from utility.startup_config import app_config
 
-from utility.tools import WEBSERVER_MODE,get_setting_int,get_setting,get_setting_bool
-
-port = get_setting_int("SERVER_PORT")
-
-print("GN2 API server URL is ["+BLUE+get_setting("GN_SERVER_URL")+ENDC+"]")
-
-if get_setting_bool("USE_GN_SERVER"):
-    import requests
-    page = requests.get(get_setting("GN_SERVER_URL"))
-    if page.status_code != 200:
-        raise Exception("API server not found!")
-
-print("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL")))
+app_config()
 
 werkzeug_logger = logging.getLogger('werkzeug')
 
diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py
index 2d3d5add..734379f7 100644
--- a/wqflask/utility/elasticsearch_tools.py
+++ b/wqflask/utility/elasticsearch_tools.py
@@ -6,6 +6,11 @@ logger = getLogger(__name__)
 
 from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT
 
+def test_elasticsearch_connection():
+    es = Elasticsearch(['http://'+ELASTICSEARCH_HOST+":"+ELASTICSEARCH_PORT+'/'], verify_certs=True)
+    if not es.ping():
+        logger.warning("Elasticsearch is DOWN")
+
 def get_elasticsearch_connection():
     logger.info("get_elasticsearch_connection")
     es = None
diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py
new file mode 100644
index 00000000..02e8e56c
--- /dev/null
+++ b/wqflask/utility/startup_config.py
@@ -0,0 +1,39 @@
+
+from wqflask import app
+from utility.tools import WEBSERVER_MODE, show_settings, get_setting_int, get_setting, get_setting_bool
+
+import utility.logger
+logger = utility.logger.getLogger(__name__ )
+
+BLUE  = '\033[94m'
+GREEN = '\033[92m'
+BOLD  = '\033[1m'
+ENDC  = '\033[0m'
+
+def app_config():
+    app.config['SESSION_TYPE'] = 'filesystem'
+    if not app.config.get('SECRET_KEY'):
+        import os
+        app.config['SECRET_KEY'] = str(os.urandom(24))
+
+    mode = WEBSERVER_MODE
+    if mode == "DEV" or mode == "DEBUG":
+        app.config['TEMPLATES_AUTO_RELOAD'] = True
+        if mode == "DEBUG":
+            app.config['EXPLAIN_TEMPLATE_LOADING'] = True
+    print("==========================================")
+    show_settings()
+
+    port = get_setting_int("SERVER_PORT")
+
+    if get_setting_bool("USE_GN_SERVER"):
+        print("GN2 API server URL is ["+BLUE+get_setting("GN_SERVER_URL")+ENDC+"]")
+        import requests
+        page = requests.get(get_setting("GN_SERVER_URL"))
+        if page.status_code != 200:
+            raise Exception("API server not found!")
+
+    import utility.elasticsearch_tools as es
+    es.test_elasticsearch_connection()
+
+    print("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL")))
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 4b4cd633..59bb49d8 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -220,7 +220,7 @@ def show_settings():
 
     logger.info(OVERRIDES)
     logger.info(BLUE+"Mr. Mojo Risin 2"+ENDC)
-    print "runserver.py: ****** Webserver configuration ******"
+    print "runserver.py: ****** Webserver configuration - k,v pairs from app.config ******"
     keylist = app.config.keys()
     keylist.sort()
     for k in keylist:
@@ -269,6 +269,8 @@ if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET:
 
 ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST')
 ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT')
+import utility.elasticsearch_tools as es
+es.test_elasticsearch_connection()
 
 SMTP_CONNECT = get_setting('SMTP_CONNECT')
 SMTP_USERNAME = get_setting('SMTP_USERNAME')
diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html
index 6d597f6b..949760b6 100644
--- a/wqflask/wqflask/templates/new_security/login_user.html
+++ b/wqflask/wqflask/templates/new_security/login_user.html
@@ -15,7 +15,6 @@
 
             <h4>Don't have an account?</h4>
 
-
             {% if es_server: %}
 	    <a href="/n/register" class="btn btn-primary modalize">Create a new account</a>
 	    {% else: %}
@@ -92,6 +91,13 @@
 		Please try again later.</p>
 	    </div>
 	    {% endif %}
+            {% if not es_server and not external_login: %}
+            <hr>
+	    <div class="alert alert-warning">
+               Note: it is safe to use GeneNetwork without a login. Login is only required for keeping track of
+            collections and getting access to some types of restricted data.
+            </div>
+            {% endif %}
         </div>
     </div>
 
-- 
cgit v1.2.3


From 496759ad08efb02b4268ea7f3bbb7905974237e9 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 18 Feb 2019 09:39:41 +0000
Subject: Updated installation instructions and SERVER_PORT for single flask
 server

---
 doc/README.org           | 71 ++++++++++++++++++++++++++++++++++--------------
 wqflask/runserver.py     |  8 ++++--
 wqflask/utility/tools.py |  1 +
 3 files changed, 57 insertions(+), 23 deletions(-)

(limited to 'wqflask/runserver.py')

diff --git a/doc/README.org b/doc/README.org
index 5dc9e994..620c946c 100644
--- a/doc/README.org
+++ b/doc/README.org
@@ -12,13 +12,15 @@
    - [[#load-the-small-database-in-mysql][Load the small database in MySQL]]
  - [[#gn2-dependency-graph][GN2 Dependency Graph]]
  - [[#working-with-the-gn2-source-code][Working with the GN2 source code]]
+ - [[#running-elasticsearch][Running ElasticSearch]]
+   - [[#systemd][SystemD]]
+ - [[#read-more][Read more]]
  - [[#trouble-shooting][Trouble shooting]]
    - [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]]
-   - [[#error-can-not-find-directory-homegn2_data][ERROR: can not find directory $HOME/gn2_data]]
+   - [[#error-can-not-find-directory-homegn2_data-or-can-not-find-directory-homegenotype_filesgenotype][ERROR: 'can not find directory $HOME/gn2_data' or 'can not find directory $HOME/genotype_files/genotype']]
    - [[#cant-run-a-module][Can't run a module]]
    - [[#rpy2-error-show-now-found][Rpy2 error 'show' now found]]
    - [[#mysql-cant-connect-server-through-socket-error][Mysql can't connect server through socket ERROR]]
- - [[#read-more][Read more]]
  - [[#irc-session][IRC session]]
 
 * Introduction
@@ -100,11 +102,13 @@ mysql (which comes as part of the GNU Guix genenetwork2 install).
 
 As root configure and run
 
-:  adduser mysql && addgroup mysql
-:  mysqld --datadir=/var/mysql --initialize-insecure
-:  mkdir -p /var/run/mysqld
-:  chown mysql.mysql ~/mysql /var/run/mysqld
-:  mysqld -u mysql --datadir=/var/mysql --explicit_defaults_for_timestamp -P 12048"
+#+BEGIN_SRC bash
+adduser mysql && addgroup mysql
+mysqld --datadir=/var/mysql --initialize-insecure
+mkdir -p /var/run/mysqld
+chown mysql.mysql ~/mysql /var/run/mysqld
+mysqld -u mysql --datadir=/var/mysql --explicit_defaults_for_timestamp -P 12048"
+#+END_SRC
 
 If you want to run as root you may have to set
 
@@ -192,6 +196,41 @@ http://biogems.info/contrib/genenetwork/gn2.svg
 
 See [[development.org]].
 
+* Running ElasticSearch
+
+In order to start up elasticsearch:
+Penguin - change user to "elasticsearch" and use the following command: "env JAVA_HOME=/opt/jdk-9.0.4 /opt/elasticsearch-6.2.1/bin/elasticsearch"
+
+
+** SystemD
+
+New server - as root run "systemctl restart elasticsearch"
+
+#+BEGIN_SRC
+tux01:/etc/systemd/system# cat elasticsearch.service
+[Unit]
+Description=Run Elasticsearch
+
+[Service]
+ExecStart=/opt/elasticsearch-6.2.1/bin/elasticsearch
+Environment=JAVA_HOME=/opt/jdk-9.0.4
+Environment="ES_JAVA_OPTS=-Xms1g -Xmx8g"
+Environment="PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games:/opt/jdk-9.0.4/bin"
+LimitNOFILE=65536
+StandardOutput=syslog
+StandardError=syslog
+User=elasticsearch
+
+[Install]
+WantedBy=multi-user.target
+#+END_SRC
+
+* Read more
+
+If you want to understand the architecture of GN2 read
+[[Architecture.org]].  The rest of this document is mostly on deployment
+of GN2.
+
 * Trouble shooting
 
 ** ImportError: No module named jinja2
@@ -210,13 +249,17 @@ On one system:
 : export GEM_PATH="$HOME/.guix-profile/lib/ruby/gems/2.2.0"
 
 and perhaps a few more.
-** ERROR: can not find directory $HOME/gn2_data
+** ERROR: 'can not find directory $HOME/gn2_data' or 'can not find directory $HOME/genotype_files/genotype'
 
 The default settings file looks in your $HOME/gn2_data. Since these
 files come with a Guix installation you should take a hint from the
 values in the installed version of default_settings.py (see above in
 this document).
 
+You can use the GENENETWORK_FILES switch to set the datadir, for example
+
+: env GN2_PROFILE=~/opt/gn-latest GENENETWORK_FILES=/gnu/data/gn2_data ./bin/genenetwork2
+
 ** Can't run a module
 
 In rare cases, development modules are not brought in with Guix
@@ -257,18 +300,6 @@ if that works run genenetwork after setting SQL_URI to something like
 
 : export SQL_URI=mysql://gn2:mysql_password@127.0.0.1/db_webqtl_s
 
-* Running ElasticSearch
-
-In order to start up elasticsearch:
-Penguin - change user to "elasticsearch" and use the following command: "env JAVA_HOME=/opt/jdk-9.0.4 /opt/elasticsearch-6.2.1/bin/elasticsearch"
-
-New server - as root run "systemctl restart elasticsearch"
-
-* Read more
-
-If you want to understand the architecture of GN2 read
-[[Architecture.org]].  The rest of this document is mostly on deployment
-of GN2.
 
 * IRC session
 
diff --git a/wqflask/runserver.py b/wqflask/runserver.py
index 5f41d04d..7c06356b 100644
--- a/wqflask/runserver.py
+++ b/wqflask/runserver.py
@@ -27,9 +27,11 @@ app_config()
 
 werkzeug_logger = logging.getLogger('werkzeug')
 
+from utility.tools import WEBSERVER_MODE, SERVER_PORT
+
 if WEBSERVER_MODE == 'DEBUG':
     app.run(host='0.0.0.0',
-            port=port,
+            port=SERVER_PORT,
             debug=True,
             use_debugger=False,
             threaded=False,
@@ -38,7 +40,7 @@ if WEBSERVER_MODE == 'DEBUG':
 elif WEBSERVER_MODE == 'DEV':
     werkzeug_logger.setLevel(logging.WARNING)
     app.run(host='0.0.0.0',
-            port=port,
+            port=SERVER_PORT,
             debug=False,
             use_debugger=False,
             threaded=False,
@@ -46,7 +48,7 @@ elif WEBSERVER_MODE == 'DEV':
             use_reloader=True)
 else: # staging/production modes
     app.run(host='0.0.0.0',
-            port=port,
+            port=SERVER_PORT,
             debug=False,
             use_debugger=False,
             threaded=True,
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 86ef2e1e..8b2260f5 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -232,6 +232,7 @@ GN_VERSION         = get_setting('GN_VERSION')
 HOME               = get_setting('HOME')
 WEBSERVER_MODE     = get_setting('WEBSERVER_MODE')
 GN_SERVER_URL      = get_setting('GN_SERVER_URL')
+SERVER_PORT        = get_setting_int('SERVER_PORT')
 SQL_URI            = get_setting('SQL_URI')
 LOG_LEVEL          = get_setting('LOG_LEVEL')
 LOG_LEVEL_DEBUG    = get_setting_int('LOG_LEVEL_DEBUG')
-- 
cgit v1.2.3