Merge branch 'testing' of github.com:genenetwork/genenetwork2 into clustered_heatmaps

28 files changed, 355 insertions, 252 deletions

diff --git a/doc/docker-container.org b/doc/docker-container.org
index ef0d71fc..79b8272f 100644
--- a/doc/docker-container.org
+++ b/doc/docker-container.org
@@ -28,13 +28,6 @@ which will be added to a base mariaDB image.
 First create the gn2 tar archive by running:
 
 #+begin_src sh
-# For the Python 2 version:
-env GUIX_PACKAGE_PATH="/home/bonface/projects/guix-bioinformatics:/home/bonface/projects/guix-past/modules" \
-    ./pre-inst-env guix pack --no-grafts\
-    -S /gn2-profile=/ \
-    screen python2-genenetwork2
-
-# For the Python 3 version:
 env GUIX_PACKAGE_PATH="/home/bonface/projects/guix-bioinformatics:/home/bonface/projects/guix-past/modules" \
     ./pre-inst-env guix pack --no-grafts\
     -S /gn2-profile=/ \
diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py
new file mode 100644
index 00000000..76c7fb4f
--- /dev/null
+++ b/scripts/authentication/group.py
@@ -0,0 +1,142 @@
+"""A script for adding users to a specific group.
+
+Example:
+
+Assuming there are no groups and 'test@bonfacemunyoki.com' does not
+exist in Redis:
+
+.. code-block:: bash
+   python group.py -g "editors" -m "test@bonfacemunyoki.com"
+
+results in::
+
+   Successfully created the group: 'editors'
+   Data: '{"admins": [], "members": []}'
+
+If 'me@bonfacemunyoki.com' exists in 'users' in Redis and we run:
+
+.. code-block:: bash
+   python group.py -g "editors" -m "me@bonfacemunyoki.com"
+
+now results in::
+
+   No new group was created.
+   Updated Data: {'admins': [], 'members': ['me@bonfacemunyoki.com']}
+
+"""
+
+import argparse
+import datetime
+import redis
+import json
+
+from typing import Dict, List, Optional, Set
+
+def create_group_data(users: Dict, target_group: str,
+                      members: Optional[str] = None,
+                      admins: Optional[str] = None) -> Dict:
+    """Return a dictionary that contains the following keys: "key",
+    "field", and "value" that can be used in a redis hash as follows:
+    HSET key field value
+
+    Parameters:
+
+    - `users`: a list of users for example:
+
+    {'8ad942fe-490d-453e-bd37-56f252e41603':
+    '{"email_address": "me@test.com",
+      "full_name": "John Doe",
+      "organization": "Genenetwork",
+      "password": {"algorithm": "pbkdf2",
+                   "hashfunc": "sha256",
+                    "salt": "gJrd1HnPSSCmzB5veMPaVk2ozzDlS1Z7Ggcyl1+pciA=",
+                    "iterations": 100000, "keylength": 32,
+                    "created_timestamp": "2021-09-22T11:32:44.971912",
+                    "password": "edcdaa60e84526c6"},
+                    "user_id": "8ad942fe", "confirmed": 1,
+                    "registration_info": {
+                        "timestamp": "2021-09-22T11:32:45.028833",
+                        "ip_address": "127.0.0.1",
+                        "user_agent": "Mozilla/5.0"}}'}
+
+    - `target_group`: the group name that will be stored inside the
+      "groups" hash in Redis.
+
+    - `members`: a comma-separated list of values that contain members
+      of the `target_group` e.g. "me@test1.com, me@test2.com,
+      me@test3.com"
+
+    - `admins`: a comma-separated list of values that contain
+      administrators of the `target_group` e.g. "me@test1.com,
+      me@test2.com, me@test3.com"
+
+    """
+
+    _members: List = "".join(members.split()).split(",") if members else []
+    _admins: List = "".join(admins.split()).split(",") if admins else []
+
+    user_ids: Dict = dict()
+    for user_id, user_details in users.items():
+        _details = json.loads(user_details)
+        if _details.get("email_address"):
+            user_ids[_details.get("email_address")] = user_id
+    print(user_ids)
+    return {"key": "groups",
+            "field": target_group,
+            "value": json.dumps({
+                "id": target_group,
+                "name": target_group,
+                "admins": [user_ids[admin] for admin in _admins
+                           if admin in user_ids],
+                "members": [user_ids[member] for member in _members
+                            if member in user_ids],
+                "changed_timestamp": datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
+            })}
+
+
+if __name__ == "__main__":
+    # Initialising the parser CLI arguments
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-g", "--group-name",
+                        help="This is the name of the GROUP mask")
+    parser.add_argument("-m", "--members",
+                        help="Members of the GROUP mask")
+    parser.add_argument("-a", "--admins",
+                        help="Admins of the GROUP mask")
+    args = parser.parse_args()
+
+    if not args.group_name:
+        exit("\nExiting. Please specify a group name to use!\n")
+
+    members = args.members if args.members else None
+    admins = args.admins if args.admins else None
+
+    REDIS_CONN = redis.Redis(decode_responses=True)
+    USERS = REDIS_CONN.hgetall("users")
+
+    if not any([members, admins]):
+        exit("\nExiting. Please provide a value for "
+             "MEMBERS(-m) or ADMINS(-a)!\n")
+
+    data = create_group_data(
+        users=USERS,
+        target_group=args.group_name,
+        members=members,
+        admins=admins)
+
+    if not REDIS_CONN.hget("groups", data.get("field", "")):
+      updated_data = json.loads(data["value"])
+      updated_data["created_timestamp"] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')
+      data["value"] = json.dumps(updated_data)
+
+    created_p = REDIS_CONN.hset(data.get("key", ""),
+                                data.get("field", ""),
+                                data.get("value", ""))
+
+    groups = json.loads(REDIS_CONN.hget("groups",
+                                        args.group_name))  # type: ignore
+    if created_p:
+        exit(f"\nSuccessfully created the group: '{args.group_name}'\n"
+             f"`HGETALL groups {args.group_name}`: {groups}\n")
+    exit("\nNo new group was created.\n"
+         f"`HGETALL groups {args.group_name}`: {groups}\n")
diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py
new file mode 100644
index 00000000..8fcf09d7
--- /dev/null
+++ b/scripts/authentication/resource.py
@@ -0,0 +1,100 @@
+"""A script that:
+
+- Optionally restores data from a json file.
+
+- By default, without any args provided, adds the group: 'editors' to
+every resource. 'editors' should have the right to edit both metadata
+and data.
+
+- Optionally creates a back-up every time you edit a resource.
+
+
+To restore a back-up:
+
+.. code-block:: python
+   python resource.py --restore <PATH/TO/RESOURCE/BACK-UP/FILE>
+
+To add editors to every resource without creating a back-up:
+
+.. code-block:: python
+   python resource.py
+
+To add editors to every resource while creating a back-up before any
+destructive edits:
+
+.. code-block:: python
+   python resource.py --enable-backup
+
+"""
+import argparse
+import json
+import redis
+import os
+
+from datetime import datetime
+
+
+def recover_hash(name: str, file_path: str, set_function) -> bool:
+    """Recover back-ups using the `set_function`
+
+    Parameters:
+
+    - `name`: Redis hash where `file_path` will be restored
+
+    - `file_path`: File path where redis hash is sourced from
+
+    - `set_function`: Function used to do the Redis backup for
+      example: HSET
+
+    """
+    try:
+        with open(file_path, "r") as f:
+            resources = json.load(f)
+            for resource_id, resource in resources.items():
+                set_function(name=name,
+                             key=resource_id,
+                             value=resource)
+            return True
+    except Exception as e:
+        print(e)
+        return False
+
+
+if __name__ == "__main__":
+    # Initialising the parser CLI arguments
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--restore",
+                        help="Restore from a given backup")
+    parser.add_argument("--enable-backup", action="store_true",
+                        help="Create a back up before edits")
+    args = parser.parse_args()
+
+    if args.restore:
+        if recover_hash(name="resources",
+                        file_path=args.back_up,
+                        set_function=redis.Redis(decode_responses=True).hset):
+            exit(f"\n Done restoring {args.back_up}!\n")
+        else:
+            exit(f"\n There was an error restoring {args.back_up}!\n")
+
+    REDIS_CONN = redis.Redis(decode_responses=True)
+    RESOURCES = REDIS_CONN.hgetall("resources")
+    BACKUP_DIR = os.path.join(os.getenv("HOME"), "redis")
+    if args.enable_backup:
+        FILENAME = ("resources-"
+                    f"{datetime.now().strftime('%Y-%m-%d-%I:%M:%S-%p')}"
+                    ".json")
+        if not os.path.exists(BACKUP_DIR):
+            os.mkdir(BACKUP_DIR)
+        with open(os.path.join(BACKUP_DIR, FILENAME), "w") as f:
+            json.dump(RESOURCES, f, indent=4)
+        print(f"\nDone backing upto {FILENAME}")
+
+    for resource_id, resource in RESOURCES.items():
+        _resource = json.loads(resource)  # str -> dict conversion
+        _resource["group_masks"] = {"editors": {"metadata": "edit",
+                                                "data": "edit"}}
+        REDIS_CONN.hset("resources",
+                        resource_id,
+                        json.dumps(_resource))
+    exit("Done updating `resources`\n")
diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_gemma_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_gemma_mapping.py
index 4003d68f..58a44b2a 100644
--- a/wqflask/tests/unit/wqflask/marker_regression/test_gemma_mapping.py
+++ b/wqflask/tests/unit/wqflask/marker_regression/test_gemma_mapping.py
@@ -81,10 +81,12 @@ class TestGemmaMapping(unittest.TestCase):
     def test_gen_pheno_txt_file(self):
         """add tests for generating pheno txt file"""
         with mock.patch("builtins.open", mock.mock_open())as mock_open:
-            gen_pheno_txt_file(this_dataset={}, genofile_name="", vals=[
-                               "x", "w", "q", "we", "R"], trait_filename="fitr.re")
+            gen_pheno_txt_file(
+                this_dataset=AttributeSetter({"name": "A"}),
+                genofile_name="", vals=[
+                    "x", "w", "q", "we", "R"])
             mock_open.assert_called_once_with(
-                '/home/user/data/gn2/fitr.re.txt', 'w')
+                '/home/user/data/gn2/PHENO_KiAEKlCvM6iGTM9Kh_TAlQ.txt', 'w')
             filehandler = mock_open()
             values = ["x", "w", "q", "we", "R"]
             write_calls = [mock.call('NA\n'), mock.call('w\n'), mock.call(
@@ -112,7 +114,7 @@ class TestGemmaMapping(unittest.TestCase):
         create_trait.side_effect = create_trait_side_effect
 
         group = MockGroup({"name": "group_X", "samplelist": samplelist})
-        this_dataset = AttributeSetter({"group": group})
+        this_dataset = AttributeSetter({"group": group, "name": "A"})
         flat_files.return_value = "Home/Genenetwork"
 
         with mock.patch("builtins.open", mock.mock_open())as mock_open:
@@ -132,7 +134,7 @@ class TestGemmaMapping(unittest.TestCase):
 
             flat_files.assert_called_once_with('mapping')
             mock_open.assert_called_once_with(
-                'Home/Genenetwork/group_X_covariates.txt', 'w')
+                'Home/Genenetwork/COVAR_anFZ_LfZYV0Ulywo+7tRCw.txt', 'w')
             filehandler = mock_open()
             filehandler.write.assert_has_calls([mock.call(
                 '-9\t'), mock.call('-9\t'), mock.call('-9\t'), mock.call('-9\t'), mock.call('\n')])
diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py
index c220a072..3747aeb8 100644
--- a/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py
+++ b/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py
@@ -47,7 +47,7 @@ class TestRunMapping(unittest.TestCase):
 
         self.chromosomes = AttributeSetter({"chromosomes": chromosomes})
         self.trait = AttributeSetter(
-            {"symbol": "IGFI", "chr": "X1", "mb": 123313})
+            {"symbol": "IGFI", "chr": "X1", "mb": 123313, "display_name": "Test Name"})
 
     def tearDown(self):
         self.dataset = AttributeSetter(
@@ -181,12 +181,15 @@ class TestRunMapping(unittest.TestCase):
             with mock.patch("wqflask.marker_regression.run_mapping.datetime.datetime", new=datetime_mock):
                 export_mapping_results(dataset=self.dataset, trait=self.trait, markers=markers,
                                        results_path="~/results", mapping_scale="physic", score_type="-log(p)",
-                                       transform="qnorm", covariates="Dataset1:Trait1,Dataset2:Trait2", n_samples="100")
+                                       transform="qnorm", covariates="Dataset1:Trait1,Dataset2:Trait2", n_samples="100",
+                                       vals_hash="")
 
                 write_calls = [
                     mock.call('Time/Date: 09/01/19 / 10:12:12\n'),
                     mock.call('Population: Human GP1_\n'), mock.call(
                         'Data Set: dataser_1\n'),
+                    mock.call('Trait: Test Name\n'),
+                    mock.call('Trait Hash: \n'),
                     mock.call('N Samples: 100\n'), mock.call(
                         'Transform - Quantile Normalized\n'),
                     mock.call('Gene Symbol: IGFI\n'), mock.call(
@@ -232,25 +235,20 @@ class TestRunMapping(unittest.TestCase):
                 "c1": "c1_value",
                 "c2": "c2_value",
                 "w1": "w1_value"
-
             },
             "S2": {
                 "w1": "w2_value",
                 "w2": "w2_value"
-
             },
             "S3": {
 
                 "c1": "c1_value",
                 "c2": "c2_value"
-
             },
-
         }})
-
         results = get_perm_strata(this_trait={}, sample_list=sample_list,
                                   categorical_vars=categorical_vars, used_samples=used_samples)
-        self.assertEqual(results, [2, 1])
+        self.assertEqual(results, [1, 1])
 
     def test_get_chr_length(self):
         """test for getting chromosome length"""
diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py
index 99295c67..de9dde46 100644
--- a/wqflask/utility/redis_tools.py
+++ b/wqflask/utility/redis_tools.py
@@ -127,20 +127,20 @@ def check_verification_code(code):
 
 
 def get_user_groups(user_id):
-    # ZS: Get the groups where a user is an admin or a member and
+    # Get the groups where a user is an admin or a member and
     # return lists corresponding to those two sets of groups
-    admin_group_ids = []  # ZS: Group IDs where user is an admin
-    user_group_ids = []  # ZS: Group IDs where user is a regular user
+    admin_group_ids = []  # Group IDs where user is an admin
+    user_group_ids = []  # Group IDs where user is a regular user
     groups_list = Redis.hgetall("groups")
-    for key in groups_list:
+    for group_id, group_details in groups_list.items():
         try:
-            group_ob = json.loads(groups_list[key])
-            group_admins = set([this_admin if this_admin else None for this_admin in group_ob['admins']])
-            group_members = set([this_member if this_member else None for this_member in group_ob['members']])
+            _details = json.loads(group_details)
+            group_admins = set([this_admin if this_admin else None for this_admin in _details['admins']])
+            group_members = set([this_member if this_member else None for this_member in _details['members']])
             if user_id in group_admins:
-                admin_group_ids.append(group_ob['id'])
+                admin_group_ids.append(group_id)
             elif user_id in group_members:
-                user_group_ids.append(group_ob['id'])
+                user_group_ids.append(group_id)
             else:
                 continue
         except:
diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py
index f0978fd3..54aa6795 100644
--- a/wqflask/wqflask/decorators.py
+++ b/wqflask/wqflask/decorators.py
@@ -1,14 +1,36 @@
 """This module contains gn2 decorators"""
 from flask import g
+from typing import Dict
 from functools import wraps
+from utility.hmac import hmac_creation
 
+import json
+import requests
 
-def admin_login_required(f):
+
+def edit_access_required(f):
     """Use this for endpoints where admins are required"""
     @wraps(f)
     def wrap(*args, **kwargs):
-        if g.user_session.record.get(b"user_email_address") not in [
-                b"labwilliams@gmail.com"]:
+        resource_id: str = ""
+        if kwargs.get("inbredset_id"):  # data type: dataset-publish
+            resource_id = hmac_creation("dataset-publish:"
+                                        f"{kwargs.get('inbredset_id')}:"
+                                        f"{kwargs.get('name')}")
+        if kwargs.get("dataset_name"):  # data type: dataset-probe
+            resource_id = hmac_creation("dataset-probeset:"
+                                        f"{kwargs.get('dataset_name')}")
+        response: Dict = {}
+        try:
+            _user_id = g.user_session.record.get(b"user_id",
+                                                 "").decode("utf-8")
+            response = json.loads(
+                requests.get("http://localhost:8080/"
+                             "available?resource="
+                             f"{resource_id}&user={_user_id}").content)
+        except:
+            response = {}
+        if "edit" not in response.get("data", []):
             return "You need to be admin", 401
         return f(*args, **kwargs)
     return wrap
diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py
index 77d6e2db..6254b9b9 100644
--- a/wqflask/wqflask/marker_regression/display_mapping_results.py
+++ b/wqflask/wqflask/marker_regression/display_mapping_results.py
@@ -357,8 +357,7 @@ class DisplayMappingResults:
         if 'use_loco' in list(start_vars.keys()) and self.mapping_method == "gemma":
             self.use_loco = start_vars['use_loco']
 
-        if 'reaper_version' in list(start_vars.keys()) and self.mapping_method == "reaper":
-            self.reaper_version = start_vars['reaper_version']
+        if self.mapping_method == "reaper":
             if 'output_files' in start_vars:
                 self.output_files = ",".join(
                     [(the_file if the_file is not None else "") for the_file in start_vars['output_files']])
diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py
index 4d6715ba..801674e1 100644
--- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py
+++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py
@@ -178,101 +178,6 @@ def parse_reaper_output(gwa_filename, permu_filename, bootstrap_filename):
     return marker_obs, permu_vals, bootstrap_vals
 
 
-def run_original_reaper(this_trait, dataset, samples_before, trait_vals, json_data, num_perm, bootCheck, num_bootstrap, do_control, control_marker, manhattan_plot):
-    genotype = dataset.group.read_genotype_file(use_reaper=True)
-
-    if manhattan_plot != True:
-        genotype = genotype.addinterval()
-
-    trimmed_samples = []
-    trimmed_values = []
-    for i in range(0, len(samples_before)):
-        try:
-            trimmed_values.append(float(trait_vals[i]))
-            trimmed_samples.append(str(samples_before[i]))
-        except:
-            pass
-
-    perm_output = []
-    bootstrap_results = []
-
-    if num_perm < 100:
-        suggestive = 0
-        significant = 0
-    else:
-        perm_output = genotype.permutation(
-            strains=trimmed_samples, trait=trimmed_values, nperm=num_perm)
-        suggestive = perm_output[int(num_perm * 0.37 - 1)]
-        significant = perm_output[int(num_perm * 0.95 - 1)]
-        # highly_significant = perm_output[int(num_perm*0.99-1)] #ZS: Currently not used, but leaving it here just in case
-
-    json_data['suggestive'] = suggestive
-    json_data['significant'] = significant
-
-    if control_marker != "" and do_control == "true":
-        reaper_results = genotype.regression(strains=trimmed_samples,
-                                             trait=trimmed_values,
-                                             control=str(control_marker))
-        if bootCheck:
-            control_geno = []
-            control_geno2 = []
-            _FIND = 0
-            for _chr in genotype:
-                for _locus in _chr:
-                    if _locus.name == control_marker:
-                        control_geno2 = _locus.genotype
-                        _FIND = 1
-                        break
-                if _FIND:
-                    break
-            if control_geno2:
-                _prgy = list(genotype.prgy)
-                for _strain in trimmed_samples:
-                    _idx = _prgy.index(_strain)
-                    control_geno.append(control_geno2[_idx])
-
-            bootstrap_results = genotype.bootstrap(strains=trimmed_samples,
-                                                   trait=trimmed_values,
-                                                   control=control_geno,
-                                                   nboot=num_bootstrap)
-    else:
-        reaper_results = genotype.regression(strains=trimmed_samples,
-                                             trait=trimmed_values)
-
-        if bootCheck:
-            bootstrap_results = genotype.bootstrap(strains=trimmed_samples,
-                                                   trait=trimmed_values,
-                                                   nboot=num_bootstrap)
-
-    json_data['chr'] = []
-    json_data['pos'] = []
-    json_data['lod.hk'] = []
-    json_data['markernames'] = []
-    # if self.additive:
-    #    self.json_data['additive'] = []
-
-    # Need to convert the QTL objects that qtl reaper returns into a json serializable dictionary
-    qtl_results = []
-    for qtl in reaper_results:
-        reaper_locus = qtl.locus
-        # ZS: Convert chr to int
-        converted_chr = reaper_locus.chr
-        if reaper_locus.chr != "X" and reaper_locus.chr != "X/Y":
-            converted_chr = int(reaper_locus.chr)
-        json_data['chr'].append(converted_chr)
-        json_data['pos'].append(reaper_locus.Mb)
-        json_data['lod.hk'].append(qtl.lrs)
-        json_data['markernames'].append(reaper_locus.name)
-        # if self.additive:
-        #    self.json_data['additive'].append(qtl.additive)
-        locus = {"name": reaper_locus.name, "chr": reaper_locus.chr,
-                 "cM": reaper_locus.cM, "Mb": reaper_locus.Mb}
-        qtl = {"lrs_value": qtl.lrs, "chr": converted_chr, "Mb": reaper_locus.Mb,
-               "cM": reaper_locus.cM, "name": reaper_locus.name, "additive": qtl.additive, "dominance": qtl.dominance}
-        qtl_results.append(qtl)
-    return qtl_results, json_data, perm_output, suggestive, significant, bootstrap_results
-
-
 def natural_sort(marker_list):
     """
     Function to naturally sort numbers + strings, adopted from user Mark Byers here: https://stackoverflow.com/questions/4836710/does-python-have-a-built-in-function-for-string-natural-sort
diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py
index a148b49c..1dca1b1b 100644
--- a/wqflask/wqflask/marker_regression/rqtl_mapping.py
+++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py
@@ -39,7 +39,7 @@ def run_rqtl(trait_name, vals, samples, dataset, mapping_scale, model, method, n
     }
 
     if do_control == "true" and control_marker:
-        post_data["control_marker"] = control_marker
+        post_data["control"] = control_marker
 
     if not manhattan_plot:
         post_data["interval"] = True
diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py
index 1df53fef..290c4a14 100644
--- a/wqflask/wqflask/marker_regression/run_mapping.py
+++ b/wqflask/wqflask/marker_regression/run_mapping.py
@@ -271,47 +271,32 @@ class RunMapping:
                     self.bootCheck = False
                     self.num_bootstrap = 0
 
-            self.reaper_version = start_vars['reaper_version']
-
             self.control_marker = start_vars['control_marker']
             self.do_control = start_vars['do_control']
             logger.info("Running qtlreaper")
 
-            if self.reaper_version == "new":
-                self.first_run = True
-                self.output_files = None
-                # ZS: check if first run so existing result files can be used if it isn't (for example zooming on a chromosome, etc)
-                if 'first_run' in start_vars:
-                    self.first_run = False
-                    if 'output_files' in start_vars:
-                        self.output_files = start_vars['output_files'].split(
-                            ",")
-
-                results, self.perm_output, self.suggestive, self.significant, self.bootstrap_results, self.output_files = qtlreaper_mapping.run_reaper(self.this_trait,
-                                                                                                                                                       self.dataset,
-                                                                                                                                                       self.samples,
-                                                                                                                                                       self.vals,
-                                                                                                                                                       self.json_data,
-                                                                                                                                                       self.num_perm,
-                                                                                                                                                       self.bootCheck,
-                                                                                                                                                       self.num_bootstrap,
-                                                                                                                                                       self.do_control,
-                                                                                                                                                       self.control_marker,
-                                                                                                                                                       self.manhattan_plot,
-                                                                                                                                                       self.first_run,
-                                                                                                                                                       self.output_files)
-            else:
-                results, self.json_data, self.perm_output, self.suggestive, self.significant, self.bootstrap_results = qtlreaper_mapping.run_original_reaper(self.this_trait,
-                                                                                                                                                             self.dataset,
-                                                                                                                                                             self.samples,
-                                                                                                                                                             self.vals,
-                                                                                                                                                             self.json_data,
-                                                                                                                                                             self.num_perm,
-                                                                                                                                                             self.bootCheck,
-                                                                                                                                                             self.num_bootstrap,
-                                                                                                                                                             self.do_control,
-                                                                                                                                                             self.control_marker,
-                                                                                                                                                             self.manhattan_plot)
+            self.first_run = True
+            self.output_files = None
+            # ZS: check if first run so existing result files can be used if it isn't (for example zooming on a chromosome, etc)
+            if 'first_run' in start_vars:
+                self.first_run = False
+                if 'output_files' in start_vars:
+                    self.output_files = start_vars['output_files'].split(
+                        ",")
+
+            results, self.perm_output, self.suggestive, self.significant, self.bootstrap_results, self.output_files = qtlreaper_mapping.run_reaper(self.this_trait,
+                                                                                                                                                    self.dataset,
+                                                                                                                                                    self.samples,
+                                                                                                                                                    self.vals,
+                                                                                                                                                    self.json_data,
+                                                                                                                                                    self.num_perm,
+                                                                                                                                                    self.bootCheck,
+                                                                                                                                                    self.num_bootstrap,
+                                                                                                                                                    self.do_control,
+                                                                                                                                                    self.control_marker,
+                                                                                                                                                    self.manhattan_plot,
+                                                                                                                                                    self.first_run,
+                                                                                                                                                    self.output_files)
         elif self.mapping_method == "plink":
             self.score_type = "-logP"
             self.manhattan_plot = True
diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py
index b28c1b04..c54dd0b3 100644
--- a/wqflask/wqflask/resource_manager.py
+++ b/wqflask/wqflask/resource_manager.py
@@ -8,8 +8,6 @@ from wqflask import app
 from utility.authentication_tools import check_owner_or_admin
 from utility.redis_tools import get_resource_info, get_group_info, get_groups_like_unique_column, get_user_id, get_user_by_unique_column, get_users_like_unique_column, add_access_mask, add_resource, change_resource_owner
 
-from utility.logger import getLogger
-logger = getLogger(__name__)
 
 
 @app.route("/resources/manage", methods=('GET', 'POST'))
diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js
index b75d658e..e42fe8c4 100644
--- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js
+++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js
@@ -145,7 +145,7 @@ var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form
                           'sample_vals', 'vals_hash', 'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars',
                           'num_bootstrap', 'bootCheck', 'bootstrap_results', 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker',
                           'do_control', 'genofile', 'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend',
-                          'haplotypeAnalystCheck', 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'reaper_version', 'primary_samples']
+                          'haplotypeAnalystCheck', 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'primary_samples']
 
 $(".rqtl-geno-tab, #rqtl_geno_compute").on("click", (function(_this) {
   return function() {
diff --git a/wqflask/wqflask/templates/admin/group_manager.html b/wqflask/wqflask/templates/admin/group_manager.html
index c0b99e75..692a7abc 100644
--- a/wqflask/wqflask/templates/admin/group_manager.html
+++ b/wqflask/wqflask/templates/admin/group_manager.html
@@ -81,7 +81,7 @@
                         <tr>
                             <td><input type="checkbox" name="read" value="{{ group.id }}"></td>
                             <td>{{ loop.index }}</td>
-                            <td>{{ group.name }}</td>
+                            <td><a href="/groups/view?id={{ group.id }}">{{ group.name }}</a></td>
                             <td>{{ group.admins|length + group.members|length }}</td>
                             <td>{{ group.created_timestamp }}</td>
                             <td>{{ group.changed_timestamp }}</td>
diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html
index c5479b17..d347a060 100644
--- a/wqflask/wqflask/templates/collections/view.html
+++ b/wqflask/wqflask/templates/collections/view.html
@@ -71,7 +71,7 @@
                     <input type="text" id="select_top" class="form-control" style="width: 200px; display: inline; padding-bottom: 9px;" placeholder="Select Top ...">
                     <button class="btn btn-default" id="deselect_all" type="button"><span class="glyphicon glyphicon-remove"></span> Deselect</button>
                     <button id="remove" class="btn btn-danger" data-url="/collections/remove" type="button" disabled><i class="icon-minus-sign"></i> Delete Rows</button>
-                    <button id="delete" class="btn btn-danger submit_special" data-url="/collections/delete" title="Delete this collection" > Delete Collection</button>
+                    <button id="delete" class="btn btn-danger submit_special" data-url="/collections/delete" type="button" title="Delete this collection" > Delete Collection</button>
                 </form>
             </div>
 	    <div id="clustered-heatmap-image-area">
diff --git a/wqflask/wqflask/templates/new_security/_scripts.html b/wqflask/wqflask/templates/new_security/_scripts.html
deleted file mode 100644
index 5fefe305..00000000
--- a/wqflask/wqflask/templates/new_security/_scripts.html
+++ /dev/null
@@ -1 +0,0 @@
-<!--<script type="text/javascript" src="/static/new/javascript/login.js"></script>-->
diff --git a/wqflask/wqflask/templates/new_security/forgot_password.html b/wqflask/wqflask/templates/new_security/forgot_password.html
index e5c42a45..60a221da 100644
--- a/wqflask/wqflask/templates/new_security/forgot_password.html
+++ b/wqflask/wqflask/templates/new_security/forgot_password.html
@@ -48,6 +48,5 @@
   {% endblock %}
 
 {% block js %}
-    {% include "new_security/_scripts.html" %}
 
 {% endblock %}
diff --git a/wqflask/wqflask/templates/new_security/forgot_password_step2.html b/wqflask/wqflask/templates/new_security/forgot_password_step2.html
index b4bf41c7..1835fd4c 100644
--- a/wqflask/wqflask/templates/new_security/forgot_password_step2.html
+++ b/wqflask/wqflask/templates/new_security/forgot_password_step2.html
@@ -20,7 +20,6 @@
 {% endblock %}
 
 {% block js %}
-    {% include "new_security/_scripts.html" %}
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='zxcvbn/zxcvbn.js') }}"></script>
     <script type="text/javascript" src="/static/new/javascript/password_strength.js"></script>
 {% endblock %}
diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html
index 095036f0..88eab6bc 100644
--- a/wqflask/wqflask/templates/new_security/login_user.html
+++ b/wqflask/wqflask/templates/new_security/login_user.html
@@ -114,31 +114,5 @@ label.error,div.error{
 {% endblock %}
 
 {% block js %}
-    <!-- Disable plugin, see https://github.com/genenetwork/genenetwork2/issues/310
-
-    <script type="text/javascript" src="/static/new/packages/ValidationPlugin/dist/jquery.validate.min.js"></script>
-    <script>
-    $(document).ready(function () {
-        $("#loginUserForm").validate({
-            onkeyup: false,
-            onsubmit: true,
-            onfocusout: function(element) { $(element).valid(); },
-            rules: {
-                email_address: {
-                    required: true,
-                    email: true
-                },
-                password: {
-                    required: true
-                }
-            }
-        });
-    });
-
-    </script>
-
-    -->
-
-    {% include "new_security/_scripts.html" %}
 
 {% endblock %}
diff --git a/wqflask/wqflask/templates/new_security/password_reset.html b/wqflask/wqflask/templates/new_security/password_reset.html
index 684c12b1..e21f075c 100644
--- a/wqflask/wqflask/templates/new_security/password_reset.html
+++ b/wqflask/wqflask/templates/new_security/password_reset.html
@@ -73,7 +73,6 @@
 
 {% block js %}
 
-    {% include "new_security/_scripts.html" %}
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='zxcvbn/zxcvbn.js') }}"></script>
     <script type="text/javascript" src="/static/new/javascript/password_strength.js"></script>
 {% endblock %}
diff --git a/wqflask/wqflask/templates/new_security/register_user.html b/wqflask/wqflask/templates/new_security/register_user.html
index 3ae4488b..c2895517 100644
--- a/wqflask/wqflask/templates/new_security/register_user.html
+++ b/wqflask/wqflask/templates/new_security/register_user.html
@@ -100,7 +100,6 @@
 
 {% block js %}
 
-    {% include "new_security/_scripts.html" %}
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='zxcvbn/zxcvbn.js') }}"></script>
     <script type="text/javascript" src="/static/new/javascript/password_strength.js"></script>
 {% endblock %}
diff --git a/wqflask/wqflask/templates/new_security/registered.html b/wqflask/wqflask/templates/new_security/registered.html
index f2f58ec1..29889a97 100644
--- a/wqflask/wqflask/templates/new_security/registered.html
+++ b/wqflask/wqflask/templates/new_security/registered.html
@@ -19,7 +19,6 @@
 
 {% block js %}
 
-    {% include "new_security/_scripts.html" %}
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='zxcvbn/zxcvbn.js') }}"></script>
     <script type="text/javascript" src="/static/new/javascript/password_strength.js"></script>
 {% endblock %}
diff --git a/wqflask/wqflask/templates/new_security/thank_you.html b/wqflask/wqflask/templates/new_security/thank_you.html
index 0ff7ee8d..d4f5e574 100644
--- a/wqflask/wqflask/templates/new_security/thank_you.html
+++ b/wqflask/wqflask/templates/new_security/thank_you.html
@@ -18,7 +18,6 @@
 {% endblock %}
 
 {% block js %}
-    {% include "new_security/_scripts.html" %}
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='zxcvbn/zxcvbn.js') }}"></script>
     <script type="text/javascript" src="/static/new/javascript/password_strength.js"></script>
 {% endblock %}
diff --git a/wqflask/wqflask/templates/new_security/verification_still_needed.html b/wqflask/wqflask/templates/new_security/verification_still_needed.html
index dc0f9e68..1f91fd8d 100644
--- a/wqflask/wqflask/templates/new_security/verification_still_needed.html
+++ b/wqflask/wqflask/templates/new_security/verification_still_needed.html
@@ -21,7 +21,6 @@
 {% endblock %}
 
 {% block js %}
-    {% include "new_security/_scripts.html" %}
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='zxcvbn/zxcvbn.js') }}"></script>
     <script type="text/javascript" src="/static/new/javascript/password_strength.js"></script>
 {% endblock %}
diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html
index 7ec335d5..c499aa8f 100644
--- a/wqflask/wqflask/templates/search_result_page.html
+++ b/wqflask/wqflask/templates/search_result_page.html
@@ -53,6 +53,7 @@
                 A total of {{ results|count }} records were found.
         </p>
 
+        {% if results|count > 0 %}
         {% if go_term is not none %}
         <p><b>The associated genes include:</b><br><br>{% for word in search_terms %}{{ word.search_term[0] }}{% endfor %}</p>
         {% endif %}
@@ -133,8 +134,11 @@
           </div>
         </div>
         {% endif %}
+        {% else %}
+        <br>
+        <button type="button" onclick="window.location.href='/'">Return To Index Page</button>
+        {% endif %}
     </div>
-
     <div id="myModal"></div>
 
 <!-- End of body -->
@@ -171,6 +175,7 @@
                 return params;
             };
 
+            {% if results|count > 0 %}
             //ZS: Need to make sort by symbol, also need to make sure blank symbol fields at the bottom and symbols starting with numbers below letters
             trait_table = $('#trait_table').DataTable( {
                 'drawCallback': function( settings ) {
@@ -412,6 +417,7 @@
                 var table = $('#trait_table').DataTable();
                 table.colReorder.reset()
             });
+            {% endif %}
 
             submit_special = function(url) {
                 $("#trait_submission_form").attr("action", url);
diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html
index 36d3c15a..2a21dd24 100644
--- a/wqflask/wqflask/templates/show_trait_details.html
+++ b/wqflask/wqflask/templates/show_trait_details.html
@@ -233,7 +233,7 @@
         <button type="button" class="btn btn-default" title="Check sequence of probes" onclick="window.open('http://gn1.genenetwork.org/webqtl/main.py?FormID=showProbeInfo&database={{ this_trait.dataset.name }}&ProbeSetID={{ this_trait.name }}&CellID={{ this_trait.cellid }}&RISet={{ dataset.group.name }}&incparentsf1=ON', '_blank')">Probes</button>
         {% endif %}
         {% endif %}
-        <button type="button" id="view_in_gn1" class="btn btn-primary" title="View Trait in GN1" onclick="window.open('http://gn1-lily.genenetwork.org/webqtl/main.py?cmd=show&db={{ this_trait.dataset.name }}&probeset={{ this_trait.name }}', '_blank')">Go to GN1</button>
+        <button type="button" id="view_in_gn1" class="btn btn-primary" title="View Trait in GN1" onclick="window.open('http://gn1.genenetwork.org/webqtl/main.py?cmd=show&db={{ this_trait.dataset.name }}&probeset={{ this_trait.name }}', '_blank')">Go to GN1</button>
         {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %}
         {% if this_trait.dataset.type == 'Publish' %}
         <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('/trait/{{ this_trait.name }}/edit/inbredset-id/{{ this_trait.dataset.id }}', '_blank')">Edit</button>
@@ -242,6 +242,9 @@
         {% if this_trait.dataset.type == 'ProbeSet' %}
         <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('/trait/edit/probeset-name/{{ this_trait.name }}', '_blank')">Edit</button>
         {% endif %}
+        {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %}
+        <button type="button" id="edit_resource" class="btn btn-success" title="Edit Resource" onclick="window.open('./resources/manage?resource_id={{ resource_id }}', '_blank')">Edit Privileges</button>
+        {% endif %}
         {% endif %}
     </div>
 </div>
diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html
index 5365140d..3af94ed6 100755
--- a/wqflask/wqflask/templates/show_trait_mapping_tools.html
+++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html
@@ -96,15 +96,6 @@
                 <div class="tab-pane" id="interval_mapping">
                     <div class="form-horizontal section-form-div">
                         <div class="mapping_method_fields form-group">
-                            <label for="reaper_version" class="col-xs-3 control-label">Version<sup><a href="https://github.com/chfi/rust-qtlreaper" target="_blank" title="'New' is the new qtlreaper implementation written in RUST by Christian Fischer. 'Original' corresponds to the original version written in C.">?</a></sup></label>
-                            <div class="col-xs-3 controls">
-                                <select name="reaper_version" class="form-control reaper-ver-select">
-                                    <option value="new">New</option>
-                                    <option value="original">Original</option>
-                                </select>
-                            </div>
-                        </div>
-                        <div class="mapping_method_fields form-group">
                             <label for="chr_select" class="col-xs-3 control-label">Chromosome</label>
                             <div class="col-xs-2 controls">
                                     <select id="chr_reaper" class="form-control chr-select">
@@ -263,21 +254,6 @@
                         </div>
                         {% endif %}
                         <div class="mapping_method_fields form-group">
-                            <label for="control_for" class="col-xs-3 control-label">Control&nbsp;for</label>
-                            <div class="col-xs-6 controls">
-                              <input name="control_rqtl_geno" value="{% if dataset.type == 'ProbeSet' and this_trait.locus_chr != '' %}{{ nearest_marker }}{% endif %}" type="text" class="form-control cofactor-input" />
-                              <label class="radio-inline">
-                                  <input type="radio" name="do_control_rqtl" value="true">
-                                  Yes
-                              </label>
-                              <label class="radio-inline">
-                                  <input type="radio" name="do_control_rqtl" value="false" checked="">
-                                  No
-                              </label>
-                            </div>
-                        </div>
-
-                        <div class="mapping_method_fields form-group">
                             <label for="mapmodel_rqtl_geno" class="col-xs-3 control-label">Model</label>
                             <div class="col-xs-4 controls">
                               <select id="mapmodel_rqtl_geno" name="mapmodel_rqtl_geno" class="form-control">
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 85aa6b17..5067ca0e 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -85,7 +85,7 @@ from wqflask.export_traits import export_search_results_csv
 from wqflask.gsearch import GSearch
 from wqflask.update_search_results import GSearch as UpdateGSearch
 from wqflask.docs import Docs, update_text
-from wqflask.decorators import admin_login_required
+from wqflask.decorators import edit_access_required
 from wqflask.db_info import InfoPage
 
 from utility import temp_data
@@ -160,28 +160,37 @@ def shutdown_session(exception=None):
 
 
 @app.errorhandler(Exception)
-def handle_bad_request(e):
+def handle_generic_exceptions(e):
+    import werkzeug
     err_msg = str(e)
-    logger.error(err_msg)
-    logger.error(request.url)
-    # get the stack trace and send it to the logger
-    exc_type, exc_value, exc_traceback = sys.exc_info()
-    logger.error(traceback.format_exc())
     now = datetime.datetime.utcnow()
     time_str = now.strftime('%l:%M%p UTC %b %d, %Y')
-    formatted_lines = [request.url
-                       + " (" + time_str + ")"] + traceback.format_exc().splitlines()
-
+    # get the stack trace and send it to the logger
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+    formatted_lines = {f"{request.url} ({time_str}) "
+                       f" {traceback.format_exc().splitlines()}"}
+
+    _message_templates = {
+        werkzeug.exceptions.NotFound: ("404: Not Found: "
+                                       f"{time_str}: {request.url}"),
+        werkzeug.exceptions.BadRequest: ("400: Bad Request: "
+                                         f"{time_str}: {request.url}"),
+        werkzeug.exceptions.RequestTimeout: ("408: Request Timeout: "
+                                             f"{time_str}: {request.url}")}
+    # Default to the lengthy stack trace!
+    logger.error(_message_templates.get(exc_type,
+                                        formatted_lines))
     # Handle random animations
     # Use a cookie to have one animation on refresh
     animation = request.cookies.get(err_msg[:32])
     if not animation:
-        list = [fn for fn in os.listdir(
-            "./wqflask/static/gif/error") if fn.endswith(".gif")]
-        animation = random.choice(list)
+        animation = random.choice([fn for fn in os.listdir(
+            "./wqflask/static/gif/error") if fn.endswith(".gif")])
 
     resp = make_response(render_template("error.html", message=err_msg,
-                                         stack=formatted_lines, error_image=animation, version=GN_VERSION))
+                                         stack=formatted_lines,
+                                         error_image=animation,
+                                         version=GN_VERSION))
 
     # logger.error("Set cookie %s with %s" % (err_msg, animation))
     resp.set_cookie(err_msg[:32], animation)
@@ -411,7 +420,7 @@ def submit_trait_form():
 
 
 @app.route("/trait/<name>/edit/inbredset-id/<inbredset_id>")
-@admin_login_required
+@edit_access_required
 def edit_phenotype(name, inbredset_id):
     conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"),
                            user=current_app.config.get("DB_USER"),
@@ -468,7 +477,7 @@ def edit_phenotype(name, inbredset_id):
 
 
 @app.route("/trait/edit/probeset-name/<dataset_name>")
-@admin_login_required
+@edit_access_required
 def edit_probeset(dataset_name):
     conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"),
                            user=current_app.config.get("DB_USER"),
@@ -511,7 +520,7 @@ def edit_probeset(dataset_name):
 
 
 @app.route("/trait/update", methods=["POST"])
-@admin_login_required
+@edit_access_required
 def update_phenotype():
     conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"),
                            user=current_app.config.get("DB_USER"),
@@ -637,7 +646,7 @@ def update_phenotype():
 
 
 @app.route("/probeset/update", methods=["POST"])
-@admin_login_required
+@edit_access_required
 def update_probeset():
     conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"),
                            user=current_app.config.get("DB_USER"),
@@ -1097,7 +1106,6 @@ def mapping_results_page():
         'mapmethod_rqtl_geno',
         'mapmodel_rqtl_geno',
         'temp_trait',
-        'reaper_version',
         'n_samples',
         'transform'
     )
@@ -1373,7 +1381,7 @@ def get_sample_data_as_csv(trait_name: int, phenotype_id: int):
 
 
 @app.route("/admin/data-sample/diffs/")
-@admin_login_required
+@edit_access_required
 def display_diffs_admin():
     TMPDIR = current_app.config.get("TMPDIR")
     DIFF_DIR = f"{TMPDIR}/sample-data/diffs"