Merge branch 'testing' of github.com:genenetwork/genenetwork2 into acenteno

10 files changed, 183 insertions, 42 deletions

diff --git a/wqflask/maintenance/convert_dryad_to_bimbam.py b/wqflask/maintenance/convert_dryad_to_bimbam.py
index e833b395..12ce35e9 100644
--- a/wqflask/maintenance/convert_dryad_to_bimbam.py
+++ b/wqflask/maintenance/convert_dryad_to_bimbam.py
@@ -6,7 +6,6 @@ Convert data dryad files to a BIMBAM _geno and _snps file
 
 """
 
-from __future__ import print_function, division, absolute_import
 import sys
 sys.path.append("..")
 
@@ -67,4 +66,4 @@ def convert_dryad_to_bimbam(filename):
 
 if __name__=="__main__":
     input_filename = "/home/zas1024/cfw_data/" + sys.argv[1] + ".txt"
-    convert_dryad_to_bimbam(input_filename)
\ No newline at end of file
+    convert_dryad_to_bimbam(input_filename)
diff --git a/wqflask/maintenance/convert_geno_to_bimbam.py b/wqflask/maintenance/convert_geno_to_bimbam.py
index 528b98cf..d49742f2 100644
--- a/wqflask/maintenance/convert_geno_to_bimbam.py
+++ b/wqflask/maintenance/convert_geno_to_bimbam.py
@@ -9,7 +9,6 @@ code
 
 """
 
-from __future__ import print_function, division, absolute_import
 import sys
 sys.path.append("..")
 import os
@@ -187,4 +186,4 @@ if __name__=="__main__":
     #convertob = ConvertGenoFile("/home/zas1024/gene/genotype_files/genotypes/SRxSHRSPF2.geno", "/home/zas1024/gene/genotype_files/new_genotypes/SRxSHRSPF2.json")
     #convertob.convert()
     ConvertGenoFile.process_all(Old_Geno_Directory, New_Geno_Directory)
-    #ConvertGenoFiles(Geno_Directory)
\ No newline at end of file
+    #ConvertGenoFiles(Geno_Directory)
diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py
index 647e58a2..544e2fd1 100644
--- a/wqflask/maintenance/gen_select_dataset.py
+++ b/wqflask/maintenance/gen_select_dataset.py
@@ -30,18 +30,10 @@ It needs to be run manually when database has been changed. Run it as
 #
 # This module is used by GeneNetwork project (www.genenetwork.org)
 
-from __future__ import print_function, division
-
-#from flask import config
-#
-#cdict = {}
-#config = config.Config(cdict).from_envvar('WQFLASK_SETTINGS')
-#print("cdict is:", cdict)
-
 import sys
 
 # NEW: Note we prepend the current path - otherwise a guix instance of GN2 may be used instead
-sys.path.insert(0,'./')
+sys.path.insert(0, './')
 # NEW: import app to avoid a circular dependency on utility.tools
 from wqflask import app
 
@@ -50,7 +42,7 @@ from utility.tools import locate, locate_ignore_error, TEMPDIR, SQL_URI
 import MySQLdb
 
 import simplejson as json
-import urlparse
+import urllib.parse
 
 
 #import sqlalchemy as sa
@@ -66,7 +58,7 @@ from pprint import pformat as pf
 def parse_db_uri():
     """Converts a database URI to the db name, host name, user name, and password"""
 
-    parsed_uri = urlparse.urlparse(SQL_URI)
+    parsed_uri = urllib.parse.urlparse(SQL_URI)
 
     db_conn_info = dict(
                         db = parsed_uri.path[1:],
@@ -108,7 +100,7 @@ def get_types(groups):
     """Build types list"""
     types = {}
     #print("Groups: ", pf(groups))
-    for species, group_dict in groups.iteritems():
+    for species, group_dict in list(groups.items()):
         types[species] = {}
         for group_name, _group_full_name in group_dict:
             # make group an alias to shorten the code
@@ -195,9 +187,9 @@ def build_types(species, group):
 def get_datasets(types):
     """Build datasets list"""
     datasets = {}
-    for species, group_dict in types.iteritems():
+    for species, group_dict in list(types.items()):
         datasets[species] = {}
-        for group, type_list in group_dict.iteritems():
+        for group, type_list in list(group_dict.items()):
             datasets[species][group] = {}
             for type_name in type_list:
                 these_datasets = build_datasets(species, group, type_name[0])
@@ -319,4 +311,4 @@ def _test_it():
 if __name__ == '__main__':
     Conn = MySQLdb.Connect(**parse_db_uri())
     Cursor = Conn.cursor()
-    main()
\ No newline at end of file
+    main()
diff --git a/wqflask/maintenance/generate_kinship_from_bimbam.py b/wqflask/maintenance/generate_kinship_from_bimbam.py
index b53f5dda..60257b28 100644
--- a/wqflask/maintenance/generate_kinship_from_bimbam.py
+++ b/wqflask/maintenance/generate_kinship_from_bimbam.py
@@ -8,7 +8,6 @@ and uses GEMMA to generate their corresponding kinship/relatedness matrix file
 
 """
 
-from __future__ import print_function, division, absolute_import
 import sys
 sys.path.append("..")
 import os
@@ -58,4 +57,4 @@ if __name__=="__main__":
     Bimbam_Directory = """/export/local/home/zas1024/genotype_files/genotype/bimbam/"""
     GenerateKinshipMatrices.process_all(Geno_Directory, Bimbam_Directory)
     
-    #./gemma -g /home/zas1024/genotype_files/genotype/bimbam/BXD_geno.txt -p /home/zas1024/genotype_files/genotype/bimbam/BXD_pheno.txt -gk 1 -o BXD
\ No newline at end of file
+    #./gemma -g /home/zas1024/genotype_files/genotype/bimbam/BXD_geno.txt -p /home/zas1024/genotype_files/genotype/bimbam/BXD_pheno.txt -gk 1 -o BXD
diff --git a/wqflask/maintenance/generate_probesetfreeze_file.py b/wqflask/maintenance/generate_probesetfreeze_file.py
index b7b2dc8e..b1e41e9a 100644
--- a/wqflask/maintenance/generate_probesetfreeze_file.py
+++ b/wqflask/maintenance/generate_probesetfreeze_file.py
@@ -1,7 +1,5 @@
 #!/usr/bin/python
 
-from __future__ import absolute_import, print_function, division
-
 import sys
 
 # sys.path.insert(0, "..") - why?
@@ -82,7 +80,7 @@ def get_probeset_vals(cursor, dataset_name):
 def trim_strains(strains, probeset_vals):
     trimmed_strains = []
     #print("probeset_vals is:", pf(probeset_vals))
-    first_probeset = list(probeset_vals.itervalues())[0]
+    first_probeset = list(probeset_vals.values())[0]
     print("\n**** first_probeset is:", pf(first_probeset))
     for strain in strains:
         print("\n**** strain is:", pf(strain))
diff --git a/wqflask/maintenance/geno_to_json.py b/wqflask/maintenance/geno_to_json.py
index 9579812a..7e7fd241 100644
--- a/wqflask/maintenance/geno_to_json.py
+++ b/wqflask/maintenance/geno_to_json.py
@@ -9,7 +9,6 @@ code
 
 """
 
-from __future__ import print_function, division, absolute_import
 import sys
 sys.path.append("..")
 import os
@@ -194,4 +193,4 @@ if __name__=="__main__":
     ConvertGenoFile.process_all(Old_Geno_Directory, New_Geno_Directory)
     #ConvertGenoFiles(Geno_Directory)
     
-    #process_csv(Input_File, Output_File)
\ No newline at end of file
+    #process_csv(Input_File, Output_File)
diff --git a/wqflask/maintenance/get_group_samplelists.py b/wqflask/maintenance/get_group_samplelists.py
index fb22898a..3f9d0278 100644
--- a/wqflask/maintenance/get_group_samplelists.py
+++ b/wqflask/maintenance/get_group_samplelists.py
@@ -1,5 +1,3 @@
-from __future__ import absolute_import, print_function, division
-
 import os
 import glob
 import gzip
diff --git a/wqflask/maintenance/print_benchmark.py b/wqflask/maintenance/print_benchmark.py
index ae327cf3..b24ce4f1 100644
--- a/wqflask/maintenance/print_benchmark.py
+++ b/wqflask/maintenance/print_benchmark.py
@@ -1,7 +1,5 @@
 #!/usr/bin/python
 
-from __future__ import absolute_import, print_function, division
-
 import time
 
 from pprint import pformat as pf
@@ -40,4 +38,4 @@ def new_main():
     print(pf(TheCounter.Counters))
 
 if __name__ == '__main__':
-    new_main()
\ No newline at end of file
+    new_main()
diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py
index 41a3aad8..701b2b50 100644
--- a/wqflask/maintenance/quantile_normalize.py
+++ b/wqflask/maintenance/quantile_normalize.py
@@ -1,12 +1,7 @@
-from __future__ import absolute_import, print_function, division
-
 import sys
-sys.path.insert(0,'./')
-
-from itertools import izip
-
+sys.path.insert(0, './')
 import MySQLdb
-import urlparse
+import urllib.parse
 
 import numpy as np
 import pandas as pd
@@ -22,7 +17,7 @@ from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT, SQL_URI
 def parse_db_uri():
     """Converts a database URI to the db name, host name, user name, and password"""
 
-    parsed_uri = urlparse.urlparse(SQL_URI)
+    parsed_uri = urllib.parse.urlparse(SQL_URI)
 
     db_conn_info = dict(
                         db = parsed_uri.path[1:],
@@ -37,7 +32,7 @@ def create_dataframe(input_file):
     with open(input_file) as f:
         ncols = len(f.readline().split("\t"))
 
-    input_array = np.loadtxt(open(input_file, "rb"), delimiter="\t", skiprows=1, usecols=range(1, ncols))
+    input_array = np.loadtxt(open(input_file, "rb"), delimiter="\t", skiprows=1, usecols=list(range(1, ncols)))
     return pd.DataFrame(input_array)
 
 #This function taken from https://github.com/ShawnLYU/Quantile_Normalize
@@ -60,7 +55,7 @@ def set_data(dataset_name):
 
     sample_list = []
     with open(orig_file, 'r') as orig_fh, open('/home/zas1024/cfw_data/quant_norm.csv', 'r') as quant_fh:
-        for i, (line1, line2) in enumerate(izip(orig_fh, quant_fh)):
+        for i, (line1, line2) in enumerate(zip(orig_fh, quant_fh)):
             trait_dict = {}
             sample_list = []
             if i == 0:
@@ -126,4 +121,4 @@ if __name__ == '__main__':
         }
     )
 
-    print(response)
\ No newline at end of file
+    print(response)
diff --git a/wqflask/maintenance/set_resource_defaults.py b/wqflask/maintenance/set_resource_defaults.py
new file mode 100644
index 00000000..4177c124
--- /dev/null
+++ b/wqflask/maintenance/set_resource_defaults.py
@@ -0,0 +1,164 @@
+"""
+
+Script that sets default resource access masks for use with the DB proxy
+
+Defaults will be:
+Owner - omni_gn
+Mask  - Public/non-confidential: { data: "view",
+                                   metadata: "view",
+                                   admin: "not-admin" }
+        Private/confidentia:     { data: "no-access",
+                                   metadata: "no-access",
+                                   admin: "not-admin" }
+
+To run:
+./bin/genenetwork2 ~/my_settings.py -c ./wqflask/maintenance/gen_select_dataset.py
+
+"""
+
+import sys
+import json
+
+# NEW: Note we prepend the current path - otherwise a guix instance of GN2 may be used instead
+sys.path.insert(0, './')
+
+# NEW: import app to avoid a circular dependency on utility.tools
+from wqflask import app
+
+from utility import hmac
+from utility.tools import SQL_URI
+from utility.redis_tools import get_redis_conn, get_user_id, add_resource, get_resources, get_resource_info
+Redis = get_redis_conn()
+
+import MySQLdb
+
+import urllib.parse
+
+from utility.logger import getLogger
+logger = getLogger(__name__)
+
+def parse_db_uri():
+    """Converts a database URI to the db name, host name, user name, and password"""
+
+    parsed_uri = urllib.parse.urlparse(SQL_URI)
+
+    db_conn_info = dict(
+                        db = parsed_uri.path[1:],
+                        host = parsed_uri.hostname,
+                        user = parsed_uri.username,
+                        passwd = parsed_uri.password)
+
+    print(db_conn_info)
+    return db_conn_info
+
+def insert_probeset_resources(default_owner_id):
+    current_resources = Redis.hgetall("resources")
+    Cursor.execute("""  SELECT
+                            ProbeSetFreeze.Id, ProbeSetFreeze.Name, ProbeSetFreeze.confidentiality, ProbeSetFreeze.public
+                        FROM
+                            ProbeSetFreeze""")
+
+    resource_results = Cursor.fetchall()
+    for i, resource in enumerate(resource_results):
+        resource_ob = {}
+        resource_ob['name'] = resource[1]
+        resource_ob['owner_id'] = default_owner_id
+        resource_ob['data'] = { "dataset" : str(resource[0])}
+        resource_ob['type'] = "dataset-probeset"
+        if resource[2] < 1 and resource[3] > 0:
+            resource_ob['default_mask'] = { "data": "view",
+                                            "metadata": "view",
+                                            "admin": "not-admin"}
+        else:
+            resource_ob['default_mask'] = { "data": "no-access",
+                                            "metadata": "no-access",
+                                            "admin": "not-admin"}
+        resource_ob['group_masks'] = {}
+
+        add_resource(resource_ob, update=False)
+
+def insert_publish_resources(default_owner_id):
+    current_resources = Redis.hgetall("resources")
+    Cursor.execute("""  SELECT 
+                            PublishXRef.Id, PublishFreeze.Id, InbredSet.InbredSetCode
+                        FROM
+                            PublishXRef, PublishFreeze, InbredSet, Publication
+                        WHERE
+                            PublishFreeze.InbredSetId = PublishXRef.InbredSetId AND
+                            InbredSet.Id = PublishXRef.InbredSetId AND
+                            Publication.Id = PublishXRef.PublicationId""")
+
+    resource_results = Cursor.fetchall()
+    for resource in resource_results:
+        if resource[2]:
+            resource_ob = {}
+            if resource[2]:
+                resource_ob['name'] = resource[2] + "_" + str(resource[0])
+            else:
+                resource_ob['name'] = str(resource[0])
+            resource_ob['owner_id'] = default_owner_id
+            resource_ob['data'] = { "dataset" : str(resource[1]) ,
+                                    "trait"   : str(resource[0])}
+            resource_ob['type'] = "dataset-publish"
+            resource_ob['default_mask'] = { "data": "view",
+                                            "metadata": "view",
+                                            "admin": "not-admin"}
+
+            resource_ob['group_masks'] = {}
+
+            add_resource(resource_ob, update=False)
+        else:
+            continue
+
+def insert_geno_resources(default_owner_id):
+    current_resources = Redis.hgetall("resources")
+    Cursor.execute("""  SELECT
+                            GenoFreeze.Id, GenoFreeze.ShortName, GenoFreeze.confidentiality
+                        FROM
+                            GenoFreeze""")
+
+    resource_results = Cursor.fetchall()
+    for i, resource in enumerate(resource_results):
+        resource_ob = {}
+        resource_ob['name'] = resource[1]
+        if resource[1] == "HET3-ITPGeno":
+            resource_ob['owner_id'] = "c5ce8c56-78a6-474f-bcaf-7129d97f56ae"
+        else:
+            resource_ob['owner_id'] = default_owner_id
+        resource_ob['data'] = { "dataset" : str(resource[0]) }
+        resource_ob['type'] = "dataset-geno"
+        if resource[2] < 1:
+            resource_ob['default_mask'] = { "data": "view",
+                                            "metadata": "view",
+                                            "admin": "not-admin"}
+        else:
+            resource_ob['default_mask'] = { "data": "no-access",
+                                            "metadata": "no-access",
+                                            "admin": "not-admin"}
+        resource_ob['group_masks'] = {}
+
+        add_resource(resource_ob, update=False)
+
+def insert_resources(default_owner_id):
+    current_resources = get_resources()
+    print("START")
+    insert_publish_resources(default_owner_id)
+    print("AFTER PUBLISH")
+    insert_geno_resources(default_owner_id)
+    print("AFTER GENO")
+    insert_probeset_resources(default_owner_id)
+    print("AFTER PROBESET")
+
+def main():
+    """Generates and outputs (as json file) the data for the main dropdown menus on the home page"""
+
+    Redis.delete("resources")
+
+    owner_id = "c5ce8c56-78a6-474f-bcaf-7129d97f56ae"
+
+    insert_resources(owner_id)
+
+if __name__ == '__main__':
+    Conn = MySQLdb.Connect(**parse_db_uri())
+    Cursor = Conn.cursor()
+    main()