diff options
89 files changed, 446 insertions, 46 deletions
diff --git a/etc/default_settings.py b/etc/default_settings.py index 27522187..a194b10e 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -73,23 +73,24 @@ SMTP_PASSWORD = "UNKNOWN" # ---- Behavioural settings (defaults) note that logger and log levels can # be overridden at the module level and with enviroment settings -WEBSERVER_MODE = 'DEV' # Python webserver mode (DEBUG|DEV|PROD) +WEBSERVER_MODE = 'DEV' # Python webserver mode (DEBUG|DEV|PROD) WEBSERVER_BRANDING = None # Set the branding (nyi) WEBSERVER_DEPLOY = None # Deployment specifics (nyi) -WEBSERVER_URL = "http://localhost:"+str(SERVER_PORT)+"/" # external URL +WEBSERVER_URL = "http://localhost:" + str(SERVER_PORT) + "/" # external URL -LOG_LEVEL = 'WARNING' # Logger mode (DEBUG|INFO|WARNING|ERROR|CRITICAL) +LOG_LEVEL = 'WARNING' # Logger mode (DEBUG|INFO|WARNING|ERROR|CRITICAL) LOG_LEVEL_DEBUG = '0' # logger.debugf log level (0-5, 5 = show all) -LOG_SQL = 'False' # Log SQL/backend and GN_SERVER calls +LOG_SQL = 'False' # Log SQL/backend and GN_SERVER calls LOG_SQL_ALCHEMY = 'False' -LOG_BENCH = True # Log bench marks +LOG_BENCH = True # Log bench marks -USE_REDIS = True # REDIS caching (note that redis will be phased out) -USE_GN_SERVER = 'False' # Use GN_SERVER SQL calls -HOME = os.environ['HOME'] +USE_REDIS = True # REDIS caching (note that redis will be phased out) +USE_GN_SERVER = 'False' # Use GN_SERVER SQL calls +HOME = os.environ['HOME'] # ---- Default locations -GENENETWORK_FILES = HOME+"/genotype_files" # base dir for all static data files +# base dir for all static data files +GENENETWORK_FILES = HOME + "/genotype_files" # ---- Path overrides for Genenetwork - the defaults are normally # picked up from Guix or in the HOME directory @@ -98,7 +99,7 @@ GENENETWORK_FILES = HOME+"/genotype_files" # base dir for all static data fil # PRIVATE_FILES = HOME+"/gn2_private_data" # private static data files (unused) # ---- Local path to JS libraries - for development modules (only) -JS_GN_PATH = os.environ['HOME']+"/genenetwork/javascript" +JS_GN_PATH = os.environ['HOME'] + "/genenetwork/javascript" # ---- GN2 Executables (overwrite for testing only) # PLINK_COMMAND = str.strip(os.popen("which plink2").read()) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 55ab45f5..c14808d6 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -311,7 +311,6 @@ class HumanMarkers(Markers): marker['Mb'] = float(splat[3]) / 1000000 self.markers.append(marker) - def add_pvalues(self, p_values): super(HumanMarkers, self).add_pvalues(p_values) diff --git a/wqflask/base/mrna_assay_tissue_data.py b/wqflask/base/mrna_assay_tissue_data.py index f3264b3d..cbc05738 100644 --- a/wqflask/base/mrna_assay_tissue_data.py +++ b/wqflask/base/mrna_assay_tissue_data.py @@ -11,6 +11,7 @@ from utility.db_tools import escape from utility.logger import getLogger logger = getLogger(__name__) + class MrnaAssayTissueData: def __init__(self, gene_symbols=None): diff --git a/wqflask/base/species.py b/wqflask/base/species.py index cf764d72..44f133b5 100644 --- a/wqflask/base/species.py +++ b/wqflask/base/species.py @@ -6,6 +6,7 @@ from flask import Flask, g from utility.logger import getLogger logger = getLogger(__name__) + class TheSpecies: def __init__(self, dataset=None, species_name=None): if species_name != None: @@ -15,6 +16,7 @@ class TheSpecies: self.dataset = dataset self.chromosomes = Chromosomes(dataset=self.dataset) + class IndChromosome: def __init__(self, name, length): self.name = name @@ -25,6 +27,7 @@ class IndChromosome: """Chromosome length in megabases""" return self.length / 1000000 + class Chromosomes: def __init__(self, dataset=None, species=None): self.chromosomes = collections.OrderedDict() diff --git a/wqflask/base/webqtlCaseData.py b/wqflask/base/webqtlCaseData.py index aa55470f..2d07ab9d 100644 --- a/wqflask/base/webqtlCaseData.py +++ b/wqflask/base/webqtlCaseData.py @@ -28,6 +28,7 @@ import utility.tools utility.tools.show_settings() + class webqtlCaseData: """one case data in one trait""" @@ -78,4 +79,4 @@ class webqtlCaseData: def display_num_cases(self): if self.num_cases is not None: return "%s" % self.num_cases - return "x"
\ No newline at end of file + return "x" diff --git a/wqflask/db/call.py b/wqflask/db/call.py index 555878ad..9412b376 100644 --- a/wqflask/db/call.py +++ b/wqflask/db/call.py @@ -16,6 +16,7 @@ logger = getLogger(__name__) # from inspect import stack + def fetch1(query, path=None, func=None): """Fetch one result as a Tuple using either a SQL query or the URI path to GN_SERVER (when USE_GN_SERVER is True). Apply func to @@ -35,6 +36,7 @@ GN_SERVER result when set (which should return a Tuple) else: return fetchone(query) + def fetchone(query): """Return tuple containing one row by calling SQL directly (the original fetchone, but with logging) @@ -46,6 +48,7 @@ original fetchone, but with logging) return res.fetchone() return logger.sql(query, helper) + def fetchall(query): """Return row iterator by calling SQL directly (the original fetchall, but with logging) @@ -57,6 +60,7 @@ original fetchall, but with logging) return res.fetchall() return logger.sql(query, helper) + def gn_server(path): """Return JSON record by calling GN_SERVER diff --git a/wqflask/db/gn_server.py b/wqflask/db/gn_server.py index 6c7383d0..f9b01658 100644 --- a/wqflask/db/gn_server.py +++ b/wqflask/db/gn_server.py @@ -5,5 +5,6 @@ from db.call import gn_server from utility.logger import getLogger logger = getLogger(__name__) + def menu_main(): return gn_server("/int/menu/main.json") diff --git a/wqflask/db/webqtlDatabaseFunction.py b/wqflask/db/webqtlDatabaseFunction.py index 18ade405..50ac06fd 100644 --- a/wqflask/db/webqtlDatabaseFunction.py +++ b/wqflask/db/webqtlDatabaseFunction.py @@ -31,6 +31,7 @@ logger = getLogger(__name__) # function: connect to database and return cursor instance ########################################################################### + def retrieve_species(group): """Get the species of a group (e.g. returns string "mouse" on "BXD" diff --git a/wqflask/maintenance/convert_dryad_to_bimbam.py b/wqflask/maintenance/convert_dryad_to_bimbam.py index 8eab66e8..e417c280 100644 --- a/wqflask/maintenance/convert_dryad_to_bimbam.py +++ b/wqflask/maintenance/convert_dryad_to_bimbam.py @@ -55,15 +55,18 @@ def read_dryad_file(filename): # # return geno_rows + def write_bimbam_files(geno_rows): with open('/home/zas1024/cfw_data/CFW_geno.txt', 'w') as geno_fh: for row in geno_rows: geno_fh.write(", ".join(row) + "\n") + def convert_dryad_to_bimbam(filename): geno_file_rows = read_dryad_file(filename) write_bimbam_files(geno_file_rows) + if __name__ == "__main__": input_filename = "/home/zas1024/cfw_data/" + sys.argv[1] + ".txt" convert_dryad_to_bimbam(input_filename) diff --git a/wqflask/maintenance/convert_geno_to_bimbam.py b/wqflask/maintenance/convert_geno_to_bimbam.py index dc01cbb3..5b2369c9 100644 --- a/wqflask/maintenance/convert_geno_to_bimbam.py +++ b/wqflask/maintenance/convert_geno_to_bimbam.py @@ -20,8 +20,10 @@ import simplejson as json from pprint import pformat as pf + class EmptyConfigurations(Exception): pass + class Marker: def __init__(self): self.name = None @@ -30,6 +32,7 @@ class Marker: self.Mb = None self.genotypes = [] + class ConvertGenoFile: def __init__(self, input_file, output_files): @@ -178,6 +181,7 @@ class ConvertGenoFile: print(" Row is:", convertob.latest_row_value) break + if __name__ == "__main__": Old_Geno_Directory = """/export/local/home/zas1024/gn2-zach/genotype_files/genotype""" New_Geno_Directory = """/export/local/home/zas1024/gn2-zach/genotype_files/genotype/bimbam""" diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py index f480d63f..583a06e1 100644 --- a/wqflask/maintenance/gen_select_dataset.py +++ b/wqflask/maintenance/gen_select_dataset.py @@ -55,6 +55,7 @@ from pprint import pformat as pf #conn = Engine.connect() + def parse_db_uri(): """Converts a database URI to the db name, host name, user name, and password""" @@ -143,6 +144,7 @@ def phenotypes_exist(group_name): else: return False + def genotypes_exist(group_name): #print("group_name:", group_name) Cursor.execute("""select Name from GenoFreeze @@ -156,6 +158,7 @@ def genotypes_exist(group_name): else: return False + def build_types(species, group): """Fetches tissues @@ -184,6 +187,7 @@ def build_types(species, group): return results + def get_datasets(types): """Build datasets list""" datasets = {} @@ -308,6 +312,7 @@ def _test_it(): datasets = build_datasets("Mouse", "BXD", "Hippocampus") #print("build_datasets:", pf(datasets)) + if __name__ == '__main__': Conn = MySQLdb.Connect(**parse_db_uri()) Cursor = Conn.cursor() diff --git a/wqflask/maintenance/generate_kinship_from_bimbam.py b/wqflask/maintenance/generate_kinship_from_bimbam.py index 664e9e48..7cc60c9e 100644 --- a/wqflask/maintenance/generate_kinship_from_bimbam.py +++ b/wqflask/maintenance/generate_kinship_from_bimbam.py @@ -13,6 +13,7 @@ sys.path.append("..") import os import glob + class GenerateKinshipMatrices: def __init__(self, group_name, geno_file, pheno_file): self.group_name = group_name diff --git a/wqflask/maintenance/generate_probesetfreeze_file.py b/wqflask/maintenance/generate_probesetfreeze_file.py index b1e41e9a..bd9c2ab4 100644 --- a/wqflask/maintenance/generate_probesetfreeze_file.py +++ b/wqflask/maintenance/generate_probesetfreeze_file.py @@ -23,10 +23,12 @@ def get_cursor(): cursor = con.cursor() return cursor + def show_progress(process, counter): if counter % 1000 == 0: print("{}: {}".format(process, counter)) + def get_strains(cursor): cursor.execute("""select Strain.Name from Strain, StrainXRef, InbredSet @@ -42,6 +44,7 @@ def get_strains(cursor): return strains + def get_probeset_vals(cursor, dataset_name): cursor.execute(""" select ProbeSet.Id, ProbeSet.Name from ProbeSetXRef, @@ -77,6 +80,7 @@ def get_probeset_vals(cursor, dataset_name): return probeset_vals + def trim_strains(strains, probeset_vals): trimmed_strains = [] #print("probeset_vals is:", pf(probeset_vals)) @@ -89,6 +93,7 @@ def trim_strains(strains, probeset_vals): print("trimmed_strains:", pf(trimmed_strains)) return trimmed_strains + def write_data_matrix_file(strains, probeset_vals, filename): with open(filename, "wb") as fh: csv_writer = csv.writer(fh, delimiter=",", quoting=csv.QUOTE_ALL) @@ -103,6 +108,7 @@ def write_data_matrix_file(strains, probeset_vals, filename): csv_writer.writerow(row_data) show_progress("Writing", counter) + def main(): filename = os.path.expanduser("~/gene/wqflask/maintenance/" + "ProbeSetFreezeId_210_FullName_Eye_AXBXA_Illumina_V6.2" + @@ -117,5 +123,6 @@ def main(): trimmed_strains = trim_strains(strains, probeset_vals) write_data_matrix_file(trimmed_strains, probeset_vals, filename) + if __name__ == '__main__': main() diff --git a/wqflask/maintenance/geno_to_json.py b/wqflask/maintenance/geno_to_json.py index fa0dcebd..ad3f2b72 100644 --- a/wqflask/maintenance/geno_to_json.py +++ b/wqflask/maintenance/geno_to_json.py @@ -25,10 +25,10 @@ from pprint import pformat as pf #from utility.tools import flat_files + class EmptyConfigurations(Exception): pass - class Marker: def __init__(self): self.name = None @@ -37,6 +37,7 @@ class Marker: self.Mb = None self.genotypes = [] + class ConvertGenoFile: def __init__(self, input_file, output_file): @@ -78,7 +79,6 @@ class ConvertGenoFile: # elif self.file_type == "snps": # self.process_snps_file() - def process_csv(self): for row_count, row in enumerate(self.process_rows()): row_items = row.split("\t") @@ -121,7 +121,6 @@ class ConvertGenoFile: # self.output_fh.write("\n") - def process_rows(self): for self.latest_row_pos, row in enumerate(self.input_fh): # if self.input_file.endswith(".geno.gz"): @@ -182,7 +181,6 @@ class ConvertGenoFile: # convertob = ConvertGenoFile(input_file, output_file) - if __name__ == "__main__": Old_Geno_Directory = """/export/local/home/zas1024/gn2-zach/genotype_files/genotype""" New_Geno_Directory = """/export/local/home/zas1024/gn2-zach/genotype_files/genotype/json""" diff --git a/wqflask/maintenance/get_group_samplelists.py b/wqflask/maintenance/get_group_samplelists.py index 3f9d0278..0a450d3f 100644 --- a/wqflask/maintenance/get_group_samplelists.py +++ b/wqflask/maintenance/get_group_samplelists.py @@ -4,12 +4,14 @@ import gzip from base import webqtlConfig + def get_samplelist(file_type, geno_file): if file_type == "geno": return get_samplelist_from_geno(geno_file) elif file_type == "plink": return get_samplelist_from_plink(geno_file) + def get_samplelist_from_geno(genofilename): if os.path.isfile(genofilename + '.gz'): genofilename += '.gz' @@ -33,6 +35,7 @@ def get_samplelist_from_geno(genofilename): samplelist = headers[3:] return samplelist + def get_samplelist_from_plink(genofilename): genofile = open(genofilename) diff --git a/wqflask/maintenance/print_benchmark.py b/wqflask/maintenance/print_benchmark.py index a1046c86..9d12da8a 100644 --- a/wqflask/maintenance/print_benchmark.py +++ b/wqflask/maintenance/print_benchmark.py @@ -15,15 +15,18 @@ class TheCounter: self.time_took = time.time() - start_time TheCounter.Counters[self.__class__.__name__] = self.time_took + class PrintAll(TheCounter): def print_it(self, counter): print(counter) + class PrintSome(TheCounter): def print_it(self, counter): if counter % 1000 == 0: print(counter) + class PrintNone(TheCounter): def print_it(self, counter): pass @@ -37,5 +40,6 @@ def new_main(): print(pf(TheCounter.Counters)) + if __name__ == '__main__': new_main() diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py index 6751a8e5..1896bc52 100644 --- a/wqflask/maintenance/quantile_normalize.py +++ b/wqflask/maintenance/quantile_normalize.py @@ -14,6 +14,7 @@ from wqflask import app from utility.elasticsearch_tools import get_elasticsearch_connection from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT, SQL_URI + def parse_db_uri(): """Converts a database URI to the db name, host name, user name, and password""" @@ -28,6 +29,7 @@ def parse_db_uri(): print(db_conn_info) return db_conn_info + def create_dataframe(input_file): with open(input_file) as f: ncols = len(f.readline().split("\t")) @@ -36,6 +38,8 @@ def create_dataframe(input_file): return pd.DataFrame(input_array) # This function taken from https://github.com/ShawnLYU/Quantile_Normalize + + def quantileNormalize(df_input): df = df_input.copy() # compute rank @@ -50,6 +54,7 @@ def quantileNormalize(df_input): df[col] = [rank[i] for i in t] return df + def set_data(dataset_name): orig_file = "/home/zas1024/cfw_data/" + dataset_name + ".txt" @@ -95,6 +100,7 @@ def set_data(dataset_name): } } + if __name__ == '__main__': Conn = MySQLdb.Connect(**parse_db_uri()) Cursor = Conn.cursor() diff --git a/wqflask/maintenance/set_resource_defaults.py b/wqflask/maintenance/set_resource_defaults.py index 286094dd..c6c4f44c 100644 --- a/wqflask/maintenance/set_resource_defaults.py +++ b/wqflask/maintenance/set_resource_defaults.py @@ -37,6 +37,7 @@ import urllib.parse from utility.logger import getLogger logger = getLogger(__name__) + def parse_db_uri(): """Converts a database URI to the db name, host name, user name, and password""" @@ -51,6 +52,7 @@ def parse_db_uri(): print(db_conn_info) return db_conn_info + def insert_probeset_resources(default_owner_id): current_resources = Redis.hgetall("resources") Cursor.execute(""" SELECT @@ -77,6 +79,7 @@ def insert_probeset_resources(default_owner_id): add_resource(resource_ob, update=False) + def insert_publish_resources(default_owner_id): current_resources = Redis.hgetall("resources") Cursor.execute(""" SELECT @@ -110,6 +113,7 @@ def insert_publish_resources(default_owner_id): else: continue + def insert_geno_resources(default_owner_id): current_resources = Redis.hgetall("resources") Cursor.execute(""" SELECT @@ -139,6 +143,7 @@ def insert_geno_resources(default_owner_id): add_resource(resource_ob, update=False) + def insert_resources(default_owner_id): current_resources = get_resources() print("START") @@ -149,6 +154,7 @@ def insert_resources(default_owner_id): insert_probeset_resources(default_owner_id) print("AFTER PROBESET") + def main(): """Generates and outputs (as json file) the data for the main dropdown menus on the home page""" @@ -158,6 +164,7 @@ def main(): insert_resources(owner_id) + if __name__ == '__main__': Conn = MySQLdb.Connect(**parse_db_uri()) Cursor = Conn.cursor() diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py index 58108e03..03f310eb 100644 --- a/wqflask/run_gunicorn.py +++ b/wqflask/run_gunicorn.py @@ -14,9 +14,11 @@ from utility.startup_config import app_config app_config() + @app.route("/gunicorn") def hello(): return "<h1 style='color:blue'>Hello There!</h1>" + if __name__ == "__main__": app.run(host='0.0.0.0') diff --git a/wqflask/tests/unit/base/test_webqtl_case_data.py b/wqflask/tests/unit/base/test_webqtl_case_data.py index 8e8ba482..cebd41ce 100644 --- a/wqflask/tests/unit/base/test_webqtl_case_data.py +++ b/wqflask/tests/unit/base/test_webqtl_case_data.py @@ -4,6 +4,7 @@ import unittest from wqflask import app # Required because of utility.tools in webqtlCaseData.py from base.webqtlCaseData import webqtlCaseData + class TestWebqtlCaseData(unittest.TestCase): """Tests for WebqtlCaseData class""" diff --git a/wqflask/tests/unit/utility/test_authentication_tools.py b/wqflask/tests/unit/utility/test_authentication_tools.py index 42dcae88..024ab43f 100644 --- a/wqflask/tests/unit/utility/test_authentication_tools.py +++ b/wqflask/tests/unit/utility/test_authentication_tools.py @@ -5,6 +5,7 @@ from unittest import mock from utility.authentication_tools import check_resource_availability from utility.authentication_tools import add_new_resource + class TestResponse: """Mock Test Response after a request""" @property diff --git a/wqflask/tests/unit/utility/test_chunks.py b/wqflask/tests/unit/utility/test_chunks.py index 8d90a1ec..1d349193 100644 --- a/wqflask/tests/unit/utility/test_chunks.py +++ b/wqflask/tests/unit/utility/test_chunks.py @@ -7,6 +7,7 @@ from utility.chunks import divide_into_chunks class TestChunks(unittest.TestCase): "Test Utility method for chunking" + def test_divide_into_chunks(self): "Check that a list is chunked correctly" self.assertEqual(divide_into_chunks([1, 2, 7, 3, 22, 8, 5, 22, 333], 3), diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_display_mapping_results.py b/wqflask/tests/unit/wqflask/marker_regression/test_display_mapping_results.py index 8ae0f09f..219a6a29 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_display_mapping_results.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_display_mapping_results.py @@ -9,6 +9,7 @@ from wqflask.marker_regression.display_mapping_results import ( class TestDisplayMappingResults(unittest.TestCase): """Basic Methods to test Mapping Results""" + def test_pil_colors(self): """Test that colors use PILLOW color format""" self.assertEqual(DisplayMappingResults.CLICKABLE_WEBQTL_REGION_COLOR, @@ -17,6 +18,7 @@ class TestDisplayMappingResults(unittest.TestCase): class TestHtmlGenWrapper(unittest.TestCase): """Test Wrapper around HTMLGen""" + def test_create_image(self): """Test HT.Image method""" self.assertEqual( diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_plink_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_plink_mapping.py index 5eec93f1..fd21a825 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_plink_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_plink_mapping.py @@ -12,9 +12,10 @@ class AttributeSetter: def __init__(self, obj): for key, val in obj.items(): setattr(self, key, val) -class TestPlinkMapping(unittest.TestCase): +class TestPlinkMapping(unittest.TestCase): + def test_build_line_list(self): """test for building line list""" line_1 = "this is line one test" diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_qtlreaper_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_qtlreaper_mapping.py index bbb79f98..5cc8fd0f 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_qtlreaper_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_qtlreaper_mapping.py @@ -4,6 +4,8 @@ from wqflask.marker_regression.qtlreaper_mapping import gen_pheno_txt_file # issues some methods in genofile object are not defined # modify samples should equal to vals + + class TestQtlReaperMapping(unittest.TestCase): @mock.patch("wqflask.marker_regression.qtlreaper_mapping.TEMPDIR", "/home/user/data") def test_gen_pheno_txt_file(self): diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py index 0e617e93..6267ce9a 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py @@ -5,6 +5,7 @@ from wqflask.marker_regression.rqtl_mapping import get_trait_data_type from wqflask.marker_regression.rqtl_mapping import sanitize_rqtl_phenotype from wqflask.marker_regression.rqtl_mapping import sanitize_rqtl_names + class TestRqtlMapping(unittest.TestCase): def setUp(self): @@ -14,7 +15,6 @@ class TestRqtlMapping(unittest.TestCase): def tearDown(self): self.app_context.pop() - @mock.patch("wqflask.marker_regression.rqtl_mapping.g") @mock.patch("wqflask.marker_regression.rqtl_mapping.logger") def test_get_trait_data(self, mock_logger, mock_db): diff --git a/wqflask/tests/unit/wqflask/test_collect.py b/wqflask/tests/unit/wqflask/test_collect.py index 9a36132d..2a914fb2 100644 --- a/wqflask/tests/unit/wqflask/test_collect.py +++ b/wqflask/tests/unit/wqflask/test_collect.py @@ -11,6 +11,7 @@ app = Flask(__name__) class MockSession: """Helper class for mocking wqflask.collect.g.user_session.logged_in""" + def __init__(self, is_logged_in=False): self.is_logged_in = is_logged_in @@ -21,6 +22,7 @@ class MockSession: class MockFlaskG: """Helper class for mocking wqflask.collect.g.user_session""" + def __init__(self, is_logged_in=False): self.is_logged_in = is_logged_in diff --git a/wqflask/tests/wqflask/show_trait/test_show_trait.py b/wqflask/tests/wqflask/show_trait/test_show_trait.py index 8c866874..24c3923e 100644 --- a/wqflask/tests/wqflask/show_trait/test_show_trait.py +++ b/wqflask/tests/wqflask/show_trait/test_show_trait.py @@ -242,7 +242,6 @@ class TestTraits(unittest.TestCase): self.assertEqual(get_genotype_scales(file_location), expected_results) mock_get_scales.assert_called_once_with(file_location) - @mock.patch("wqflask.show_trait.show_trait.locate_ignore_error") def test_get_scales_from_genofile_found(self, mock_ignore_location): """"add test for get scales from genofile where file is found""" diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index 68c2cb72..d35b2089 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -47,6 +47,7 @@ COUR_FILE = "./wqflask/static/fonts/courbd.ttf" TAHOMA_FILE = "./wqflask/static/fonts/tahoma.ttf" # ---- END: FONT FILES ---- # + def cformat(d, rank=0): 'custom string format' strD = "%2.6f" % d @@ -68,6 +69,7 @@ def cformat(d, rank=0): strD = '0.0' return strD + def frange(start, end=None, inc=1.0): "A faster range-like function that does accept float increments..." if end == None: @@ -84,6 +86,7 @@ def frange(start, end=None, inc=1.0): L[i] = start + i * inc return L + def find_outliers(vals): """Calculates the upper and lower bounds of a set of sample/case values @@ -119,6 +122,8 @@ def find_outliers(vals): # parameter: data is either object returned by reaper permutation function (called by MarkerRegressionPage.py) # or the first object returned by direct (pair-scan) permu function (called by DirectPlotPage.py) + + def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLabel=None, YLabel=None, title=None, offset=(60, 20, 40, 40), zoom=1): im_drawer = ImageDraw.Draw(canvas) xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset @@ -230,6 +235,8 @@ def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLab font=labelFont, fill=labelColor) # This function determines the scale of the plot + + def detScaleOld(min, max): if min >= max: return None @@ -246,6 +253,7 @@ def detScaleOld(min, max): high = c * ceil(max / c) return [low, high, round((high - low) / c)] + def detScale(min=0, max=0): if min >= max: @@ -283,15 +291,19 @@ def detScale(min=0, max=0): return [low, high, n] + def bluefunc(x): return 1.0 / (1.0 + exp(-10 * (x - 0.6))) + def redfunc(x): return 1.0 / (1.0 + exp(10 * (x - 0.5))) + def greenfunc(x): return 1 - pow(redfunc(x + 0.2), 2) - bluefunc(x - 0.3) + def colorSpectrum(n=100): multiple = 10 if n == 1: @@ -319,6 +331,7 @@ def colorSpectrum(n=100): out2.append(out[-1]) return out2 + def _test(): import doctest doctest.testmod() diff --git a/wqflask/utility/__init__.py b/wqflask/utility/__init__.py index ec7e72d0..d540c96e 100644 --- a/wqflask/utility/__init__.py +++ b/wqflask/utility/__init__.py @@ -2,8 +2,10 @@ from pprint import pformat as pf # Todo: Move these out of __init__ + class Bunch: """Like a dictionary but using object notation""" + def __init__(self, **kw): self.__dict__ = kw diff --git a/wqflask/utility/after.py b/wqflask/utility/after.py index 06091ecb..2b560e48 100644 --- a/wqflask/utility/after.py +++ b/wqflask/utility/after.py @@ -7,6 +7,7 @@ from flask import g from wqflask import app + def after_this_request(f): if not hasattr(g, 'after_request_callbacks'): g.after_request_callbacks = [] diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py index 672b36d5..57dbf8ba 100644 --- a/wqflask/utility/authentication_tools.py +++ b/wqflask/utility/authentication_tools.py @@ -11,6 +11,7 @@ from utility.redis_tools import (get_redis_conn, add_resource) Redis = get_redis_conn() + def check_resource_availability(dataset, trait_id=None): # At least for now assume temporary entered traits are accessible if type(dataset) == str or dataset.type == "Temp": diff --git a/wqflask/utility/benchmark.py b/wqflask/utility/benchmark.py index 3d40a3b8..48ab1dc0 100644 --- a/wqflask/utility/benchmark.py +++ b/wqflask/utility/benchmark.py @@ -6,6 +6,7 @@ from utility.tools import LOG_BENCH from utility.logger import getLogger logger = getLogger(__name__) + class Bench: entries = collections.OrderedDict() diff --git a/wqflask/utility/corestats.py b/wqflask/utility/corestats.py index 15d1cb8d..523280a1 100644 --- a/wqflask/utility/corestats.py +++ b/wqflask/utility/corestats.py @@ -16,6 +16,8 @@ import sys # ZS: Should switch to using some third party library for this; maybe scipy has an equivalent + + class Stats: def __init__(self, sequence): diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index ae1181e4..9415cef0 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -47,11 +47,13 @@ logger = getLogger(__name__) from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + def test_elasticsearch_connection(): es = Elasticsearch(['http://' + ELASTICSEARCH_HOST + ":" + str(ELASTICSEARCH_PORT) + '/'], verify_certs=True) if not es.ping(): logger.warning("Elasticsearch is DOWN") + def get_elasticsearch_connection(for_user=True): """Return a connection to ES. Returns None on failure""" logger.info("get_elasticsearch_connection") @@ -77,6 +79,7 @@ def get_elasticsearch_connection(for_user=True): return es + def setup_users_index(es_connection): if es_connection: index_settings = { @@ -87,12 +90,15 @@ def setup_users_index(es_connection): es_connection.indices.create(index='users', ignore=400) es_connection.indices.put_mapping(body=index_settings, index="users", doc_type="local") + def get_user_by_unique_column(es, column_name, column_value, index="users", doc_type="local"): return get_item_by_unique_column(es, column_name, column_value, index=index, doc_type=doc_type) + def save_user(es, user, user_id): es_save_data(es, "users", "local", user, user_id) + def get_item_by_unique_column(es, column_name, column_value, index, doc_type): item_details = None try: @@ -106,6 +112,7 @@ def get_item_by_unique_column(es, column_name, column_value, index, doc_type): pass return item_details + def es_save_data(es, index, doc_type, data_item, data_id,): from time import sleep es.create(index, doc_type, body=data_item, id=data_id) diff --git a/wqflask/utility/external.py b/wqflask/utility/external.py index c1bf4043..805d2ffe 100644 --- a/wqflask/utility/external.py +++ b/wqflask/utility/external.py @@ -4,6 +4,7 @@ import os import sys import subprocess + def shell(command): if subprocess.call(command, shell=True) != 0: raise Exception("ERROR: failed on " + command) diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py index 9cfa39f9..24604e58 100644 --- a/wqflask/utility/gen_geno_ob.py +++ b/wqflask/utility/gen_geno_ob.py @@ -1,6 +1,7 @@ import utility.logger logger = utility.logger.getLogger(__name__) + class genotype: """ Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure @@ -119,6 +120,7 @@ class genotype: self.chromosomes.append(chr_ob) + class Chr: def __init__(self, name, geno_ob): self.name = name @@ -140,6 +142,7 @@ class Chr: def add_marker(self, marker_row): self.loci.append(Locus(self.geno_ob, marker_row)) + class Locus: def __init__(self, geno_ob, marker_row=None): self.chr = None diff --git a/wqflask/utility/genofile_parser.py b/wqflask/utility/genofile_parser.py index f8e96d19..94a08c17 100644 --- a/wqflask/utility/genofile_parser.py +++ b/wqflask/utility/genofile_parser.py @@ -12,6 +12,7 @@ import simplejson as json from pprint import pformat as pf + class Marker: def __init__(self): self.name = None diff --git a/wqflask/utility/logger.py b/wqflask/utility/logger.py index 16912e58..47079818 100644 --- a/wqflask/utility/logger.py +++ b/wqflask/utility/logger.py @@ -35,6 +35,7 @@ import datetime from utility.tools import LOG_LEVEL, LOG_LEVEL_DEBUG, LOG_SQL + class GNLogger: """A logger class with some additional functionality, such as multiple parameter logging, SQL logging, timing, colors, and lazy @@ -139,6 +140,8 @@ LOG_LEVEL_DEBUG (NYI). # Get the module logger. You can override log levels at the # module level + + def getLogger(name, level=None): gnlogger = GNLogger(name) logger = gnlogger.logger diff --git a/wqflask/utility/pillow_utils.py b/wqflask/utility/pillow_utils.py index 6e95beb0..5713e155 100644 --- a/wqflask/utility/pillow_utils.py +++ b/wqflask/utility/pillow_utils.py @@ -9,6 +9,8 @@ BLACK = ImageColor.getrgb("black") WHITE = ImageColor.getrgb("white") # def draw_rotated_text(canvas: Image, text: str, font: ImageFont, xy: tuple, fill: ImageColor=BLACK, angle: int=-90): + + def draw_rotated_text(canvas, text, font, xy, fill=BLACK, angle=-90): # type: (Image, str, ImageFont, tuple, ImageColor, int) """Utility function draw rotated text""" @@ -20,6 +22,8 @@ def draw_rotated_text(canvas, text, font, xy, fill=BLACK, angle=-90): canvas.paste(im=tmp_img2, box=tuple([int(i) for i in xy])) # def draw_open_polygon(canvas: Image, xy: tuple, fill: ImageColor=WHITE, outline: ImageColor=BLACK): + + def draw_open_polygon(canvas, xy, fill=None, outline=BLACK, width=0): # type: (Image, tuple, ImageColor, ImageColor) draw_ctx = ImageDraw.Draw(canvas) diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py index f22f4b14..92f944bc 100644 --- a/wqflask/utility/startup_config.py +++ b/wqflask/utility/startup_config.py @@ -10,6 +10,7 @@ GREEN = '\033[92m' BOLD = '\033[1m' ENDC = '\033[0m' + def app_config(): app.config['SESSION_TYPE'] = 'filesystem' if not app.config.get('SECRET_KEY'): diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index 4c478c36..f5ef81e1 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -459,11 +459,13 @@ class rect(SVGelement): if stroke_width != None: self.attributes['stroke-width'] = stroke_width + class ellipse(SVGelement): """e=ellipse(rx,ry,x,y,fill,stroke,stroke_width,**args) an ellipse is defined as a center and a x and y radius. """ + def __init__(self, cx=None, cy=None, rx=None, ry=None,fill=None,stroke=None,stroke_width=None,**args): if rx == None or ry == None: raise ValueError('both rx and ry are required') @@ -486,6 +488,7 @@ class circle(SVGelement): The circle creates an element using a x, y and radius values eg """ + def __init__(self, cx=None, cy=None, r=None, fill=None,stroke=None,stroke_width=None,**args): if r == None: raise ValueError('r is required') @@ -501,20 +504,24 @@ class circle(SVGelement): if stroke_width != None: self.attributes['stroke-width'] = stroke_width + class point(circle): """p=point(x,y,color) A point is defined as a circle with a size 1 radius. It may be more efficient to use a very small rectangle if you use many points because a circle is difficult to render. """ + def __init__(self, x, y, fill='black', **args): circle.__init__(self, x, y, 1, fill, **args) + class line(SVGelement): """l=line(x1,y1,x2,y2,stroke,stroke_width,**args) A line is defined by a begin x,y pair and an end x,y pair """ + def __init__(self, x1=None, y1=None, x2=None, y2=None,stroke=None,stroke_width=None,**args): SVGelement.__init__(self, 'line', **args) if x1 != None: @@ -530,11 +537,13 @@ class line(SVGelement): if stroke != None: self.attributes['stroke'] = stroke + class polyline(SVGelement): """pl=polyline([[x1,y1],[x2,y2],...],fill,stroke,stroke_width,**args) a polyline is defined by a list of xy pairs """ + def __init__(self, points, fill=None, stroke=None, stroke_width=None,**args): SVGelement.__init__(self, 'polyline', {'points': _xypointlist(points)}, **args) if fill != None: @@ -544,11 +553,13 @@ class polyline(SVGelement): if stroke != None: self.attributes['stroke'] = stroke + class polygon(SVGelement): """pl=polyline([[x1,y1],[x2,y2],...],fill,stroke,stroke_width,**args) a polygon is defined by a list of xy pairs """ + def __init__(self, points, fill=None, stroke=None, stroke_width=None,**args): SVGelement.__init__(self, 'polygon', {'points': _xypointlist(points)}, **args) if fill != None: @@ -558,11 +569,13 @@ class polygon(SVGelement): if stroke != None: self.attributes['stroke'] = stroke + class path(SVGelement): """p=path(path,fill,stroke,stroke_width,**args) a path is defined by a path object and optional width, stroke and fillcolor """ + def __init__(self, pathdata, fill=None, stroke=None, stroke_width=None,id=None,**args): SVGelement.__init__(self, 'path', {'d': str(pathdata)}, **args) if stroke != None: @@ -580,6 +593,7 @@ class text(SVGelement): a text element can bge used for displaying text on the screen """ + def __init__(self, x=None, y=None, text=None, font_size=None,font_family=None,text_anchor=None,**args): SVGelement.__init__(self, 'text', **args) if x != None: @@ -601,11 +615,13 @@ class textpath(SVGelement): a textpath places a text on a path which is referenced by a link. """ + def __init__(self, link, text=None, **args): SVGelement.__init__(self, 'textPath', {'xlink:href': link}, **args) if text != None: self.text = text + class pattern(SVGelement): """p=pattern(x,y,width,height,patternUnits,**args) @@ -613,6 +629,7 @@ class pattern(SVGelement): graphic object which can be replicated ("tiled") at fixed intervals in x and y to cover the areas to be painted. """ + def __init__(self, x=None, y=None, width=None, height=None,patternUnits=None,**args): SVGelement.__init__(self, 'pattern', **args) if x != None: @@ -626,34 +643,40 @@ class pattern(SVGelement): if patternUnits != None: self.attributes['patternUnits'] = patternUnits + class title(SVGelement): """t=title(text,**args) a title is a text element. The text is displayed in the title bar add at least one to the root svg element """ + def __init__(self, text=None, **args): SVGelement.__init__(self, 'title', **args) if text != None: self.text = text + class description(SVGelement): """d=description(text,**args) a description can be added to any element and is used for a tooltip Add this element before adding other elements. """ + def __init__(self, text=None, **args): SVGelement.__init__(self, 'desc', **args) if text != None: self.text = text + class lineargradient(SVGelement): """lg=lineargradient(x1,y1,x2,y2,id,**args) defines a lineargradient using two xy pairs. stop elements van be added to define the gradient colors. """ + def __init__(self, x1=None, y1=None, x2=None, y2=None,id=None,**args): SVGelement.__init__(self, 'linearGradient', **args) if x1 != None: @@ -667,12 +690,14 @@ class lineargradient(SVGelement): if id != None: self.attributes['id'] = id + class radialgradient(SVGelement): """rg=radialgradient(cx,cy,r,fx,fy,id,**args) defines a radial gradient using a outer circle which are defined by a cx,cy and r and by using a focalpoint. stop elements van be added to define the gradient colors. """ + def __init__(self, cx=None, cy=None, r=None, fx=None,fy=None,id=None,**args): SVGelement.__init__(self, 'radialGradient', **args) if cx != None: @@ -688,21 +713,25 @@ class radialgradient(SVGelement): if id != None: self.attributes['id'] = id + class stop(SVGelement): """st=stop(offset,stop_color,**args) Puts a stop color at the specified radius """ + def __init__(self, offset, stop_color=None, **args): SVGelement.__init__(self, 'stop', {'offset': offset}, **args) if stop_color != None: self.attributes['stop-color'] = stop_color + class style(SVGelement): """st=style(type,cdata=None,**args) Add a CDATA element to this element for defing in line stylesheets etc.. """ + def __init__(self, type, cdata=None, **args): SVGelement.__init__(self, 'style', {'type': type}, cdata=cdata, **args) @@ -712,6 +741,7 @@ class image(SVGelement): adds an image to the drawing. Supported formats are .png, .jpg and .svg. """ + def __init__(self, url, x=None, y=None, width=None,height=None,**args): if width == None or height == None: raise ValueError('both height and width are required') @@ -721,11 +751,13 @@ class image(SVGelement): if y != None: self.attributes['y'] = y + class cursor(SVGelement): """c=cursor(url,**args) defines a custom cursor for a element or a drawing """ + def __init__(self, url, **args): SVGelement.__init__(self, 'cursor', {'xlink:href': url}, **args) @@ -736,6 +768,7 @@ class marker(SVGelement): defines a marker which can be used as an endpoint for a line or other pathtypes add an element to it which should be used as a marker. """ + def __init__(self, id=None, viewBox=None, refx=None, refy=None,markerWidth=None,markerHeight=None,**args): SVGelement.__init__(self, 'marker', **args) if id != None: @@ -751,17 +784,20 @@ class marker(SVGelement): if markerHeight != None: self.attributes['markerHeight'] = markerHeight + class group(SVGelement): """g=group(id,**args) a group is defined by an id and is used to contain elements g.addElement(SVGelement) """ + def __init__(self, id=None, **args): SVGelement.__init__(self, 'g', **args) if id != None: self.attributes['id'] = id + class symbol(SVGelement): """sy=symbol(id,viewbox,**args) @@ -778,14 +814,17 @@ class symbol(SVGelement): if viewBox != None: self.attributes['viewBox'] = _viewboxlist(viewBox) + class defs(SVGelement): """d=defs(**args) container for defining elements """ + def __init__(self, **args): SVGelement.__init__(self, 'defs', **args) + class switch(SVGelement): """sw=switch(**args) @@ -793,6 +832,7 @@ class switch(SVGelement): requiredFeatures, requiredExtensions and systemLanguage. Refer to the SVG specification for details. """ + def __init__(self, **args): SVGelement.__init__(self, 'switch', **args) @@ -802,6 +842,7 @@ class use(SVGelement): references a symbol by linking to its id and its position, height and width """ + def __init__(self, link, x=None, y=None, width=None,height=None,**args): SVGelement.__init__(self, 'use', {'xlink:href': link}, **args) if x != None: @@ -821,32 +862,39 @@ class link(SVGelement): a link is defined by a hyperlink. add elements which have to be linked a.addElement(SVGelement) """ + def __init__(self, link='', **args): SVGelement.__init__(self, 'a', {'xlink:href': link}, **args) + class view(SVGelement): """v=view(id,**args) a view can be used to create a view with different attributes""" + def __init__(self, id=None, **args): SVGelement.__init__(self, 'view', **args) if id != None: self.attributes['id'] = id + class script(SVGelement): """sc=script(type,type,cdata,**args) adds a script element which contains CDATA to the SVG drawing """ + def __init__(self, type, cdata=None, **args): SVGelement.__init__(self, 'script', {'type': type}, cdata=cdata, **args) + class animate(SVGelement): """an=animate(attribute,from,to,during,**args) animates an attribute. """ + def __init__(self, attribute, fr=None, to=None, dur=None,**args): SVGelement.__init__(self, 'animate', {'attributeName': attribute}, **args) if fr != None: @@ -856,11 +904,13 @@ class animate(SVGelement): if dur != None: self.attributes['dur'] = dur + class animateMotion(SVGelement): """an=animateMotion(pathdata,dur,**args) animates a SVGelement over the given path in dur seconds """ + def __init__(self, pathdata, dur, **args): SVGelement.__init__(self, 'animateMotion', **args) if pathdata != None: @@ -868,11 +918,13 @@ class animateMotion(SVGelement): if dur != None: self.attributes['dur'] = dur + class animateTransform(SVGelement): """antr=animateTransform(type,from,to,dur,**args) transform an element from and to a value. """ + def __init__(self, type=None, fr=None, to=None, dur=None,**args): SVGelement.__init__(self, 'animateTransform', {'attributeName': 'transform'}, **args) # As far as I know the attributeName is always transform @@ -884,11 +936,14 @@ class animateTransform(SVGelement): self.attributes['to'] = to if dur != None: self.attributes['dur'] = dur + + class animateColor(SVGelement): """ac=animateColor(attribute,type,from,to,dur,**args) Animates the color of a element """ + def __init__(self, attribute, type=None, fr=None, to=None,dur=None,**args): SVGelement.__init__(self, 'animateColor', {'attributeName': attribute}, **args) if type != None: @@ -899,11 +954,14 @@ class animateColor(SVGelement): self.attributes['to'] = to if dur != None: self.attributes['dur'] = dur + + class set(SVGelement): """st=set(attribute,to,during,**args) sets an attribute to a value for a """ + def __init__(self, attribute, to=None, dur=None, **args): SVGelement.__init__(self, 'set', {'attributeName': attribute}, **args) if to != None: @@ -912,7 +970,6 @@ class set(SVGelement): self.attributes['dur'] = dur - class svg(SVGelement): """s=svg(viewbox,width,height,**args) @@ -928,6 +985,7 @@ class svg(SVGelement): d.setSVG(s) d.toXml() """ + def __init__(self, viewBox=None, width=None, height=None, **args): SVGelement.__init__(self, 'svg', **args) if viewBox != None: @@ -938,6 +996,7 @@ class svg(SVGelement): self.attributes['height'] = height self.namespace = "http://www.w3.org/2000/svg" + class drawing: """d=drawing() @@ -952,6 +1011,7 @@ class drawing: def __init__(self, entity={}): self.svg = None self.entity = entity + def setSVG(self, svg): self.svg = svg # Voeg een element toe aan de grafiek toe. @@ -1005,6 +1065,7 @@ class drawing: root = implementation.createDocument(None, None, doctype) # Create the xml document. global appender + def appender(element, elementroot): """This recursive function appends elements to an element and sets the attributes and type. It stops when alle elements have been appended""" @@ -1053,6 +1114,7 @@ class drawing: f.close() except: print(("Cannot write SVG file: " + filename)) + def validate(self): try: import xml.parsers.xmlproc.xmlval @@ -1066,9 +1128,10 @@ class drawing: raise Exception("SVG is not well formed, see messages above") else: print("SVG well formed") -if __name__ == '__main__': +if __name__ == '__main__': + d = drawing() s = svg((0, 0, 100, 100)) r = rect(-100, -100, 300, 300, 'cyan') diff --git a/wqflask/utility/temp_data.py b/wqflask/utility/temp_data.py index 2088ba9a..07c5a318 100644 --- a/wqflask/utility/temp_data.py +++ b/wqflask/utility/temp_data.py @@ -2,6 +2,7 @@ from redis import Redis import simplejson as json + class TempData: def __init__(self, temp_uuid): diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 9b751344..4fe4db08 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -13,11 +13,13 @@ logger = logging.getLogger(__name__) OVERRIDES = {} + def app_set(command_id, value): """Set application wide value""" app.config.setdefault(command_id, value) return value + def get_setting(command_id, guess=None): """Resolve a setting from the environment or the global settings in app.config, with valid_path is a function checking whether the @@ -66,12 +68,14 @@ def get_setting(command_id, guess=None): # print("Set "+command_id+"="+str(command)) return command + def get_setting_bool(id): v = get_setting(id) if v not in [0, False, 'False', 'FALSE', None]: return True return False + def get_setting_int(id): v = get_setting(id) if isinstance(v, str): @@ -80,21 +84,25 @@ def get_setting_int(id): return 0 return v + def valid_bin(bin): if os.path.islink(bin) or valid_file(bin): return bin return None + def valid_file(fn): if os.path.isfile(fn): return fn return None + def valid_path(dir): if os.path.isdir(dir): return dir return None + def js_path(module=None): """ Find the JS module in the two paths @@ -107,38 +115,47 @@ def js_path(module=None): return try_guix raise "No JS path found for " + module + " (if not in Guix check JS_GN_PATH)" + def reaper_command(guess=None): return get_setting("REAPER_COMMAND", guess) + def gemma_command(guess=None): return assert_bin(get_setting("GEMMA_COMMAND", guess)) + def gemma_wrapper_command(guess=None): return assert_bin(get_setting("GEMMA_WRAPPER_COMMAND", guess)) + def plink_command(guess=None): return assert_bin(get_setting("PLINK_COMMAND", guess)) + def flat_file_exists(subdir): base = get_setting("GENENETWORK_FILES") return valid_path(base + "/" + subdir) + def flat_files(subdir=None): base = get_setting("GENENETWORK_FILES") if subdir: return assert_dir(base + "/" + subdir) return assert_dir(base) + def assert_bin(fn): if not valid_bin(fn): raise Exception("ERROR: can not find binary " + fn) return fn + def assert_dir(dir): if not valid_path(dir): raise Exception("ERROR: can not find directory " + dir) return dir + def assert_writable_dir(dir): try: fn = dir + "/test.txt" @@ -150,16 +167,19 @@ def assert_writable_dir(dir): raise Exception('Unable to write test.txt to directory ' + dir) return dir + def assert_file(fn): if not valid_file(fn): raise Exception('Unable to find file ' + fn) return fn + def mk_dir(dir): if not valid_path(dir): os.makedirs(dir) return assert_dir(dir) + def locate(name, subdir=None): """ Locate a static flat file in the GENENETWORK_FILES environment. @@ -179,9 +199,11 @@ def locate(name, subdir=None): if subdir: sys.stderr.write(subdir) raise Exception("Can not locate " + name + " in " + base) + def locate_phewas(name, subdir=None): return locate(name, '/phewas/' + subdir) + def locate_ignore_error(name, subdir=None): """ Locate a static flat file in the GENENETWORK_FILES environment. @@ -200,17 +222,20 @@ def locate_ignore_error(name, subdir=None): logger.info("WARNING: file " + name + " not found\n") return None + def tempdir(): """ Get UNIX TMPDIR by default """ return valid_path(get_setting("TMPDIR", "/tmp")) + BLUE = '\033[94m' GREEN = '\033[92m' BOLD = '\033[1m' ENDC = '\033[0m' + def show_settings(): from utility.tools import LOG_LEVEL diff --git a/wqflask/utility/type_checking.py b/wqflask/utility/type_checking.py index 662bf794..00f14ba9 100644 --- a/wqflask/utility/type_checking.py +++ b/wqflask/utility/type_checking.py @@ -7,6 +7,7 @@ def is_float(value): except: return False + def is_int(value): try: int(value) @@ -14,6 +15,7 @@ def is_int(value): except: return False + def is_str(value): if value is None: return False @@ -23,18 +25,21 @@ def is_str(value): except: return False + def get_float(vars_obj, name, default=None): if name in vars_obj: if is_float(vars_obj[name]): return float(vars_obj[name]) return default + def get_int(vars_obj, name, default=None): if name in vars_obj: if is_int(vars_obj[name]): return float(vars_obj[name]) return default + def get_string(vars_obj, name, default=None): if name in vars_obj: if not vars_obj[name] is None: diff --git a/wqflask/utility/webqtlUtil.py b/wqflask/utility/webqtlUtil.py index ed59b0eb..f355a865 100644 --- a/wqflask/utility/webqtlUtil.py +++ b/wqflask/utility/webqtlUtil.py @@ -64,6 +64,7 @@ ParInfo = { # Accessory Functions ######################################### + def genRandStr(prefix="", length=8, chars=string.ascii_letters + string.digits): from random import choice _str = prefix[:] @@ -71,6 +72,7 @@ def genRandStr(prefix="", length=8, chars=string.ascii_letters + string.digits): _str += choice(chars) return _str + def ListNotNull(lst): '''Obsolete - Use built in function any (or all or whatever) @@ -83,6 +85,7 @@ def ListNotNull(lst): return 1 return None + def readLineCSV(line): # dcrowell July 2008 """Parses a CSV string of text and returns a list containing each element as a string. Used by correlationPage""" @@ -91,6 +94,7 @@ def readLineCSV(line): # dcrowell July 2008 returnList[0] = returnList[0][1:] return returnList + def cmpEigenValue(A, B): try: if A[0] > B[0]: @@ -102,6 +106,7 @@ def cmpEigenValue(A, B): except: return 0 + def hasAccessToConfidentialPhenotypeTrait(privilege, userName, authorized_users): access_to_confidential_phenotype_trait = 0 if webqtlConfig.USERDICT[privilege] > webqtlConfig.USERDICT['user']: diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index bab87115..712517a3 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -30,6 +30,7 @@ app.register_blueprint(policies_blueprint, url_prefix="/policies") app.register_blueprint(environments_blueprint, url_prefix="/environments") app.register_blueprint(facilities_blueprint, url_prefix="/facilities") + @app.before_request def before_request(): g.request_start_time = time.time() diff --git a/wqflask/wqflask/api/correlation.py b/wqflask/wqflask/api/correlation.py index e0b7fea0..f1dd148f 100644 --- a/wqflask/wqflask/api/correlation.py +++ b/wqflask/wqflask/api/correlation.py @@ -18,6 +18,7 @@ from utility.benchmark import Bench import utility.logger logger = utility.logger.getLogger(__name__) + def do_correlation(start_vars): assert('db' in start_vars) assert('target_db' in start_vars) @@ -66,6 +67,7 @@ def do_correlation(start_vars): return final_results + def calculate_results(this_trait, this_dataset, target_dataset, corr_params): corr_results = {} @@ -91,6 +93,7 @@ def calculate_results(this_trait, this_dataset, target_dataset, corr_params): return sorted_results + def do_tissue_correlation_for_all_traits(this_trait, trait_symbol_dict, corr_params, tissue_dataset_id=1): # Gets tissue expression values for the primary trait primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values(symbol_list=[this_trait.symbol]) @@ -113,6 +116,7 @@ def do_tissue_correlation_for_all_traits(this_trait, trait_symbol_dict, corr_par return tissue_corr_data + def do_literature_correlation_for_all_traits(this_trait, target_dataset, trait_geneid_dict, corr_params): input_trait_mouse_gene_id = convert_to_mouse_gene_id(target_dataset.group.species.lower(), this_trait.geneid) @@ -145,6 +149,7 @@ def do_literature_correlation_for_all_traits(this_trait, target_dataset, trait_g return lit_corr_data + def get_sample_r_and_p_values(this_trait, this_dataset, target_vals, target_dataset, type): """ Calculates the sample r (or rho) and p-value @@ -176,6 +181,7 @@ def get_sample_r_and_p_values(this_trait, this_dataset, target_vals, target_data else: return [sample_r, sample_p, num_overlap] + def convert_to_mouse_gene_id(species=None, gene_id=None): """If the species is rat or human, translate the gene_id to the mouse geneid @@ -212,6 +218,7 @@ def convert_to_mouse_gene_id(species=None, gene_id=None): return mouse_gene_id + def init_corr_params(start_vars): method = "pearson" if 'method' in start_vars: diff --git a/wqflask/wqflask/api/mapping.py b/wqflask/wqflask/api/mapping.py index fbfbc879..662090d5 100644 --- a/wqflask/wqflask/api/mapping.py +++ b/wqflask/wqflask/api/mapping.py @@ -10,6 +10,7 @@ from wqflask.marker_regression import gemma_mapping, rqtl_mapping, qtlreaper_map import utility.logger logger = utility.logger.getLogger(__name__) + def do_mapping_for_api(start_vars): assert('db' in start_vars) assert('trait_id' in start_vars) @@ -74,7 +75,6 @@ def do_mapping_for_api(start_vars): return result_markers, None - def initialize_parameters(start_vars, dataset, this_trait): mapping_params = {} diff --git a/wqflask/wqflask/api/router.py b/wqflask/wqflask/api/router.py index b0559a07..4f9cc6e5 100644 --- a/wqflask/wqflask/api/router.py +++ b/wqflask/wqflask/api/router.py @@ -27,10 +27,12 @@ logger = utility.logger.getLogger(__name__) version = "pre1" + @app.route("/api/v_{}/".format(version)) def hello_world(): return flask.jsonify({"hello": "world"}) + @app.route("/api/v_{}/species".format(version)) def get_species_list(): results = g.db.execute("SELECT SpeciesId, Name, FullName, TaxonomyId FROM Species;") @@ -47,6 +49,7 @@ def get_species_list(): return flask.jsonify(species_list) + @app.route("/api/v_{}/species/<path:species_name>".format(version)) @app.route("/api/v_{}/species/<path:species_name>.<path:file_format>".format(version)) def get_species_info(species_name, file_format="json"): @@ -64,6 +67,7 @@ def get_species_info(species_name, file_format="json"): return flask.jsonify(species_dict) + @app.route("/api/v_{}/groups".format(version)) @app.route("/api/v_{}/groups/<path:species_name>".format(version)) def get_groups_list(species_name=None): @@ -102,6 +106,7 @@ def get_groups_list(species_name=None): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/group/<path:group_name>".format(version)) @app.route("/api/v_{}/group/<path:group_name>.<path:file_format>".format(version)) @app.route("/api/v_{}/group/<path:species_name>/<path:group_name>".format(version)) @@ -145,6 +150,7 @@ def get_group_info(group_name, species_name=None, file_format="json"): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/datasets/<path:group_name>".format(version)) @app.route("/api/v_{}/datasets/<path:species_name>/<path:group_name>".format(version)) def get_datasets_for_group(group_name, species_name=None): @@ -197,6 +203,7 @@ def get_datasets_for_group(group_name, species_name=None): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/dataset/<path:dataset_name>".format(version)) @app.route("/api/v_{}/dataset/<path:dataset_name>.<path:file_format>".format(version)) @app.route("/api/v_{}/dataset/<path:group_name>/<path:dataset_name>".format(version)) @@ -302,6 +309,7 @@ def get_dataset_info(dataset_name, group_name=None, file_format="json"): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/traits/<path:dataset_name>".format(version), methods=("GET",)) @app.route("/api/v_{}/traits/<path:dataset_name>.<path:file_format>".format(version), methods=("GET",)) def fetch_traits(dataset_name, file_format="json"): @@ -430,6 +438,7 @@ def fetch_traits(dataset_name, file_format="json"): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/sample_data/<path:dataset_name>".format(version)) @app.route("/api/v_{}/sample_data/<path:dataset_name>.<path:file_format>".format(version)) def all_sample_data(dataset_name, file_format="csv"): @@ -536,6 +545,7 @@ def all_sample_data(dataset_name, file_format="csv"): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/sample_data/<path:dataset_name>/<path:trait_name>".format(version)) @app.route("/api/v_{}/sample_data/<path:dataset_name>/<path:trait_name>.<path:file_format>".format(version)) def trait_sample_data(dataset_name, trait_name, file_format="json"): @@ -625,6 +635,7 @@ def trait_sample_data(dataset_name, trait_name, file_format="json"): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/trait/<path:dataset_name>/<path:trait_name>".format(version)) @app.route("/api/v_{}/trait/<path:dataset_name>/<path:trait_name>.<path:file_format>".format(version)) @app.route("/api/v_{}/trait_info/<path:dataset_name>/<path:trait_name>".format(version)) @@ -694,6 +705,7 @@ def get_trait_info(dataset_name, trait_name, file_format="json"): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/correlation".format(version), methods=("GET",)) def get_corr_results(): results = correlation.do_correlation(request.args) @@ -703,6 +715,7 @@ def get_corr_results(): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/mapping".format(version), methods=("GET",)) def get_mapping_results(): results, format = mapping.do_mapping_for_api(request.args) @@ -726,6 +739,7 @@ def get_mapping_results(): else: return return_error(code=204, source=request.url_rule.rule, title="No Results", details="") + @app.route("/api/v_{}/genotypes/<string:file_format>/<string:group_name>/<string:dataset_name>.zip".format(version)) @app.route("/api/v_{}/genotypes/<string:file_format>/<string:group_name>/<string:dataset_name>".format(version)) @app.route("/api/v_{}/genotypes/<string:file_format>/<string:group_name>.zip".format(version)) @@ -813,6 +827,7 @@ def get_genotypes(group_name, file_format="csv", dataset_name=None): return output + @app.route("/api/v_{}/gen_dropdown".format(version), methods=("GET",)) def gen_dropdown_menu(): results = gen_menu.gen_dropdown_json() @@ -822,6 +837,7 @@ def gen_dropdown_menu(): else: return return_error(code=500, source=request.url_rule.rule, title="Some error occurred", details="") + def return_error(code, source, title, details): json_ob = {"errors": [ { @@ -834,6 +850,7 @@ def return_error(code, source, title, details): return flask.jsonify(json_ob) + def get_dataset_trait_ids(dataset_name, start_vars): if 'limit_to' in start_vars: @@ -906,6 +923,7 @@ def get_dataset_trait_ids(dataset_name, start_vars): dataset_id = results[0][2] return trait_ids, trait_names, data_type, dataset_id + def get_samplelist(dataset_name): group_id = get_group_id_from_dataset(dataset_name) @@ -922,6 +940,7 @@ def get_samplelist(dataset_name): return samplelist + def get_group_id_from_dataset(dataset_name): if "Publish" in dataset_name: query = """ @@ -962,6 +981,7 @@ def get_group_id_from_dataset(dataset_name): else: return None + def get_group_id(group_name): query = """ SELECT InbredSet.Id diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 6a1b88ca..61f73106 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -40,6 +40,7 @@ def process_traits(unprocessed_traits): return traits + def report_change(len_before, len_now): new_length = len_now - len_before if new_length: @@ -48,6 +49,7 @@ def report_change(len_before, len_now): else: logger.debug("No new traits were added.") + @app.route("/collections/store_trait_list", methods=('POST',)) def store_traits_list(): params = request.form @@ -59,6 +61,7 @@ def store_traits_list(): return hash + @app.route("/collections/add") def collections_add(): @@ -82,6 +85,7 @@ def collections_add(): collections=collections, ) + @app.route("/collections/new") def collections_new(): params = request.args @@ -118,6 +122,7 @@ def collections_new(): # CauseAnError pass + def create_new(collection_name): params = request.args @@ -133,6 +138,7 @@ def create_new(collection_name): return redirect(url_for('view_collection', uc_id=uc_id)) + @app.route("/collections/list") def list_collections(): params = request.args @@ -143,6 +149,7 @@ def list_collections(): collections=user_collections, ) + @app.route("/collections/remove", methods=('POST',)) def remove_traits(): params = request.form @@ -216,6 +223,7 @@ def view_collection(): **collection_info ) + @app.route("/collections/change_name", methods=('POST',)) def change_collection_name(): params = request.form diff --git a/wqflask/wqflask/correlation/corr_scatter_plot.py b/wqflask/wqflask/correlation/corr_scatter_plot.py index 22941ad5..6afea715 100644 --- a/wqflask/wqflask/correlation/corr_scatter_plot.py +++ b/wqflask/wqflask/correlation/corr_scatter_plot.py @@ -11,6 +11,7 @@ import numpy as np import utility.logger logger = utility.logger.getLogger(__name__) + class CorrScatterPlot: """Page that displays a correlation scatterplot with a line fitted to it""" diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 051ac1cb..e2fe1ff4 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -172,7 +172,6 @@ class CorrelationResults: self.correlation_data = collections.OrderedDict(sorted(list(self.correlation_data.items()), key=lambda t: -abs(t[1][0]))) - # ZS: Convert min/max chromosome to an int for the location range option range_chr_as_int = None for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): @@ -348,7 +347,6 @@ class CorrelationResults: else: trait.lit_corr = 0 - def do_lit_correlation_for_all_traits(self): input_trait_mouse_gene_id = self.convert_to_mouse_gene_id(self.dataset.group.species.lower(), self.this_trait.geneid) @@ -484,6 +482,7 @@ def do_bicor(this_trait_vals, target_trait_vals): return the_r, the_p + def generate_corr_json(corr_results, this_trait, dataset, target_dataset, for_api=False): results_list = [] for i, trait in enumerate(corr_results): @@ -574,6 +573,7 @@ def generate_corr_json(corr_results, this_trait, dataset, target_dataset, for_ap return json.dumps(results_list) + def get_header_fields(data_type, corr_method): if data_type == "ProbeSet": if corr_method == "spearman": diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 94c8931f..331cb1dc 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -233,6 +233,7 @@ class CorrelationMatrix: loadings_array.append(loadings_row) return loadings_array + def export_corr_matrix(corr_results): corr_matrix_filename = "corr_matrix_" + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) matrix_export_path = "{}{}.csv".format(GENERATED_TEXT_DIR, corr_matrix_filename) @@ -263,6 +264,7 @@ def export_corr_matrix(corr_results): return corr_matrix_filename, matrix_export_path + def zScore(trait_data_array): NN = len(trait_data_array[0]) if NN < 10: @@ -283,6 +285,7 @@ def zScore(trait_data_array): i += 1 return trait_data_array + def sortEigenVectors(vector): try: eigenValues = vector[0].tolist() diff --git a/wqflask/wqflask/ctl/ctl_analysis.py b/wqflask/wqflask/ctl/ctl_analysis.py index 2fa90a93..ec66e59f 100644 --- a/wqflask/wqflask/ctl/ctl_analysis.py +++ b/wqflask/wqflask/ctl/ctl_analysis.py @@ -39,6 +39,7 @@ r_write_table = ro.r["write.table"] # Map the write.table function r_data_frame = ro.r["data.frame"] # Map the write.table function r_as_numeric = ro.r["as.numeric"] # Map the write.table function + class CTL: def __init__(self): logger.info("Initialization of CTL") diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py index b6e85494..e743c4b3 100644 --- a/wqflask/wqflask/database.py +++ b/wqflask/wqflask/database.py @@ -17,6 +17,7 @@ db_session = scoped_session(sessionmaker(autocommit=False, Base = declarative_base() Base.query = db_session.query_property() + def init_db(): # import all modules here that might define models so that # they will be registered properly on the metadata. Otherwise @@ -27,4 +28,5 @@ def init_db(): Base.metadata.create_all(bind=engine) logger.info("Done creating all model metadata") + init_db() diff --git a/wqflask/wqflask/db_info.py b/wqflask/wqflask/db_info.py index 25e624ef..c7558ed8 100644 --- a/wqflask/wqflask/db_info.py +++ b/wqflask/wqflask/db_info.py @@ -90,6 +90,7 @@ class InfoPage: except Exception as e: pass + def process_query_results(results): info_ob = { 'info_page_name': results[0], diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 115182e4..7442dc72 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -73,6 +73,7 @@ class DoSearch: else: return None + class MrnaAssaySearch(DoSearch): """A search within an expression dataset, including mRNA, protein, SNP, but not phenotype or metabolites""" @@ -296,6 +297,7 @@ class PhenotypeSearch(DoSearch): return self.execute(query) + class GenotypeSearch(DoSearch): """A search within a genotype dataset""" @@ -339,7 +341,6 @@ class GenotypeSearch(DoSearch): from_clause = self.normalize_spaces(from_clause) - if self.search_term[0] == "*": query = (self.base_query + """WHERE Geno.Id = GenoXRef.GenoId @@ -367,6 +368,7 @@ class GenotypeSearch(DoSearch): return self.execute(self.query) + class RifSearch(MrnaAssaySearch): """Searches for traits with a Gene RIF entry including the search term.""" @@ -390,6 +392,7 @@ class RifSearch(MrnaAssaySearch): return self.execute(query) + class WikiSearch(MrnaAssaySearch): """Searches GeneWiki for traits other people have annotated""" @@ -415,6 +418,7 @@ class WikiSearch(MrnaAssaySearch): return self.execute(query) + class GoSearch(MrnaAssaySearch): """Searches for synapse-associated genes listed in the Gene Ontology.""" @@ -449,6 +453,8 @@ class GoSearch(MrnaAssaySearch): return self.execute(query) # ZS: Not sure what the best way to deal with LRS searches is + + class LrsSearch(DoSearch): """Searches for genes with a QTL within the given LRS values @@ -526,7 +532,6 @@ class LrsSearch(DoSearch): return where_clause - def run(self): self.from_clause = self.get_from_clause() @@ -550,6 +555,7 @@ class MrnaLrsSearch(LrsSearch, MrnaAssaySearch): return self.execute(self.query) + class PhenotypeLrsSearch(LrsSearch, PhenotypeSearch): for search_key in ('LRS', 'LOD'): @@ -649,6 +655,7 @@ class CisTransLrsSearch(DoSearch): return where_clause + class CisLrsSearch(CisTransLrsSearch, MrnaAssaySearch): """ Searches for genes on a particular chromosome with a cis-eQTL within the given LRS values @@ -680,6 +687,7 @@ class CisLrsSearch(CisTransLrsSearch, MrnaAssaySearch): return self.execute(self.query) + class TransLrsSearch(CisTransLrsSearch, MrnaAssaySearch): """Searches for genes on a particular chromosome with a cis-eQTL within the given LRS values @@ -744,6 +752,7 @@ class MeanSearch(MrnaAssaySearch): return self.execute(self.query) + class RangeSearch(MrnaAssaySearch): """Searches for genes with a range of expression varying between two values""" @@ -779,6 +788,7 @@ class RangeSearch(MrnaAssaySearch): return self.execute(self.query) + class PositionSearch(DoSearch): """Searches for genes/markers located within a specified range on a specified chromosome""" @@ -800,7 +810,6 @@ class PositionSearch(DoSearch): self.dataset.type, max(self.mb_min, self.mb_max)) - return where_clause def get_chr(self): @@ -819,6 +828,7 @@ class PositionSearch(DoSearch): return self.execute(self.query) + class MrnaPositionSearch(PositionSearch, MrnaAssaySearch): """Searches for genes located within a specified range on a specified chromosome""" @@ -832,6 +842,7 @@ class MrnaPositionSearch(PositionSearch, MrnaAssaySearch): return self.execute(self.query) + class GenotypePositionSearch(PositionSearch, GenotypeSearch): """Searches for genes located within a specified range on a specified chromosome""" @@ -845,6 +856,7 @@ class GenotypePositionSearch(PositionSearch, GenotypeSearch): return self.execute(self.query) + class PvalueSearch(MrnaAssaySearch): """Searches for traits with a permutationed p-value between low and high""" @@ -878,6 +890,7 @@ class PvalueSearch(MrnaAssaySearch): logger.sql(self.query) return self.execute(self.query) + class AuthorSearch(PhenotypeSearch): """Searches for phenotype traits with specified author(s)""" @@ -900,6 +913,7 @@ def is_number(s): except ValueError: return False + def get_aliases(symbol, species): if species == "mouse": symbol_string = symbol.capitalize() @@ -923,6 +937,7 @@ def get_aliases(symbol, species): return filtered_aliases + if __name__ == "__main__": # Usually this will be used as a library, but call it from the command line for testing # And it runs the code below diff --git a/wqflask/wqflask/docs.py b/wqflask/wqflask/docs.py index 207767c4..81424b9c 100644 --- a/wqflask/wqflask/docs.py +++ b/wqflask/wqflask/docs.py @@ -5,6 +5,7 @@ from flask import g from utility.logger import getLogger logger = getLogger(__name__) + class Docs: def __init__(self, entry, start_vars={}): @@ -23,7 +24,6 @@ class Docs: self.title = result[0] self.content = result[1].decode("utf-8") - self.editable = "false" # ZS: Removing option to edit to see if text still gets vandalized try: diff --git a/wqflask/wqflask/export_traits.py b/wqflask/wqflask/export_traits.py index 95c20673..d0745ef7 100644 --- a/wqflask/wqflask/export_traits.py +++ b/wqflask/wqflask/export_traits.py @@ -15,6 +15,7 @@ from pprint import pformat as pf from utility.logger import getLogger logger = getLogger(__name__) + def export_search_results_csv(targs): table_data = json.loads(targs['export_data']) @@ -132,6 +133,7 @@ def export_search_results_csv(targs): return file_list + def sort_traits_by_group(trait_list=[]): traits_by_group = {} for trait in trait_list: diff --git a/wqflask/wqflask/external_tools/send_to_bnw.py b/wqflask/wqflask/external_tools/send_to_bnw.py index 9836eb9c..1556c6a0 100644 --- a/wqflask/wqflask/external_tools/send_to_bnw.py +++ b/wqflask/wqflask/external_tools/send_to_bnw.py @@ -24,6 +24,7 @@ from utility import helper_functions, corr_result_helpers import utility.logger logger = utility.logger.getLogger(__name__) + class SendToBNW: def __init__(self, start_vars): trait_db_list = [trait.strip() for trait in start_vars['trait_list'].split(',')] diff --git a/wqflask/wqflask/external_tools/send_to_geneweaver.py b/wqflask/wqflask/external_tools/send_to_geneweaver.py index 36f1b8e5..c55c43e6 100644 --- a/wqflask/wqflask/external_tools/send_to_geneweaver.py +++ b/wqflask/wqflask/external_tools/send_to_geneweaver.py @@ -29,6 +29,7 @@ from utility import helper_functions, corr_result_helpers import utility.logger logger = utility.logger.getLogger(__name__) + class SendToGeneWeaver: def __init__(self, start_vars): trait_db_list = [trait.strip() for trait in start_vars['trait_list'].split(',')] @@ -58,6 +59,7 @@ class SendToGeneWeaver: 'list': ",".join(trait_name_list), } + def get_trait_name_list(trait_list): name_list = [] for trait_db in trait_list: @@ -65,6 +67,7 @@ def get_trait_name_list(trait_list): return name_list + def test_chip(trait_list): final_chip_name = "" diff --git a/wqflask/wqflask/external_tools/send_to_webgestalt.py b/wqflask/wqflask/external_tools/send_to_webgestalt.py index f50eeb8b..6b78725c 100644 --- a/wqflask/wqflask/external_tools/send_to_webgestalt.py +++ b/wqflask/wqflask/external_tools/send_to_webgestalt.py @@ -29,6 +29,7 @@ from utility import helper_functions, corr_result_helpers import utility.logger logger = utility.logger.getLogger(__name__) + class SendToWebGestalt: def __init__(self, start_vars): trait_db_list = [trait.strip() for trait in start_vars['trait_list'].split(',')] @@ -69,6 +70,7 @@ class SendToWebGestalt: else: self.hidden_vars['organism'] = "others" + def test_chip(trait_list): final_chip_name = "" @@ -113,6 +115,7 @@ def test_chip(trait_list): return chip_name + def gen_gene_id_list(trait_list): trait_name_list = [] gene_id_list = [] diff --git a/wqflask/wqflask/group_manager.py b/wqflask/wqflask/group_manager.py index f29c0e4d..92a65112 100644 --- a/wqflask/wqflask/group_manager.py +++ b/wqflask/wqflask/group_manager.py @@ -12,6 +12,7 @@ from utility.redis_tools import get_user_groups, get_group_info, save_user, crea from utility.logger import getLogger logger = getLogger(__name__) + @app.route("/groups/manage", methods=('GET', 'POST')) def manage_groups(): params = request.form if request.form else request.args @@ -21,6 +22,7 @@ def manage_groups(): admin_groups, member_groups = get_user_groups(g.user_session.user_id) return render_template("admin/group_manager.html", admin_groups=admin_groups, member_groups=member_groups) + @app.route("/groups/view", methods=('GET', 'POST')) def view_group(): params = request.form if request.form else request.args @@ -58,6 +60,7 @@ def view_group(): return render_template("admin/view_group.html", group_info=group_info, admins=admins_info, members=members_info, user_is_admin=user_is_admin, resources=resources_info) + @app.route("/groups/remove", methods=('POST',)) def remove_groups(): group_ids_to_remove = request.form['selected_group_ids'] @@ -66,6 +69,7 @@ def remove_groups(): return redirect(url_for('manage_groups')) + @app.route("/groups/remove_users", methods=('POST',)) def remove_users(): group_id = request.form['group_id'] @@ -77,6 +81,7 @@ def remove_users(): return redirect(url_for('view_group', id=group_id)) + @app.route("/groups/add_<path:user_type>", methods=('POST',)) def add_users(user_type='members'): group_id = request.form['group_id'] @@ -89,6 +94,7 @@ def add_users(user_type='members'): return redirect(url_for('view_group', id=group_id)) + @app.route("/groups/change_name", methods=('POST',)) def change_name(): group_id = request.form['group_id'] @@ -97,6 +103,7 @@ def change_name(): return new_name + @app.route("/groups/create", methods=('GET', 'POST')) def add_or_edit_group(): params = request.form if request.form else request.args @@ -125,6 +132,8 @@ def add_or_edit_group(): return render_template("admin/create_group.html") # ZS: Will integrate this later, for now just letting users be added directly + + def send_group_invites(group_id, user_email_list=[], user_type="members"): for user_email in user_email_list: user_details = get_user_by_unique_column("email_address", user_email) diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index 8cb81dcc..a21dae84 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -18,6 +18,7 @@ from utility.type_checking import is_float, is_int, is_str, get_float, get_int, from utility.logger import getLogger logger = getLogger(__name__) + class GSearch: def __init__(self, kw): diff --git a/wqflask/wqflask/heatmap/heatmap.py b/wqflask/wqflask/heatmap/heatmap.py index f8ef7028..0b477446 100644 --- a/wqflask/wqflask/heatmap/heatmap.py +++ b/wqflask/wqflask/heatmap/heatmap.py @@ -14,6 +14,7 @@ Redis = Redis() logger = getLogger(__name__) + class Heatmap: def __init__(self, start_vars, temp_uuid): @@ -132,6 +133,7 @@ class Heatmap: else: self.trait_results[this_trait.name].append(float(qtl['lrs_value'])) + def gen_pheno_txt_file(samples, vals, filename): """Generates phenotype file for GEMMA""" @@ -151,6 +153,7 @@ def gen_pheno_txt_file(samples, vals, filename): values_string = "\t".join(filtered_vals_list) outfile.write(values_string) + def parse_reaper_output(gwa_filename): included_markers = [] p_values = [] diff --git a/wqflask/wqflask/interval_analyst/GeneUtil.py b/wqflask/wqflask/interval_analyst/GeneUtil.py index 9779878e..8dd1c7c0 100644 --- a/wqflask/wqflask/interval_analyst/GeneUtil.py +++ b/wqflask/wqflask/interval_analyst/GeneUtil.py @@ -4,6 +4,8 @@ from flask import Flask, g # Just return a list of dictionaries # each dictionary contains sub-dictionary + + def loadGenes(chrName, diffCol, startMb, endMb, species='mouse'): fetchFields = ['SpeciesId', 'Id', 'GeneSymbol', 'GeneDescription', 'Chromosome', 'TxStart', 'TxEnd', 'Strand', 'GeneID', 'NM_ID', 'kgID', 'GenBankID', 'UnigenID', 'ProteinID', 'AlignID', diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 3753d1ce..5c7b81dd 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -509,9 +509,6 @@ class DisplayMappingResults: self.graphHeight = self.graphHeight + 2 * (self.NR_INDIVIDUALS + 10) * self.EACH_GENE_HEIGHT # END HaplotypeAnalyst - - - ######################### # Get the sorting column ######################### @@ -1640,7 +1637,6 @@ class DisplayMappingResults: geneYLocation + 2 *ind*self.EACH_GENE_HEIGHT + 2*self.EACH_GENE_HEIGHT*zoom)), outline=outlineColor, fill=fillColor) - COORDS = "%d, %d, %d, %d" % (geneStartPix, geneYLocation + ind * self.EACH_GENE_HEIGHT, geneEndPix + 1, (geneYLocation + ind * self.EACH_GENE_HEIGHT)) TITLE = "Strain: %s, marker (%s) \n Position %2.3f Mb." % (samplelist[k], _chr[j].name, float(txStart)) HREF = '' @@ -1663,7 +1659,6 @@ class DisplayMappingResults: geneYLocation + 7 + 2*ind*self.EACH_GENE_HEIGHT*zoom)), fill= mylineColor, width=zoom * (self.EACH_GENE_HEIGHT + 2)) - if lastGene == 0: draw_rotated_text( canvas, text="%s" % (_chr[j].name), @@ -2085,7 +2080,6 @@ class DisplayMappingResults: im_drawer.line(xy=((xLeftOffset, yZero), (xLeftOffset + plotWidth, yZero)), fill=BLACK, width=X_AXIS_THICKNESS) # Draw the X axis itself - def drawQTL(self, canvas, drawAreaHeight, gifmap, plotXScale, offset=(40, 120, 80, 10), zoom=1, startMb= None, endMb = None): im_drawer = ImageDraw.Draw(canvas) xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset @@ -2427,7 +2421,6 @@ class DisplayMappingResults: yLRS = yZero - (item / LRS_LOD_Max) * LRSHeightThresh - if 'lrs_value' in qtlresult: if self.LRS_LOD == "LOD" or self.LRS_LOD == "-logP": if qtlresult['lrs_value'] > 460 or qtlresult['lrs_value'] == 'inf': @@ -2621,7 +2614,6 @@ class DisplayMappingResults: # , clipX=(xLeftOffset, xLeftOffset + plotWidth) ) - # draw additive scale if not self.multipleInterval and self.additiveChecked: additiveScaleFont = ImageFont.truetype(font=VERDANA_FILE, size=16 * zoom) @@ -2653,7 +2645,6 @@ class DisplayMappingResults: xy=((xLeftOffset, yZero), (xLeftOffset, yTopOffset + 30 * (zoom - 1))), fill=self.LRS_COLOR, width=1 * zoom) # the blue line running up the y axis - def drawGraphBackground(self, canvas, gifmap, offset=(80, 120, 80, 50), zoom=1, startMb= None, endMb = None): # conditions # multiple Chromosome view diff --git a/wqflask/wqflask/marker_regression/plink_mapping.py b/wqflask/wqflask/marker_regression/plink_mapping.py index e6c78536..22a50bb8 100644 --- a/wqflask/wqflask/marker_regression/plink_mapping.py +++ b/wqflask/wqflask/marker_regression/plink_mapping.py @@ -8,11 +8,11 @@ from utility.tools import flat_files, PLINK_COMMAND import utility.logger logger = utility.logger.getLogger(__name__) + def run_plink(this_trait, dataset, species, vals, maf): plink_output_filename = webqtlUtil.genRandStr(f"{dataset.group.name}_{this_trait.name}_") gen_pheno_txt_file(dataset, vals) - plink_command = f"{PLINK_COMMAND} --noweb --bfile {flat_files('mapping')}/{dataset.group.name} --no-pheno --no-fid --no-parents --no-sex --maf {maf} --out { TMPDIR}{plink_output_filename} --assoc " logger.debug("plink_command:", plink_command) @@ -25,6 +25,7 @@ def run_plink(this_trait, dataset, species, vals, maf): return dataset.group.markers.markers + def gen_pheno_txt_file(this_dataset, vals): """Generates phenotype file for GEMMA/PLINK""" @@ -42,6 +43,7 @@ def gen_pheno_txt_file(this_dataset, vals): this_val = vals[i] outfile.write("0 " + line[1] + " " + line[2] + " " + line[3] + " " + line[4] + " " + str(this_val) + "\n") + def gen_pheno_txt_file_plink(this_trait, dataset, vals, pheno_filename=''): ped_sample_list = get_samples_from_ped_file(dataset) output_file = open(f"{TMPDIR}{pheno_filename}.txt", "wb") @@ -77,6 +79,8 @@ def gen_pheno_txt_file_plink(this_trait, dataset, vals, pheno_filename=''): output_file.close() # get strain name from ped file in order + + def get_samples_from_ped_file(dataset): ped_file = open(f"{flat_files('mapping')}{dataset.group.name}.ped", "r") line = ped_file.readline() @@ -93,6 +97,7 @@ def get_samples_from_ped_file(dataset): return sample_list + def parse_plink_output(output_filename, species): plink_results = {} @@ -154,6 +159,8 @@ def parse_plink_output(output_filename, species): # function: convert line from str to list; # output: lineList list ####################################################### + + def build_line_list(line=""): line_list = line.strip().split(' ') # irregular number of whitespaces between columns line_list = [item for item in line_list if item != ''] diff --git a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py index dd044cb0..313c40ca 100644 --- a/wqflask/wqflask/marker_regression/qtlreaper_mapping.py +++ b/wqflask/wqflask/marker_regression/qtlreaper_mapping.py @@ -8,6 +8,7 @@ from utility.tools import flat_files, REAPER_COMMAND, TEMPDIR import utility.logger logger = utility.logger.getLogger(__name__) + def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boot_check, num_bootstrap, do_control, control_marker, manhattan_plot, first_run=True, output_files=None): """Generates p-values for each marker using qtlreaper""" @@ -73,6 +74,7 @@ def run_reaper(this_trait, this_dataset, samples, vals, json_data, num_perm, boo return (marker_obs, permu_vals, suggestive, significant, bootstrap_vals, [output_filename, permu_filename, bootstrap_filename]) + def gen_pheno_txt_file(samples, vals, trait_filename): """Generates phenotype file for GEMMA""" @@ -92,6 +94,7 @@ def gen_pheno_txt_file(samples, vals, trait_filename): values_string = "\t".join(filtered_vals_list) outfile.write(values_string) + def parse_reaper_output(gwa_filename, permu_filename, bootstrap_filename): included_markers = [] p_values = [] @@ -163,6 +166,7 @@ def parse_reaper_output(gwa_filename, permu_filename, bootstrap_filename): return marker_obs, permu_vals, bootstrap_vals + def run_original_reaper(this_trait, dataset, samples_before, trait_vals, json_data, num_perm, bootCheck, num_bootstrap, do_control, control_marker, manhattan_plot): genotype = dataset.group.read_genotype_file(use_reaper=True) @@ -255,6 +259,7 @@ def run_original_reaper(this_trait, dataset, samples_before, trait_vals, json_da qtl_results.append(qtl) return qtl_results, json_data, perm_output, suggestive, significant, bootstrap_results + def natural_sort(marker_list): """ Function to naturally sort numbers + strings, adopted from user Mark Byers here: https://stackoverflow.com/questions/4836710/does-python-have-a-built-in-function-for-string-natural-sort diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 2bd94512..588600f5 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -16,6 +16,8 @@ import utility.logger logger = utility.logger.getLogger(__name__) # Get a trait's type (numeric, categorical, etc) from the DB + + def get_trait_data_type(trait_db_string): logger.info("get_trait_data_type"); the_query = "SELECT value FROM TraitMetadata WHERE type='trait_data_type'" @@ -133,6 +135,7 @@ def run_rqtl_geno(vals, samples, dataset, mapping_scale, method, model, permChec else: return process_rqtl_results(result_data_frame, dataset.group.species) + def generate_cross_from_rdata(dataset): rdata_location = locate(dataset.group.name + ".RData", "genotype/rdata") ro.r(""" @@ -143,6 +146,7 @@ def generate_cross_from_rdata(dataset): } """ % (rdata_location)) + def generate_cross_from_geno(dataset, scale_units): # TODO: Need to figure out why some genofiles have the wrong format and don't convert properly ro.r(""" @@ -187,6 +191,7 @@ def generate_cross_from_geno(dataset, scale_units): # TODO: Need to figur } """ % (dataset.group.genofile, scale_units)) + def add_perm_strata(cross, perm_strata): col_string = 'c("the_strata")' perm_strata_string = "c(" @@ -201,6 +206,7 @@ def add_perm_strata(cross, perm_strata): return cross, strata_ob + def sanitize_rqtl_phenotype(vals): pheno_as_string = "c(" for i, val in enumerate(vals): @@ -218,6 +224,7 @@ def sanitize_rqtl_phenotype(vals): return pheno_as_string + def sanitize_rqtl_names(vals): pheno_as_string = "c(" for i, val in enumerate(vals): @@ -235,12 +242,14 @@ def sanitize_rqtl_names(vals): return pheno_as_string + def add_phenotype(cross, pheno_as_string, col_name): ro.globalenv["the_cross"] = cross ro.r('pheno <- data.frame(pull.pheno(the_cross))') ro.r('the_cross$pheno <- cbind(pheno, ' + col_name + ' = as.numeric(' + pheno_as_string + '))') return ro.r["the_cross"] + def add_categorical_covar(cross, covar_as_string, i): ro.globalenv["the_cross"] = cross logger.info("cross set"); @@ -275,12 +284,14 @@ def add_names(cross, names_as_string, col_name): ro.r('the_cross$pheno <- cbind(pheno, ' + col_name + ' = ' + names_as_string + ')') return ro.r["the_cross"] + def pull_var(var_name, cross, var_string): ro.globalenv["the_cross"] = cross ro.r(var_name + ' <- pull.pheno(the_cross, ' + var_string + ')') return ro.r[var_name] + def add_cofactors(cross, this_dataset, covariates, samples): ro.numpy2ri.activate() @@ -341,6 +352,7 @@ def add_cofactors(cross, this_dataset, covariates, samples): covars_ob = pull_var("trait_covars", cross, covar_name_string) return cross, covars_ob + def create_marker_covariates(control_marker, cross): ro.globalenv["the_cross"] = cross ro.r('genotypes <- pull.geno(the_cross)') # Get the genotype matrix @@ -358,6 +370,7 @@ def create_marker_covariates(control_marker, cross): # TODO: Create a design matrix from the marker covars for the markers in case of an F2, 4way, etc return ro.r["marker_covars"] + def process_pair_scan_results(result): pair_scan_results = [] @@ -374,6 +387,7 @@ def process_pair_scan_results(result): return pair_scan_results + def process_rqtl_perm_results(num_perm, results): perm_vals = [] for line in str(results).split("\n")[1:(num_perm + 1)]: @@ -386,6 +400,7 @@ def process_rqtl_perm_results(num_perm, results): return perm_output, suggestive, significant + def process_rqtl_results(result, species_name): # TODO: how to make this a one liner and not copy the stuff in a loop qtl_results = [] output = [tuple([result[j][i] for j in range(result.ncol)]) for i in range(result.nrow)] diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index f1665570..31c58083 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -45,6 +45,7 @@ from base.webqtlConfig import TMPDIR, GENERATED_TEXT_DIR import utility.logger logger = utility.logger.getLogger(__name__) + class RunMapping: def __init__(self, start_vars, temp_uuid): @@ -504,6 +505,7 @@ class RunMapping: trimmed_genotype_data.append(new_genotypes) return trimmed_genotype_data + def export_mapping_results(dataset, trait, markers, results_path, mapping_scale, score_type, transform, covariates, n_samples): with open(results_path, "w+") as output_file: output_file.write("Time/Date: " + datetime.datetime.now().strftime("%x / %X") + "\n") @@ -564,6 +566,7 @@ def export_mapping_results(dataset, trait, markers, results_path, mapping_scale, if i < (len(markers) - 1): output_file.write("\n") + def trim_markers_for_figure(markers): if 'p_wald' in list(markers[0].keys()): score_type = 'p_wald' @@ -624,6 +627,7 @@ def trim_markers_for_figure(markers): filtered_markers.append(marker) return filtered_markers + def trim_markers_for_table(markers): if 'lod_score' in list(markers[0].keys()): sorted_markers = sorted(markers, key=lambda k: k['lod_score'], reverse=True) @@ -637,6 +641,7 @@ def trim_markers_for_table(markers): else: return sorted_markers + def write_input_for_browser(this_dataset, gwas_results, annotations): file_base = this_dataset.group.name + "_" + ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) gwas_filename = file_base + "_GWAS" @@ -650,6 +655,7 @@ def write_input_for_browser(this_dataset, gwas_results, annotations): return [gwas_filename, annot_filename] + def geno_db_exists(this_dataset): geno_db_name = this_dataset.group.name + "Geno" try: @@ -658,6 +664,7 @@ def geno_db_exists(this_dataset): except: return "False" + def get_chr_lengths(mapping_scale, mapping_method, dataset, qtl_results): chr_lengths = [] if mapping_scale == "physic": @@ -696,6 +703,7 @@ def get_chr_lengths(mapping_scale, mapping_method, dataset, qtl_results): return chr_lengths + def get_genofile_samplelist(dataset): genofile_samplelist = [] @@ -706,6 +714,7 @@ def get_genofile_samplelist(dataset): return genofile_samplelist + def get_perm_strata(this_trait, sample_list, categorical_vars, used_samples): perm_strata_strings = [] for sample in used_samples: diff --git a/wqflask/wqflask/model.py b/wqflask/wqflask/model.py index 8abd6516..d7c9ef95 100644 --- a/wqflask/wqflask/model.py +++ b/wqflask/wqflask/model.py @@ -14,6 +14,7 @@ from sqlalchemy.orm import relationship from wqflask.database import Base, init_db + class User(Base): __tablename__ = "user" id = Column(Unicode(36), primary_key=True, default=lambda: str(uuid.uuid4())) @@ -63,7 +64,6 @@ class User(Base): print("Couldn't display_num_collections:", why) return "" - def get_collection_by_name(self, collection_name): try: collect = self.user_collections.filter_by(name=collection_name).first() @@ -83,7 +83,6 @@ class User(Base): def login_count(self): return self.logins.filter_by(successful=True).count() - @property def confirmed_at(self): if self.confirmed: @@ -116,6 +115,7 @@ class User(Base): except IndexError: return None + class Login(Base): __tablename__ = "login" id = Column(Unicode(36), primary_key=True, default=lambda: str(uuid.uuid4())) @@ -134,6 +134,7 @@ class Login(Base): ################################################################################################## + class UserCollection(Base): __tablename__ = "user_collection" id = Column(Unicode(36), primary_key=True, default=lambda: str(uuid.uuid4())) @@ -158,12 +159,14 @@ class UserCollection(Base): def members_as_set(self): return set(json.loads(self.members)) + def display_collapsible(number): if number: return number else: return "" + def user_uuid(): """Unique cookie for a user""" user_uuid = request.cookies.get('user_uuid') diff --git a/wqflask/wqflask/network_graph/network_graph.py b/wqflask/wqflask/network_graph/network_graph.py index 93785a3a..e089643e 100644 --- a/wqflask/wqflask/network_graph/network_graph.py +++ b/wqflask/wqflask/network_graph/network_graph.py @@ -27,6 +27,7 @@ from utility import helper_functions from utility import corr_result_helpers from utility.tools import GN2_BRANCH_URL + class NetworkGraph: def __init__(self, start_vars): diff --git a/wqflask/wqflask/news.py b/wqflask/wqflask/news.py index 861a93f2..e262dd51 100644 --- a/wqflask/wqflask/news.py +++ b/wqflask/wqflask/news.py @@ -1,5 +1,6 @@ from flask import g + class News: def __init__(self): diff --git a/wqflask/wqflask/parser.py b/wqflask/wqflask/parser.py index dfd374e2..6b836e20 100644 --- a/wqflask/wqflask/parser.py +++ b/wqflask/wqflask/parser.py @@ -24,6 +24,7 @@ from pprint import pformat as pf from utility.logger import getLogger logger = getLogger(__name__) + def parse(pstring): """ diff --git a/wqflask/wqflask/pbkdf2.py b/wqflask/wqflask/pbkdf2.py index 6346df03..1a965fc5 100644 --- a/wqflask/wqflask/pbkdf2.py +++ b/wqflask/wqflask/pbkdf2.py @@ -4,6 +4,8 @@ from werkzeug.security import safe_str_cmp as ssc # Replace this because it just wraps around Python3's internal # functions. Added this during migration. + + def pbkdf2_hex(data, salt, iterations=1000, keylen=24, hashfunc="sha1"): """Wrapper function of python's hashlib.pbkdf2_hmac. """ diff --git a/wqflask/wqflask/resource_manager.py b/wqflask/wqflask/resource_manager.py index a1fe0f8f..4591710c 100644 --- a/wqflask/wqflask/resource_manager.py +++ b/wqflask/wqflask/resource_manager.py @@ -11,6 +11,7 @@ from utility.redis_tools import get_resource_info, get_group_info, get_groups_li from utility.logger import getLogger logger = getLogger(__name__) + @app.route("/resources/manage", methods=('GET', 'POST')) def manage_resource(): params = request.form if request.form else request.args @@ -40,6 +41,7 @@ def manage_resource(): return render_template("admin/manage_resource.html", owner_name=owner_display_name, resource_id=resource_id, resource_info=resource_info, default_mask=default_mask, group_masks=group_masks_with_names, admin_status=admin_status) + @app.route("/search_for_users", methods=('POST',)) def search_for_user(): params = request.form @@ -49,6 +51,7 @@ def search_for_user(): return json.dumps(user_list) + @app.route("/search_for_groups", methods=('POST',)) def search_for_groups(): params = request.form @@ -65,6 +68,7 @@ def search_for_groups(): return json.dumps(group_list) + @app.route("/resources/change_owner", methods=('POST',)) def change_owner(): resource_id = request.form['resource_id'] @@ -81,6 +85,7 @@ def change_owner(): else: return render_template("admin/change_resource_owner.html", resource_id=resource_id) + @app.route("/resources/change_default_privileges", methods=('POST',)) def change_default_privileges(): resource_id = request.form['resource_id'] @@ -99,6 +104,7 @@ def change_default_privileges(): else: return redirect(url_for("no_access_page")) + @app.route("/resources/add_group", methods=('POST',)) def add_group_to_resource(): resource_id = request.form['resource_id'] @@ -125,6 +131,7 @@ def add_group_to_resource(): else: return redirect(url_for("no_access_page")) + def get_group_names(group_masks): group_masks_with_names = {} for group_id, group_mask in list(group_masks.items()): diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 16eb1864..ed5f9bad 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -24,6 +24,7 @@ from utility.type_checking import is_str from utility.logger import getLogger logger = getLogger(__name__) + class SearchResultPage: #maxReturn = 3000 @@ -270,6 +271,7 @@ class SearchResultPage: else: return None + def get_GO_symbols(a_search): query = """SELECT genes FROM GORef @@ -287,6 +289,7 @@ def get_GO_symbols(a_search): return new_terms + def insert_newlines(string, every=64): """ This is because it is seemingly impossible to change the width of the description column, so I'm just manually adding line breaks """ lines = [] @@ -294,6 +297,7 @@ def insert_newlines(string, every=64): lines.append(string[i:i + every]) return '\n'.join(lines) + def get_aliases(symbol_list, species): updated_symbols = [] diff --git a/wqflask/wqflask/send_mail.py b/wqflask/wqflask/send_mail.py index 86e8a558..299c866a 100644 --- a/wqflask/wqflask/send_mail.py +++ b/wqflask/wqflask/send_mail.py @@ -8,10 +8,12 @@ Redis = StrictRedis() import mailer + def timestamp(): ts = datetime.datetime.utcnow() return ts.isoformat() + def main(): while True: print("I'm alive!") @@ -31,7 +33,6 @@ def main(): process_message(msg) - def process_message(msg): msg = json.loads(msg) diff --git a/wqflask/wqflask/server_side.py b/wqflask/wqflask/server_side.py index 48761fa0..8b3a4faa 100644 --- a/wqflask/wqflask/server_side.py +++ b/wqflask/wqflask/server_side.py @@ -1,7 +1,6 @@ # handles server side table processing - class ServerSideTable: """ This class is used to do server-side processing diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index 388f831f..496dee57 100644 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -8,6 +8,7 @@ from pprint import pformat as pf from utility import Plot from utility import Bunch + class SampleList: def __init__(self, dataset, diff --git a/wqflask/wqflask/show_trait/export_trait_data.py b/wqflask/wqflask/show_trait/export_trait_data.py index 379b746c..f0fcd27d 100644 --- a/wqflask/wqflask/show_trait/export_trait_data.py +++ b/wqflask/wqflask/show_trait/export_trait_data.py @@ -5,6 +5,7 @@ from functools import cmp_to_key from base.trait import create_trait from base import data_set + def export_sample_table(targs): sample_data = json.loads(targs['export_data']) @@ -28,6 +29,7 @@ def export_sample_table(targs): return trait_name, final_sample_data + def get_export_metadata(trait_id, dataset_name): dataset = data_set.create_dataset(dataset_name) this_trait = create_trait(dataset=dataset, @@ -64,6 +66,7 @@ def dict_to_sorted_list(dictionary): sorted_values = [item[1] for item in sorted_list] return sorted_values + def cmp_samples(a, b): if b[0] == 'name': return 1 diff --git a/wqflask/wqflask/snp_browser/snp_browser.py b/wqflask/wqflask/snp_browser/snp_browser.py index 7a0df94b..e5c67165 100644 --- a/wqflask/wqflask/snp_browser/snp_browser.py +++ b/wqflask/wqflask/snp_browser/snp_browser.py @@ -9,6 +9,7 @@ logger = getLogger(__name__) from base import species from base import webqtlConfig + class SnpBrowser: def __init__(self, start_vars): @@ -472,7 +473,6 @@ class SnpBrowser: base_color_dict = {"A": "#C33232", "C": "#1569C7", "T": "#CFCF32", "G": "#32C332", "t": "#FF6", "c": "#5CB3FF", "a": "#F66", "g": "#CF9", ":": "#FFFFFF", "-": "#FFFFFF", "?": "#FFFFFF"} - the_bases = [] for j, item in enumerate(allele_value_list): if item and isinstance(item, str): @@ -641,6 +641,7 @@ class SnpBrowser: # for i in range(n_click): # href = url_for('snp_browser', first_run="false", chosen_strains_mouse=self.chosen_strains_mouse, chosen_strains_rat=self.chosen_strains_rat, variant=self.variant_type, species=self.species_name, gene_name=self.gene_name, chr=self.chr, start_mb=self.start_mb, end_mb=self.end_mb, limit_strains=self.limit_strains, domain=self.domain, function=self.function, criteria=self.criteria, score=self.score, diff_alleles=self.diff_alleles) + def get_browser_sample_lists(species_id=1): strain_lists = {} mouse_strain_list = [] @@ -660,6 +661,7 @@ def get_browser_sample_lists(species_id=1): return strain_lists + def get_header_list(variant_type, strains, species=None, empty_columns=None): if species == "Mouse": strain_list = strains['mouse'] @@ -715,6 +717,7 @@ def get_header_list(variant_type, strains, species=None, empty_columns=None): return header_fields, empty_field_count, header_data_names + def get_effect_details_by_category(effect_name=None, effect_value=None): gene_list = [] transcript_list = [] @@ -764,6 +767,7 @@ def get_effect_details_by_category(effect_name=None, effect_value=None): return [gene_list, transcript_list, exon_list, function_list, function_detail_list] + def get_effect_info(effect_list): domain = "" effect_detail_list = [] @@ -836,6 +840,7 @@ def get_effect_info(effect_list): return effect_info_dict + def get_gene_id(species_id, gene_name): query = """ SELECT @@ -853,6 +858,7 @@ def get_gene_id(species_id, gene_name): else: return "" + def get_gene_id_name_dict(species_id, gene_name_list): gene_id_name_dict = {} if len(gene_name_list) == 0: @@ -877,6 +883,7 @@ def get_gene_id_name_dict(species_id, gene_name_list): return gene_id_name_dict + def check_if_in_gene(species_id, chr, mb): if species_id != 0: # ZS: Check if this is necessary query = """SELECT geneId, geneSymbol diff --git a/wqflask/wqflask/submit_bnw.py b/wqflask/wqflask/submit_bnw.py index 4ad6f9e3..b21a88cc 100644 --- a/wqflask/wqflask/submit_bnw.py +++ b/wqflask/wqflask/submit_bnw.py @@ -5,5 +5,6 @@ from utility import helper_functions import utility.logger logger = utility.logger.getLogger(__name__) + def get_bnw_input(start_vars): logger.debug("BNW VARS:", start_vars) diff --git a/wqflask/wqflask/update_search_results.py b/wqflask/wqflask/update_search_results.py index 08b4f9f5..07073d6a 100644 --- a/wqflask/wqflask/update_search_results.py +++ b/wqflask/wqflask/update_search_results.py @@ -10,6 +10,7 @@ from utility.benchmark import Bench from utility.logger import getLogger logger = getLogger(__name__) + class GSearch: def __init__(self, kw): diff --git a/wqflask/wqflask/user_login.py b/wqflask/wqflask/user_login.py index 3f5b43ee..2a2f8484 100644 --- a/wqflask/wqflask/user_login.py +++ b/wqflask/wqflask/user_login.py @@ -29,9 +29,11 @@ from utility.tools import SMTP_CONNECT, SMTP_USERNAME, SMTP_PASSWORD, LOG_SQL_AL THREE_DAYS = 60 * 60 * 24 * 3 + def timestamp(): return datetime.datetime.utcnow().isoformat() + def basic_info(): return dict(timestamp=timestamp(), ip_address=request.remote_addr, @@ -54,6 +56,7 @@ def encode_password(pass_gen_fields, unencrypted_password): return pass_gen_fields + def set_password(password): pass_gen_fields = { "unencrypted_password": password, @@ -71,6 +74,7 @@ def set_password(password): return encoded_password + def get_signed_session_id(user): session_id = str(uuid.uuid4()) @@ -109,6 +113,7 @@ def get_signed_session_id(user): return session_id_signed + def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"): """Send an E-mail through SMTP_CONNECT host. If SMTP_USERNAME is not 'UNKNOWN' TLS is used @@ -125,6 +130,7 @@ def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"): server.quit() logger.info("Successfully sent email to " + toaddr) + def send_verification_email(user_details, template_name="email/user_verification.txt", key_prefix="verification_code", subject = "GeneNetwork e-mail verification"): verification_code = str(uuid.uuid4()) key = key_prefix + ":" + verification_code @@ -139,12 +145,14 @@ def send_verification_email(user_details, template_name="email/user_verification send_email(recipient, subject, body) return {"recipient": recipient, "subject": subject, "body": body} + def send_invitation_email(user_email, temp_password, template_name="email/user_invitation.txt", subject= "You've been added to a GeneNetwork user group"): recipient = user_email body = render_template(template_name, temp_password) send_email(recipient, subject, body) return {"recipient": recipient, "subject": subject, "body": body} + @app.route("/manage/verify_email") def verify_email(): if 'code' in request.args: @@ -160,6 +168,7 @@ def verify_email(): else: flash("Invalid code: Password reset code does not exist or might have expired!", "error") + @app.route("/n/login", methods=('GET', 'POST')) def login(): params = request.form if request.form else request.args @@ -232,6 +241,7 @@ def login(): return response + @app.route("/n/login/github_oauth2", methods=('GET', 'POST')) def github_oauth2(): from utility.tools import GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET, GITHUB_AUTH_URL @@ -264,12 +274,14 @@ def github_oauth2(): url = "/n/login?type=github&uid=" + user_details["user_id"] return redirect(url) + def get_github_user_details(access_token): from utility.tools import GITHUB_API_URL result = requests.get(GITHUB_API_URL, headers={'Authorization': 'token ' + access_token}).content return json.loads(result) + @app.route("/n/login/orcid_oauth2", methods=('GET', 'POST')) def orcid_oauth2(): from uuid import uuid4 @@ -308,6 +320,7 @@ def orcid_oauth2(): flash("There was an error getting code from ORCID") return redirect(url) + def get_github_user_details(access_token): from utility.tools import GITHUB_API_URL result = requests.get(GITHUB_API_URL, headers={'Authorization': 'token ' + access_token}).content @@ -325,6 +338,7 @@ def logout(): response.set_cookie(UserSession.user_cookie_name, '', expires=0) return response + @app.route("/n/forgot_password", methods=['GET']) def forgot_password(): """Entry point for forgotten password""" @@ -333,6 +347,7 @@ def forgot_password(): print("ERRORS: ", errors) return render_template("new_security/forgot_password.html", errors=errors) + def send_forgot_password_email(verification_email): from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText @@ -365,6 +380,7 @@ def send_forgot_password_email(verification_email): return subject + @app.route("/n/forgot_password_submit", methods=('POST',)) def forgot_password_submit(): """When a forgotten password form is submitted we get here""" @@ -386,6 +402,7 @@ def forgot_password_submit(): flash("You MUST provide an email", "alert-danger") return redirect(url_for("forgot_password")) + @app.route("/n/password_reset", methods=['GET']) def password_reset(): """Entry point after user clicks link in E-mail""" @@ -405,6 +422,7 @@ def password_reset(): else: return redirect(url_for("login")) + @app.route("/n/password_reset_step2", methods=('POST',)) def password_reset_step2(): """Handle confirmation E-mail for password reset""" @@ -422,6 +440,7 @@ def password_reset_step2(): flash("Password changed successfully. You can now sign in.", "alert-info") return redirect(url_for('login')) + def register_user(params): thank_you_mode = False errors = [] @@ -461,6 +480,7 @@ def register_user(params): return errors + @app.route("/n/register", methods=('GET', 'POST')) def register(): errors = [] @@ -478,6 +498,7 @@ def register(): return render_template("new_security/register_user.html", values=params, errors=errors) + @app.errorhandler(401) def unauthorized(error): return redirect(url_for('login')) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 013920f9..a9bd65e6 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -151,6 +151,7 @@ def verify_cookie(cookie): assert the_signature == actual_hmac_creation(the_uuid), "Uh-oh, someone tampering with the cookie?" return the_uuid + def create_signed_cookie(): the_uuid = str(uuid.uuid4()) signature = actual_hmac_creation(the_uuid) @@ -158,6 +159,7 @@ def create_signed_cookie(): logger.debug("uuid_signed:", uuid_signed) return the_uuid, uuid_signed + class UserSession: """Logged in user handling""" @@ -341,6 +343,7 @@ class UserSession: Redis.delete(self.cookie_name) logger.debug("At end of delete_session") + @app.before_request def get_cookie(): logger.info("@app.before_request get cookie") @@ -348,16 +351,20 @@ def get_cookie(): g.cookie_session = AnonUser() # @app.after_request + + def set_cookie(response): if not request.cookies.get(g.cookie_session.cookie_name): response.set_cookie(g.cookie_session.cookie_name, g.cookie_session.cookie) return response + class UsersManager: def __init__(self): self.users = model.User.query.all() logger.debug("Users are:", self.users) + class UserManager: def __init__(self, kw): self.user_id = kw['user_id'] @@ -419,6 +426,7 @@ class RegisterUser: self.user.registration_info = json.dumps(basic_info(), sort_keys=True) save_user(self.user.__dict__, self.user.user_id) + def set_password(password, user): pwfields = Bunch() @@ -476,6 +484,7 @@ class VerificationEmail: verification_code=verification_code) send_email(to, subject, body) + class ForgotPasswordEmail(VerificationEmail): template_name = "email/forgot_password.txt" key_prefix = "forgot_password_code" @@ -496,7 +505,6 @@ class ForgotPasswordEmail(VerificationEmail): save_verification_code(toaddr, verification_code) - subject = self.subject body = render_template( self.template_name, @@ -530,6 +538,8 @@ def basic_info(): user_agent=request.headers.get('User-Agent')) # @app.route("/manage/verify_email") + + def verify_email(): user = DecodeUser(VerificationEmail.key_prefix).user user.confirmed = json.dumps(basic_info(), sort_keys=True) @@ -544,6 +554,8 @@ def verify_email(): return response # @app.route("/n/password_reset", methods=['GET']) + + def password_reset(): """Entry point after user clicks link in E-mail""" logger.debug("in password_reset request.url is:", request.url) @@ -568,6 +580,8 @@ def password_reset(): return redirect(url_for("login")) # @app.route("/n/password_reset_step2", methods=('POST',)) + + def password_reset_step2(): """Handle confirmation E-mail for password reset""" logger.debug("in password_reset request.url is:", request.url) @@ -577,7 +591,6 @@ def password_reset_step2(): logger.debug("locals are:", locals()) - user = Bunch() password = request.form['password'] set_password(password, user) @@ -589,6 +602,7 @@ def password_reset_step2(): return response + class DecodeUser: def __init__(self, code_prefix): @@ -612,6 +626,8 @@ class DecodeUser: return model.User.query.get(data['id']) # @app.route("/n/login", methods=('GET', 'POST')) + + def login(): lu = LoginUser() login_type = request.args.get("type") @@ -622,6 +638,8 @@ def login(): return lu.standard_login() # @app.route("/n/login/github_oauth2", methods=('GET', 'POST')) + + def github_oauth2(): from utility.tools import GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET code = request.args.get("code") @@ -646,6 +664,8 @@ def github_oauth2(): return redirect(url) # @app.route("/n/login/orcid_oauth2", methods=('GET', 'POST')) + + def orcid_oauth2(): from uuid import uuid4 from utility.tools import ORCID_CLIENT_ID, ORCID_CLIENT_SECRET, ORCID_TOKEN_URL, ORCID_AUTH_URL @@ -673,11 +693,13 @@ def orcid_oauth2(): flash("There was an error getting code from ORCID") return redirect(url) + def get_github_user_details(access_token): from utility.tools import GITHUB_API_URL result = requests.get(GITHUB_API_URL, params={"access_token": access_token}) return result.json() + class LoginUser: remember_time = 60 * 60 * 24 * 30 # One month in seconds @@ -814,6 +836,8 @@ class LoginUser: db_session.commit() # @app.route("/n/logout") + + def logout(): logger.debug("Logging out...") UserSession().delete_session() @@ -833,6 +857,8 @@ def forgot_password(): return render_template("new_security/forgot_password.html", errors=errors) # @app.route("/n/forgot_password_submit", methods=('POST',)) + + def forgot_password_submit(): """When a forgotten password form is submitted we get here""" params = request.form @@ -853,10 +879,12 @@ def forgot_password_submit(): flash("You MUST provide an email", "alert-danger") return redirect(url_for("forgot_password")) + @app.errorhandler(401) def unauthorized(error): return redirect(url_for('login')) + def is_redis_available(): try: Redis.ping() @@ -922,7 +950,6 @@ def register(): params = None errors = None - params = request.form if request.form else request.args params = params.to_dict(flat=True) @@ -952,6 +979,7 @@ def url_for_hmac(endpoint, **values): combiner = "?" return url + combiner + "hm=" + hm + def data_hmac(stringy): """Takes arbitray data string and appends :hmac so we know data hasn't been tampered with""" return stringy + ":" + actual_hmac_creation(stringy) @@ -974,6 +1002,7 @@ def verify_url_hmac(url): assert hm == hmac, "Unexpected url (stage 3)" + def actual_hmac_creation(stringy): """Helper function to create the actual hmac""" @@ -986,6 +1015,7 @@ def actual_hmac_creation(stringy): hm = hm[:20] return hm + app.jinja_env.globals.update(url_for_hmac=url_for_hmac, data_hmac=data_hmac) @@ -998,6 +1028,7 @@ app.jinja_env.globals.update(url_for_hmac=url_for_hmac, # Body=body)) # Redis.rpush("mail_queue", msg) + def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"): """Send an E-mail through SMTP_CONNECT host. If SMTP_USERNAME is not 'UNKNOWN' TLS is used @@ -1020,6 +1051,7 @@ def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"): server.quit() logger.info("Successfully sent email to " + toaddr) + class GroupsManager: def __init__(self, kw): self.datasets = create_datasets_list() diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py index 78db7bd2..f0f0d60c 100644 --- a/wqflask/wqflask/user_session.py +++ b/wqflask/wqflask/user_session.py @@ -20,6 +20,7 @@ logger = getLogger(__name__) THREE_DAYS = 60 * 60 * 24 * 3 THIRTY_DAYS = 60 * 60 * 24 * 30 + @app.before_request def get_user_session(): logger.info("@app.before_request get_session") @@ -30,6 +31,7 @@ def get_user_session(): response.set_cookie('session_id_v2', '', expires=0) return response + @app.after_request def set_user_session(response): if hasattr(g, 'user_session'): @@ -37,6 +39,7 @@ def set_user_session(response): response.set_cookie(g.user_session.cookie_name, g.user_session.cookie) return response + def verify_cookie(cookie): the_uuid, separator, the_signature = cookie.partition(':') assert len(the_uuid) == 36, "Is session_id a uuid?" @@ -44,6 +47,7 @@ def verify_cookie(cookie): assert the_signature == hmac.hmac_creation(the_uuid), "Uh-oh, someone tampering with the cookie?" return the_uuid + def create_signed_cookie(): the_uuid = str(uuid.uuid4()) signature = hmac.hmac_creation(the_uuid) @@ -51,6 +55,7 @@ def create_signed_cookie(): logger.debug("uuid_signed:", uuid_signed) return the_uuid, uuid_signed + @app.route("/user/manage", methods=('GET', 'POST')) def manage_user(): params = request.form if request.form else request.args @@ -63,6 +68,7 @@ def manage_user(): return render_template("admin/manage_user.html", user_details=user_details) + class UserSession: """Logged in user handling""" |