From 5745c3bdd086f7c499ee63a580df822db5af2826 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 18 May 2021 19:43:50 +0000 Subject: Added unit test for computations/rqtl.py --- tests/unit/computations/test_rqtl.py | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 tests/unit/computations/test_rqtl.py (limited to 'tests') diff --git a/tests/unit/computations/test_rqtl.py b/tests/unit/computations/test_rqtl.py new file mode 100644 index 0000000..b16f136 --- /dev/null +++ b/tests/unit/computations/test_rqtl.py @@ -0,0 +1,41 @@ +"""Test cases for procedures defined in computations.rqtl""" +import unittest + +from unittest import mock +from gn3.computations.rqtl import generate_rqtl_cmd + +class TestRqtl(unittest.TestCase): + """Test cases for computations.rqtl module""" + @mock.patch("gn3.computations.rqtl.generate_hash_of_string") + @mock.patch("gn3.computations.rqtl.get_hash_of_files") + def test_generate_rqtl_command(self, mock_get_hash_files, mock_generate_hash_string): + """Test computing mapping results with R/qtl""" + mock_get_hash_files.return_value = "my-hash1" + mock_generate_hash_string.return_value = "my-hash2" + + self.assertEqual( + generate_rqtl_cmd(rqtl_wrapper_cmd="rqtl-wrapper", + rqtl_wrapper_kwargs={ + "g": "genofile", + "p": "phenofile", + "model": "normal", + "method": "hk", + "nperm": 1000, + "scale": "Mb", + "control": "rs123456" + }, + rqtl_wrapper_bool_kwargs=[ + "addcovar", + "interval" + ]), { + "output_file": + "my-hash1my-hash2my-hash2-output.json", + "rqtl_cmd": ( + "rqtl-wrapper " + "--g genofile --p phenofile " + "--model normal --method hk " + "--nperm 1000 --scale Mb " + "--control rs123456 " + "--addcovar --interval" + ) + }) -- cgit v1.2.3 From d66b71a1e149ccddbbcc66e439067250827e0b6f Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 18 May 2021 20:06:57 +0000 Subject: Added test for compose_rqtl_cmd in tests/unit/test_commands.py --- tests/unit/test_commands.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'tests') diff --git a/tests/unit/test_commands.py b/tests/unit/test_commands.py index aafb3a2..a3d0273 100644 --- a/tests/unit/test_commands.py +++ b/tests/unit/test_commands.py @@ -6,6 +6,7 @@ from datetime import datetime from typing import Callable from unittest import mock from gn3.commands import compose_gemma_cmd +from gn3.commands import compose_rqtl_cmd from gn3.commands import queue_cmd from gn3.commands import run_cmd from gn3.exceptions import RedisConnectionError @@ -53,6 +54,31 @@ class TestCommands(unittest.TestCase): "-p /tmp/gf13Ad0tRX/phenofile.txt" " -gk")) + def test_compose_rqtl_cmd(self): + """Test that the R/qtl cmd is composed correctly""" + self.assertEqual( + compose_rqtl_cmd(rqtl_wrapper_cmd="rqtl-wrapper", + rqtl_wrapper_kwargs={ + "g": "genofile", + "p": "phenofile", + "model": "normal", + "method": "hk", + "nperm": 1000, + "scale": "Mb", + "control": "rs123456" + }, + rqtl_wrapper_bool_kwargs=[ + "addcovar", + "interval" + ]), + ("rqtl-wrapper " + "--g genofile --p phenofile " + "--model normal --method hk " + "--nperm 1000 --scale Mb " + "--control rs123456 " + "--addcovar --interval") + ) + def test_queue_cmd_exception_raised_when_redis_is_down(self): """Test that the correct error is raised when Redis is unavailable""" self.assertRaises(RedisConnectionError, -- cgit v1.2.3 From c73c4d4edf88d2af5636962f1e4710be17516ea1 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 17 May 2021 14:34:25 +0300 Subject: tests: test_phenotypes: New test cases for loading phenotypes --- tests/unit/db/test_phenotypes.py | 41 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 tests/unit/db/test_phenotypes.py (limited to 'tests') diff --git a/tests/unit/db/test_phenotypes.py b/tests/unit/db/test_phenotypes.py new file mode 100644 index 0000000..8b810fe --- /dev/null +++ b/tests/unit/db/test_phenotypes.py @@ -0,0 +1,41 @@ +"""Tests for db/phenotypes.py""" +from unittest import TestCase +from unittest import mock + +from gn3.db.phenotypes import Phenotype +from gn3.db.phenotypes import update_phenotype + + +class TestPhenotypes(TestCase): + """Test cases for fetching chromosomes""" + def test_update_phenotype_with_no_data(self): + """ + Test that a phenotype is updated correctly if an empty Phenotype dataclass + is provided + """ + db_mock = mock.MagicMock() + self.assertEqual(update_phenotype( + db_mock, data=Phenotype(), where=Phenotype()), None) + + def test_update_phenotype_with_data(self): + """ + Test that a phenotype is updated correctly if some + data is provided + """ + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + type(cursor).rowcount = 1 + self.assertEqual(update_phenotype( + db_mock, data=Phenotype( + pre_pub_description="Test Pre Pub", + submitter="Rob", + post_pub_description="Test Post Pub"), + where=Phenotype(id_=1)), 1) + cursor.execute.assert_called_once_with( + "UPDATE Phenotype SET " + "Pre_publication_description = " + "'Test Pre Pub', " + "Post_publication_description = " + "'Test Post Pub', Submitter = 'Rob' " + "WHERE id = '1'" + ) -- cgit v1.2.3 From 856d4a72ace3584b731a7c45470e3df2704be02f Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 19 May 2021 22:12:45 +0300 Subject: db: phenotypes: Update failing tests --- tests/unit/db/test_phenotypes.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/tests/unit/db/test_phenotypes.py b/tests/unit/db/test_phenotypes.py index 8b810fe..6b394d7 100644 --- a/tests/unit/db/test_phenotypes.py +++ b/tests/unit/db/test_phenotypes.py @@ -3,19 +3,20 @@ from unittest import TestCase from unittest import mock from gn3.db.phenotypes import Phenotype -from gn3.db.phenotypes import update_phenotype +from gn3.db.phenotypes import update class TestPhenotypes(TestCase): """Test cases for fetching chromosomes""" def test_update_phenotype_with_no_data(self): - """ - Test that a phenotype is updated correctly if an empty Phenotype dataclass + """Test that a phenotype is updated correctly if an empty Phenotype dataclass is provided + """ db_mock = mock.MagicMock() - self.assertEqual(update_phenotype( - db_mock, data=Phenotype(), where=Phenotype()), None) + self.assertEqual(update( + conn=db_mock, table="Phenotype", + data=Phenotype(), where=Phenotype()), None) def test_update_phenotype_with_data(self): """ @@ -25,8 +26,9 @@ class TestPhenotypes(TestCase): db_mock = mock.MagicMock() with db_mock.cursor() as cursor: type(cursor).rowcount = 1 - self.assertEqual(update_phenotype( - db_mock, data=Phenotype( + self.assertEqual(update( + conn=db_mock, table="Phenotype", + data=Phenotype( pre_pub_description="Test Pre Pub", submitter="Rob", post_pub_description="Test Post Pub"), -- cgit v1.2.3 From 786d48ede007f0134081495f3f63be3a33b8f71e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 20 May 2021 21:26:34 +0300 Subject: tests: test_phenotype: Add function that tests "fetchone" --- tests/unit/db/test_phenotypes.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'tests') diff --git a/tests/unit/db/test_phenotypes.py b/tests/unit/db/test_phenotypes.py index 6b394d7..9fed524 100644 --- a/tests/unit/db/test_phenotypes.py +++ b/tests/unit/db/test_phenotypes.py @@ -2,6 +2,7 @@ from unittest import TestCase from unittest import mock +from gn3.db.phenotypes import fetchone from gn3.db.phenotypes import Phenotype from gn3.db.phenotypes import update @@ -41,3 +42,24 @@ class TestPhenotypes(TestCase): "'Test Post Pub', Submitter = 'Rob' " "WHERE id = '1'" ) + + def test_fetch_phenotype(self): + """Test that a single phenotype is fetched properly + + """ + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + test_data = ( + 35, "Test pre-publication", "Test post-publication", + "Original description A", "cm^2", "pre-abbrev", + "post-abbrev", "LAB001", "R. W.", "R. W.", "R. W." + ) + cursor.fetchone.return_value = test_data + phenotype = fetchone(db_mock, + "Phenotype", + where=Phenotype(id_=35)) + self.assertEqual(phenotype.id_, 35) + self.assertEqual(phenotype.pre_pub_description, + "Test pre-publication") + cursor.execute.assert_called_once_with( + "SELECT * FROM Phenotype WHERE id = '35'") -- cgit v1.2.3 From 93ab68fe650eed0bb53d77225f47f72e527e48c4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 May 2021 12:05:08 +0300 Subject: Move the methods, "update" and "fetch", to gn3.db --- gn3/db/__init__.py | 70 ++++++++++++++++++++++++++++++++++++++++ gn3/db/phenotypes.py | 64 ++---------------------------------- tests/unit/db/test_phenotypes.py | 4 +-- 3 files changed, 74 insertions(+), 64 deletions(-) (limited to 'tests') diff --git a/gn3/db/__init__.py b/gn3/db/__init__.py index e69de29..fae4d29 100644 --- a/gn3/db/__init__.py +++ b/gn3/db/__init__.py @@ -0,0 +1,70 @@ +# pylint: disable=[R0902, R0903] +"""Module that exposes common db operations""" +from typing import Optional, Dict, Any +from dataclasses import dataclass, asdict, astuple +from typing_extensions import Protocol +from MySQLdb import escape_string + +from gn3.db.phenotypes import Phenotype +from gn3.db.phenotypes import PublishXRef +from gn3.db.phenotypes import Publication + +from gn3.db.phenotypes import phenotype_mapping +from gn3.db.phenotypes import publish_x_ref_mapping +from gn3.db.phenotypes import publication_mapping + +TABLEMAP = { + "Phenotype": phenotype_mapping, + "PublishXRef": publish_x_ref_mapping, + "Publication": publication_mapping, +} + +DATACLASSMAP = { + "Phenotype": Phenotype, + "PublishXRef": PublishXRef, + "Publication": Publication, +} + + +class Dataclass(Protocol): + """Type Definition for a Dataclass""" + __dataclass_fields__: Dict + + +def update(conn: Any, + table: str, + data: Dataclass, + where: Dataclass) -> Optional[int]: + """Run an UPDATE on a table""" + if not any(astuple(data) + astuple(where)): + return None + sql = f"UPDATE {table} SET " + sql += ", ".join(f"{TABLEMAP[table].get(k)} " + f"= '{escape_string(str(v)).decode('utf-8')}'" for + k, v in asdict(data).items() + if v is not None and k in TABLEMAP[table]) + sql += " WHERE " + sql += "AND ".join(f"{TABLEMAP[table].get(k)} = " + f"'{escape_string(str(v)).decode('utf-8')}'" for + k, v in asdict(where).items() + if v is not None and k in TABLEMAP[table]) + with conn.cursor() as cursor: + cursor.execute(sql) + return cursor.rowcount + + +def fetchone(conn: Any, + table: str, + where: Dataclass) -> Optional[Dataclass]: + """Run a SELECT on a table. Returns only one result!""" + if not any(astuple(where)): + return None + sql = f"SELECT * FROM {table} " + sql += "WHERE " + sql += "AND ".join(f"{TABLEMAP[table].get(k)} = " + f"'{escape_string(str(v)).decode('utf-8')}'" for + k, v in asdict(where).items() + if v is not None and k in TABLEMAP[table]) + with conn.cursor() as cursor: + cursor.execute(sql) + return DATACLASSMAP[table](*cursor.fetchone()) diff --git a/gn3/db/phenotypes.py b/gn3/db/phenotypes.py index ee523ad..2b93c85 100644 --- a/gn3/db/phenotypes.py +++ b/gn3/db/phenotypes.py @@ -1,17 +1,9 @@ # pylint: disable=[R0902, R0903] """This contains all the necessary functions that access the phenotypes from the db""" -from dataclasses import dataclass, asdict, astuple +from dataclasses import dataclass -from typing import Any, Dict, Optional -from MySQLdb import escape_string - -from typing_extensions import Protocol - - -class Dataclass(Protocol): - """Type Definition for a Dataclass""" - __dataclass_fields__: Dict +from typing import Optional @dataclass(frozen=True) @@ -107,55 +99,3 @@ publication_mapping = { "month": "Month", "year": "Year", } - - -TABLEMAP = { - "Phenotype": phenotype_mapping, - "PublishXRef": publish_x_ref_mapping, - "Publication": publication_mapping, -} - -DATACLASSMAP = { - "Phenotype": Phenotype, - "PublishXRef": PublishXRef, - "Publication": Publication, -} - - -def update(conn: Any, - table: str, - data: Dataclass, - where: Dataclass) -> Optional[int]: - """Run an UPDATE on a table""" - if not any(astuple(data) + astuple(where)): - return None - sql = f"UPDATE {table} SET " - sql += ", ".join(f"{TABLEMAP[table].get(k)} " - f"= '{escape_string(str(v)).decode('utf-8')}'" for - k, v in asdict(data).items() - if v is not None and k in TABLEMAP[table]) - sql += " WHERE " - sql += "AND ".join(f"{TABLEMAP[table].get(k)} = " - f"'{escape_string(str(v)).decode('utf-8')}'" for - k, v in asdict(where).items() - if v is not None and k in TABLEMAP[table]) - with conn.cursor() as cursor: - cursor.execute(sql) - return cursor.rowcount - - -def fetchone(conn: Any, - table: str, - where: Dataclass) -> Optional[Dataclass]: - """Run a SELECT on a table. Returns only one result!""" - if not any(astuple(where)): - return None - sql = f"SELECT * FROM {table} " - sql += "WHERE " - sql += "AND ".join(f"{TABLEMAP[table].get(k)} = " - f"'{escape_string(str(v)).decode('utf-8')}'" for - k, v in asdict(where).items() - if v is not None and k in TABLEMAP[table]) - with conn.cursor() as cursor: - cursor.execute(sql) - return DATACLASSMAP[table](*cursor.fetchone()) diff --git a/tests/unit/db/test_phenotypes.py b/tests/unit/db/test_phenotypes.py index 9fed524..505714a 100644 --- a/tests/unit/db/test_phenotypes.py +++ b/tests/unit/db/test_phenotypes.py @@ -2,9 +2,9 @@ from unittest import TestCase from unittest import mock -from gn3.db.phenotypes import fetchone +from gn3.db import fetchone +from gn3.db import update from gn3.db.phenotypes import Phenotype -from gn3.db.phenotypes import update class TestPhenotypes(TestCase): -- cgit v1.2.3 From 07464f44f48895cc31ba2b088d6125e7777e1073 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sun, 30 May 2021 13:26:15 +0300 Subject: fix index error (#16) --- gn3/computations/correlations.py | 2 +- tests/unit/computations/test_correlation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/gn3/computations/correlations.py b/gn3/computations/correlations.py index 25dd26d..f0ce502 100644 --- a/gn3/computations/correlations.py +++ b/gn3/computations/correlations.py @@ -247,7 +247,7 @@ def fetch_lit_correlation_data( cursor.execute(query_formatter(query, *tuple(reversed(query_values)))) lit_corr_results = cursor.fetchone() - lit_results = (gene_id, lit_corr_results[1])\ + lit_results = (gene_id, lit_corr_results[0])\ if lit_corr_results else (gene_id, 0) return lit_results return (gene_id, 0) diff --git a/tests/unit/computations/test_correlation.py b/tests/unit/computations/test_correlation.py index d264738..5746adf 100644 --- a/tests/unit/computations/test_correlation.py +++ b/tests/unit/computations/test_correlation.py @@ -276,7 +276,7 @@ class TestCorrelation(TestCase): input trait mouse gene id and mouse gene id """ - expected_db_results = [("val", x*0.1) + expected_db_results = [[x*0.1] for x in range(1, 4)] conn = DataBase(expected_results=expected_db_results) expected_results = ("1", 0.1) -- cgit v1.2.3 From ece41f5f971595c5d005c4beaa984c45471a6647 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 2 Jun 2021 07:54:35 +0300 Subject: Get the diff between 2 dicts and return that as a dict --- gn3/db/__init__.py | 16 ++++++++++++++++ tests/unit/db/test_phenotypes.py | 8 ++++++++ 2 files changed, 24 insertions(+) (limited to 'tests') diff --git a/gn3/db/__init__.py b/gn3/db/__init__.py index 1eb7b12..19135fc 100644 --- a/gn3/db/__init__.py +++ b/gn3/db/__init__.py @@ -68,3 +68,19 @@ def fetchone(conn: Any, with conn.cursor() as cursor: cursor.execute(sql) return DATACLASSMAP[table](*cursor.fetchone()) + + +def diff_from_dict(old: Dict, new: Dict) -> Dict: + """Construct a new dict with a specific structure that contains the difference +between the 2 dicts in the structure: + +diff_from_dict({"id": 1, "data": "a"}, {"id": 2, "data": "b"}) + +Should return: + +{"id": {"old": 1, "new": 2}, "data": {"old": "a", "new": "b"}} + """ + dict_ = {} + for key, value in old.items(): + dict_[key] = {"old": old[key], "new": new[key]} + return dict_ diff --git a/tests/unit/db/test_phenotypes.py b/tests/unit/db/test_phenotypes.py index 505714a..b53db23 100644 --- a/tests/unit/db/test_phenotypes.py +++ b/tests/unit/db/test_phenotypes.py @@ -4,6 +4,7 @@ from unittest import mock from gn3.db import fetchone from gn3.db import update +from gn3.db import diff_from_dict from gn3.db.phenotypes import Phenotype @@ -63,3 +64,10 @@ class TestPhenotypes(TestCase): "Test pre-publication") cursor.execute.assert_called_once_with( "SELECT * FROM Phenotype WHERE id = '35'") + + def test_diff_from_dict(self): + """Test that a correct diff is generated""" + self.assertEqual(diff_from_dict({"id": 1, "data": "a"}, + {"id": 2, "data": "b"}), + {"id": {"old": 1, "new": 2}, + "data": {"old": "a", "new": "b"}}) -- cgit v1.2.3 From a2ef9618dbba2d3f0416cbe8a527ed12070aa67e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 2 Jun 2021 08:22:11 +0300 Subject: unit: test_phenotypes: Test for multiple "WHERE" clauses --- tests/unit/db/test_phenotypes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/unit/db/test_phenotypes.py b/tests/unit/db/test_phenotypes.py index b53db23..fdeca5e 100644 --- a/tests/unit/db/test_phenotypes.py +++ b/tests/unit/db/test_phenotypes.py @@ -34,14 +34,14 @@ class TestPhenotypes(TestCase): pre_pub_description="Test Pre Pub", submitter="Rob", post_pub_description="Test Post Pub"), - where=Phenotype(id_=1)), 1) + where=Phenotype(id_=1, owner="Rob")), 1) cursor.execute.assert_called_once_with( "UPDATE Phenotype SET " "Pre_publication_description = " "'Test Pre Pub', " "Post_publication_description = " "'Test Post Pub', Submitter = 'Rob' " - "WHERE id = '1'" + "WHERE id = '1' AND Owner = 'Rob'" ) def test_fetch_phenotype(self): @@ -58,12 +58,12 @@ class TestPhenotypes(TestCase): cursor.fetchone.return_value = test_data phenotype = fetchone(db_mock, "Phenotype", - where=Phenotype(id_=35)) + where=Phenotype(id_=35, owner="Rob")) self.assertEqual(phenotype.id_, 35) self.assertEqual(phenotype.pre_pub_description, "Test pre-publication") cursor.execute.assert_called_once_with( - "SELECT * FROM Phenotype WHERE id = '35'") + "SELECT * FROM Phenotype WHERE id = '35' AND Owner = 'Rob'") def test_diff_from_dict(self): """Test that a correct diff is generated""" -- cgit v1.2.3 From c96b29e63577f7189afd02df2ced26b150830341 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 3 Jun 2021 11:08:58 +0300 Subject: Add data structures for the table metadata_audit --- gn3/db/__init__.py | 5 +++++ gn3/db/metadata_audit.py | 26 ++++++++++++++++++++++++++ tests/unit/db/test_audit.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 gn3/db/metadata_audit.py create mode 100644 tests/unit/db/test_audit.py (limited to 'tests') diff --git a/gn3/db/__init__.py b/gn3/db/__init__.py index d89dbf4..175a640 100644 --- a/gn3/db/__init__.py +++ b/gn3/db/__init__.py @@ -5,21 +5,26 @@ from dataclasses import dataclass, asdict, astuple from typing_extensions import Protocol from MySQLdb import escape_string +from gn3.db.metadata_audit import MetadataAudit from gn3.db.phenotypes import Phenotype from gn3.db.phenotypes import PublishXRef from gn3.db.phenotypes import Publication +from gn3.db.metadata_audit import metadata_audit_mapping from gn3.db.phenotypes import phenotype_mapping from gn3.db.phenotypes import publish_x_ref_mapping from gn3.db.phenotypes import publication_mapping + TABLEMAP = { + "metadata_audit": metadata_audit_mapping, "Phenotype": phenotype_mapping, "PublishXRef": publish_x_ref_mapping, "Publication": publication_mapping, } DATACLASSMAP = { + "MetadataAudit": MetadataAudit, "Phenotype": Phenotype, "PublishXRef": PublishXRef, "Publication": Publication, diff --git a/gn3/db/metadata_audit.py b/gn3/db/metadata_audit.py new file mode 100644 index 0000000..6e22b32 --- /dev/null +++ b/gn3/db/metadata_audit.py @@ -0,0 +1,26 @@ +# pylint: disable=[R0902, R0903] +"""This contains all the necessary functions that access the metadata_audit +table from the db + +""" +from dataclasses import dataclass +from typing import Optional + + +@dataclass(frozen=True) +class MetadataAudit: + """Data Type that represents a Phenotype""" + dataset_id: int + editor: str + json_data: str + time_stamp: Optional[str] = None + + +# Mapping from the MetadataAudit dataclass to the actual column names in the +# database +metadata_audit_mapping = { + "dataset_id": "dataset_id", + "editor": "editor", + "json_data": "json_data", + "time_stamp": "time_stamp", +} diff --git a/tests/unit/db/test_audit.py b/tests/unit/db/test_audit.py new file mode 100644 index 0000000..22787bb --- /dev/null +++ b/tests/unit/db/test_audit.py @@ -0,0 +1,28 @@ +"""Tests for db/phenotypes.py""" +import json +from unittest import TestCase +from unittest import mock + +from gn3.db import insert +from gn3.db.metadata_audit import MetadataAudit + + +class TestMetadatAudit(TestCase): + """Test cases for fetching chromosomes""" + + def test_insert_into_metadata_audit(self): + """Test that data is inserted correctly in the audit table + + """ + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + type(cursor).rowcount = 1 + self.assertEqual(insert( + conn=db_mock, table="metadata_audit", + data=MetadataAudit(dataset_id=35, + editor="Bonface", + json_data=json.dumps({"a": "b"}))), 1) + cursor.execute.assert_called_once_with( + "INSERT INTO metadata_audit ('dataset_id', " + "'editor', 'json_data') " + 'VALUES (\'35\', \'Bonface\', \'{\\"a\\": \\"b\\"}\')') -- cgit v1.2.3 From de834809dbf5f054a5f75c35dbee653cac8311f3 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 3 Jun 2021 21:11:33 +0300 Subject: tests: test_audit: Update test case to check for prepared statements --- tests/unit/db/test_audit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/tests/unit/db/test_audit.py b/tests/unit/db/test_audit.py index 22787bb..1449281 100644 --- a/tests/unit/db/test_audit.py +++ b/tests/unit/db/test_audit.py @@ -23,6 +23,6 @@ class TestMetadatAudit(TestCase): editor="Bonface", json_data=json.dumps({"a": "b"}))), 1) cursor.execute.assert_called_once_with( - "INSERT INTO metadata_audit ('dataset_id', " - "'editor', 'json_data') " - 'VALUES (\'35\', \'Bonface\', \'{\\"a\\": \\"b\\"}\')') + "INSERT INTO metadata_audit (dataset_id, " + "editor, json_data) VALUES (%s, %s, %s)", + (35, 'Bonface', '{"a": "b"}')) -- cgit v1.2.3 From 4c9bbe6d4229b79a1bc62cf2f641fbc4c4f00abc Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 3 Jun 2021 21:38:58 +0300 Subject: Use prepared statements for UPDATE sql function --- gn3/db/__init__.py | 16 +++++++++------- tests/unit/db/test_phenotypes.py | 10 ++++------ 2 files changed, 13 insertions(+), 13 deletions(-) (limited to 'tests') diff --git a/gn3/db/__init__.py b/gn3/db/__init__.py index 8b6bf73..ce92a7d 100644 --- a/gn3/db/__init__.py +++ b/gn3/db/__init__.py @@ -43,18 +43,20 @@ def update(conn: Any, """Run an UPDATE on a table""" if not (any(astuple(data)) and any(astuple(where))): return None + data_ = {k: v for k, v in asdict(data).items() + if v is not None and k in TABLEMAP[table]} + where_ = {k: v for k, v in asdict(where).items() + if v is not None and k in TABLEMAP[table]} sql = f"UPDATE {table} SET " sql += ", ".join(f"{TABLEMAP[table].get(k)} " - f"= '{escape_string(str(v)).decode('utf-8')}'" for - k, v in asdict(data).items() - if v is not None and k in TABLEMAP[table]) + "= %s" for k in data_.keys()) sql += " WHERE " sql += " AND ".join(f"{TABLEMAP[table].get(k)} = " - f"'{escape_string(str(v)).decode('utf-8')}'" for - k, v in asdict(where).items() - if v is not None and k in TABLEMAP[table]) + "%s" for k in where_.keys()) with conn.cursor() as cursor: - cursor.execute(sql) + cursor.execute(sql, + tuple(data_.values()) + tuple(where_.values())) + conn.commit() return cursor.rowcount diff --git a/tests/unit/db/test_phenotypes.py b/tests/unit/db/test_phenotypes.py index fdeca5e..21eb757 100644 --- a/tests/unit/db/test_phenotypes.py +++ b/tests/unit/db/test_phenotypes.py @@ -37,12 +37,10 @@ class TestPhenotypes(TestCase): where=Phenotype(id_=1, owner="Rob")), 1) cursor.execute.assert_called_once_with( "UPDATE Phenotype SET " - "Pre_publication_description = " - "'Test Pre Pub', " - "Post_publication_description = " - "'Test Post Pub', Submitter = 'Rob' " - "WHERE id = '1' AND Owner = 'Rob'" - ) + "Pre_publication_description = %s, " + "Post_publication_description = %s, " + "Submitter = %s WHERE id = %s AND Owner = %s", + ('Test Pre Pub', 'Test Post Pub', 'Rob', 1, 'Rob')) def test_fetch_phenotype(self): """Test that a single phenotype is fetched properly -- cgit v1.2.3 From 8210c46fde908b8815ab97f2f91039f87365369b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 3 Jun 2021 21:45:25 +0300 Subject: Use prepared statements for FETCH sql function --- gn3/db/__init__.py | 8 ++++---- tests/unit/db/test_phenotypes.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'tests') diff --git a/gn3/db/__init__.py b/gn3/db/__init__.py index d62b575..fea43ec 100644 --- a/gn3/db/__init__.py +++ b/gn3/db/__init__.py @@ -66,14 +66,14 @@ def fetchone(conn: Any, """Run a SELECT on a table. Returns only one result!""" if not any(astuple(where)): return None + where_ = {k: v for k, v in asdict(where).items() + if v is not None and k in TABLEMAP[table]} sql = f"SELECT * FROM {table} " sql += "WHERE " sql += " AND ".join(f"{TABLEMAP[table].get(k)} = " - f"'{escape_string(str(v)).decode('utf-8')}'" for - k, v in asdict(where).items() - if v is not None and k in TABLEMAP[table]) + "%s" for k in where_.keys()) with conn.cursor() as cursor: - cursor.execute(sql) + cursor.execute(sql, tuple(where_.values())) return DATACLASSMAP[table](*cursor.fetchone()) diff --git a/tests/unit/db/test_phenotypes.py b/tests/unit/db/test_phenotypes.py index 21eb757..824d186 100644 --- a/tests/unit/db/test_phenotypes.py +++ b/tests/unit/db/test_phenotypes.py @@ -61,7 +61,8 @@ class TestPhenotypes(TestCase): self.assertEqual(phenotype.pre_pub_description, "Test pre-publication") cursor.execute.assert_called_once_with( - "SELECT * FROM Phenotype WHERE id = '35' AND Owner = 'Rob'") + "SELECT * FROM Phenotype WHERE id = %s AND Owner = %s", + (35, 'Rob')) def test_diff_from_dict(self): """Test that a correct diff is generated""" -- cgit v1.2.3 From 77e7fb0d6a8090770460cc05e1707056b09e6802 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 7 Jun 2021 10:18:22 +0300 Subject: tests: test_phenotype: Rename to test_db --- tests/unit/db/test_db.py | 73 ++++++++++++++++++++++++++++++++++++++++ tests/unit/db/test_phenotypes.py | 72 --------------------------------------- 2 files changed, 73 insertions(+), 72 deletions(-) create mode 100644 tests/unit/db/test_db.py delete mode 100644 tests/unit/db/test_phenotypes.py (limited to 'tests') diff --git a/tests/unit/db/test_db.py b/tests/unit/db/test_db.py new file mode 100644 index 0000000..a3b1d71 --- /dev/null +++ b/tests/unit/db/test_db.py @@ -0,0 +1,73 @@ +"""Tests for db/phenotypes.py""" +from unittest import TestCase +from unittest import mock + +from gn3.db import fetchone +from gn3.db import update +from gn3.db import diff_from_dict +from gn3.db.phenotypes import Phenotype + + +class TestPhenotypes(TestCase): + """Test cases for fetching chromosomes""" + + def test_update_phenotype_with_no_data(self): + """Test that a phenotype is updated correctly if an empty Phenotype dataclass + is provided + + """ + db_mock = mock.MagicMock() + self.assertEqual(update( + conn=db_mock, table="Phenotype", + data=Phenotype(), where=Phenotype()), None) + + def test_update_phenotype_with_data(self): + """ + Test that a phenotype is updated correctly if some + data is provided + """ + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + type(cursor).rowcount = 1 + self.assertEqual(update( + conn=db_mock, table="Phenotype", + data=Phenotype( + pre_pub_description="Test Pre Pub", + submitter="Rob", + post_pub_description="Test Post Pub"), + where=Phenotype(id_=1, owner="Rob")), 1) + cursor.execute.assert_called_once_with( + "UPDATE Phenotype SET " + "Pre_publication_description = %s, " + "Post_publication_description = %s, " + "Submitter = %s WHERE id = %s AND Owner = %s", + ('Test Pre Pub', 'Test Post Pub', 'Rob', 1, 'Rob')) + + def test_fetch_phenotype(self): + """Test that a single phenotype is fetched properly + + """ + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + test_data = ( + 35, "Test pre-publication", "Test post-publication", + "Original description A", "cm^2", "pre-abbrev", + "post-abbrev", "LAB001", "R. W.", "R. W.", "R. W." + ) + cursor.fetchone.return_value = test_data + phenotype = fetchone(db_mock, + "Phenotype", + where=Phenotype(id_=35, owner="Rob")) + self.assertEqual(phenotype.id_, 35) + self.assertEqual(phenotype.pre_pub_description, + "Test pre-publication") + cursor.execute.assert_called_once_with( + "SELECT * FROM Phenotype WHERE id = %s AND Owner = %s", + (35, 'Rob')) + + def test_diff_from_dict(self): + """Test that a correct diff is generated""" + self.assertEqual(diff_from_dict({"id": 1, "data": "a"}, + {"id": 2, "data": "b"}), + {"id": {"old": 1, "new": 2}, + "data": {"old": "a", "new": "b"}}) diff --git a/tests/unit/db/test_phenotypes.py b/tests/unit/db/test_phenotypes.py deleted file mode 100644 index 824d186..0000000 --- a/tests/unit/db/test_phenotypes.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Tests for db/phenotypes.py""" -from unittest import TestCase -from unittest import mock - -from gn3.db import fetchone -from gn3.db import update -from gn3.db import diff_from_dict -from gn3.db.phenotypes import Phenotype - - -class TestPhenotypes(TestCase): - """Test cases for fetching chromosomes""" - def test_update_phenotype_with_no_data(self): - """Test that a phenotype is updated correctly if an empty Phenotype dataclass - is provided - - """ - db_mock = mock.MagicMock() - self.assertEqual(update( - conn=db_mock, table="Phenotype", - data=Phenotype(), where=Phenotype()), None) - - def test_update_phenotype_with_data(self): - """ - Test that a phenotype is updated correctly if some - data is provided - """ - db_mock = mock.MagicMock() - with db_mock.cursor() as cursor: - type(cursor).rowcount = 1 - self.assertEqual(update( - conn=db_mock, table="Phenotype", - data=Phenotype( - pre_pub_description="Test Pre Pub", - submitter="Rob", - post_pub_description="Test Post Pub"), - where=Phenotype(id_=1, owner="Rob")), 1) - cursor.execute.assert_called_once_with( - "UPDATE Phenotype SET " - "Pre_publication_description = %s, " - "Post_publication_description = %s, " - "Submitter = %s WHERE id = %s AND Owner = %s", - ('Test Pre Pub', 'Test Post Pub', 'Rob', 1, 'Rob')) - - def test_fetch_phenotype(self): - """Test that a single phenotype is fetched properly - - """ - db_mock = mock.MagicMock() - with db_mock.cursor() as cursor: - test_data = ( - 35, "Test pre-publication", "Test post-publication", - "Original description A", "cm^2", "pre-abbrev", - "post-abbrev", "LAB001", "R. W.", "R. W.", "R. W." - ) - cursor.fetchone.return_value = test_data - phenotype = fetchone(db_mock, - "Phenotype", - where=Phenotype(id_=35, owner="Rob")) - self.assertEqual(phenotype.id_, 35) - self.assertEqual(phenotype.pre_pub_description, - "Test pre-publication") - cursor.execute.assert_called_once_with( - "SELECT * FROM Phenotype WHERE id = %s AND Owner = %s", - (35, 'Rob')) - - def test_diff_from_dict(self): - """Test that a correct diff is generated""" - self.assertEqual(diff_from_dict({"id": 1, "data": "a"}, - {"id": 2, "data": "b"}), - {"id": {"old": 1, "new": 2}, - "data": {"old": "a", "new": "b"}}) -- cgit v1.2.3 From 92a6ed26a6990fd37b86efe4caa9f6cd2365f476 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 7 Jun 2021 10:21:33 +0300 Subject: tests: test_db: Rename TestPhenotype --- tests/unit/db/test_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/unit/db/test_db.py b/tests/unit/db/test_db.py index a3b1d71..2e38d4c 100644 --- a/tests/unit/db/test_db.py +++ b/tests/unit/db/test_db.py @@ -8,8 +8,8 @@ from gn3.db import diff_from_dict from gn3.db.phenotypes import Phenotype -class TestPhenotypes(TestCase): - """Test cases for fetching chromosomes""" +class TestCrudMethods(TestCase): + """Test cases for CRUD methods""" def test_update_phenotype_with_no_data(self): """Test that a phenotype is updated correctly if an empty Phenotype dataclass -- cgit v1.2.3 From ea90f7d8276ec9df583b94d7d00c128e63f147ef Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 7 Jun 2021 11:16:45 +0300 Subject: tests: test_db: Add a test-case for fetchall --- tests/unit/db/test_db.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'tests') diff --git a/tests/unit/db/test_db.py b/tests/unit/db/test_db.py index 2e38d4c..485678f 100644 --- a/tests/unit/db/test_db.py +++ b/tests/unit/db/test_db.py @@ -2,10 +2,12 @@ from unittest import TestCase from unittest import mock +from gn3.db import fetchall from gn3.db import fetchone from gn3.db import update from gn3.db import diff_from_dict from gn3.db.phenotypes import Phenotype +from gn3.db.metadata_audit import MetadataAudit class TestCrudMethods(TestCase): @@ -65,6 +67,35 @@ class TestCrudMethods(TestCase): "SELECT * FROM Phenotype WHERE id = %s AND Owner = %s", (35, 'Rob')) + def test_fetchall_metadataaudit(self): + """Test that multiple metadata_audit entries are fetched properly + + """ + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + test_data = ( + 35, "Rob", ('{"pages": ' + '{"old": "5099-5109", ' + '"new": "5099-5110"}, ' + '"month": {"old": "July", ' + '"new": "June"}, ' + '"year": {"old": "2001", ' + '"new": "2002"}}'), + "2021-06-04 09:01:05" + ) + cursor.fetchall.return_value = (test_data,) + metadata = list(fetchall(db_mock, + "metadata_audit", + where=MetadataAudit(dataset_id=35, + editor="Rob")))[0] + self.assertEqual(metadata.dataset_id, 35) + self.assertEqual(metadata.time_stamp, + "2021-06-04 09:01:05") + cursor.execute.assert_called_once_with( + ("SELECT * FROM metadata_audit WHERE " + "dataset_id = %s AND editor = %s"), + (35, 'Rob')) + def test_diff_from_dict(self): """Test that a correct diff is generated""" self.assertEqual(diff_from_dict({"id": 1, "data": "a"}, -- cgit v1.2.3 From bb55cf948974d85fb31d2f424f7ee94f7ab5e3d6 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 7 Jun 2021 19:44:00 +0300 Subject: Rename json_data column to json_diff_data --- gn3/db/metadata_audit.py | 2 +- sql/metadata_audit.sql | 12 ++++++------ tests/unit/db/test_audit.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) (limited to 'tests') diff --git a/gn3/db/metadata_audit.py b/gn3/db/metadata_audit.py index 8765738..9c4474d 100644 --- a/gn3/db/metadata_audit.py +++ b/gn3/db/metadata_audit.py @@ -23,6 +23,6 @@ metadata_audit_mapping = { "id_": "id", "dataset_id": "dataset_id", "editor": "editor", - "json_data": "json_data", + "json_data": "json_diff_data", "time_stamp": "time_stamp", } diff --git a/sql/metadata_audit.sql b/sql/metadata_audit.sql index 9771e74..514a2fc 100644 --- a/sql/metadata_audit.sql +++ b/sql/metadata_audit.sql @@ -20,10 +20,10 @@ -- This table stores data on diffs when editing a Published dataset's data CREATE TABLE metadata_audit ( PRIMARY KEY (id), - id INTEGER AUTO_INCREMENT NOT NULL, - dataset_id INTEGER NOT NULL, - editor VARCHAR(255) NOT NULL, - json_data VARCHAR(2048) NOT NULL, - time_stamp timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CHECK (JSON_VALID(json_data)) + id INTEGER AUTO_INCREMENT NOT NULL, + dataset_id INTEGER NOT NULL, + editor VARCHAR(255) NOT NULL, + json_diff_data VARCHAR(2048) NOT NULL, + time_stamp timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, + CHECK (JSON_VALID(json_diff_data)) ); diff --git a/tests/unit/db/test_audit.py b/tests/unit/db/test_audit.py index 1449281..7480169 100644 --- a/tests/unit/db/test_audit.py +++ b/tests/unit/db/test_audit.py @@ -24,5 +24,5 @@ class TestMetadatAudit(TestCase): json_data=json.dumps({"a": "b"}))), 1) cursor.execute.assert_called_once_with( "INSERT INTO metadata_audit (dataset_id, " - "editor, json_data) VALUES (%s, %s, %s)", + "editor, json_diff_data) VALUES (%s, %s, %s)", (35, 'Bonface', '{"a": "b"}')) -- cgit v1.2.3 From b05c273eae3f19387ab0a6afea4277afd52a9378 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 7 Jun 2021 19:44:31 +0300 Subject: db: test_db: Add extra variable in test data for "id" --- tests/unit/db/test_db.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'tests') diff --git a/tests/unit/db/test_db.py b/tests/unit/db/test_db.py index 485678f..520e781 100644 --- a/tests/unit/db/test_db.py +++ b/tests/unit/db/test_db.py @@ -74,15 +74,14 @@ class TestCrudMethods(TestCase): db_mock = mock.MagicMock() with db_mock.cursor() as cursor: test_data = ( - 35, "Rob", ('{"pages": ' - '{"old": "5099-5109", ' - '"new": "5099-5110"}, ' - '"month": {"old": "July", ' - '"new": "June"}, ' - '"year": {"old": "2001", ' - '"new": "2002"}}'), - "2021-06-04 09:01:05" - ) + 1, 35, "Rob", ('{"pages": ' + '{"old": "5099-5109", ' + '"new": "5099-5110"}, ' + '"month": {"old": "July", ' + '"new": "June"}, ' + '"year": {"old": "2001", ' + '"new": "2002"}}'), + "2021-06-04 09:01:05") cursor.fetchall.return_value = (test_data,) metadata = list(fetchall(db_mock, "metadata_audit", -- cgit v1.2.3 From f3d62f491dedbf0fe824446ced5d1f12d5837a52 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 8 Jun 2021 01:40:47 +0300 Subject: initial commit for queries perfomance tests --- tests/performance/__init__.py | 0 tests/performance/test_query.py | 119 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 tests/performance/__init__.py create mode 100644 tests/performance/test_query.py (limited to 'tests') diff --git a/tests/performance/__init__.py b/tests/performance/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/performance/test_query.py b/tests/performance/test_query.py new file mode 100644 index 0000000..c0a3210 --- /dev/null +++ b/tests/performance/test_query.py @@ -0,0 +1,119 @@ +"""module contains performance tests for queries""" + +import time +import sys + +from inspect import getmembers +from inspect import isfunction + +from functools import wraps +from gn3.db_utils import database_connector + + +def timer(func): + """time function""" + @wraps(func) + def wrapper_time(*args, **kwargs): + """time wrapper""" + start_time = time.perf_counter() + results = func(*args, **kwargs) + end_time = time.perf_counter() + run_time = end_time - start_time + print(f"the time taken is {run_time:.3f} seconds") + return results + + return wrapper_time + + +def query_executor(query, fetch_all=True): + """function to execute a query""" + conn, _ = database_connector() + + with conn: + cursor = conn.cursor() + cursor.execute(query) + + if fetch_all: + return cursor.fetchall() + return cursor.fetchone() + + +def fetch_probeset_query(dataset_name): + """contains queries for datasets""" + + query = """SELECT * from ProbeSetData + where StrainID in (4, 5, 6, 7, 8, 9, 10, 11, 12, + 14, 15, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, + 29, 30, 31, 35, 36, 37, 39, 98, 99, 100, 103, + 487, 105, 106, 110, 115,116, 117, 118, 119, + 120, 919, 147, 121, 40, 41, 124, 125, 128, 135, + 129, 130, 131, 132, 134, 138, 139, 140, 141, 142, + 144, 145, 148, 149, 920, 922, 2, 3, 1, 1100) + and id in (SELECT ProbeSetXRef.DataId + FROM (ProbeSet, ProbeSetXRef, ProbeSetFreeze) + WHERE ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id + and ProbeSetFreeze.Name = '{}' + and ProbeSet.Id = ProbeSetXRef.ProbeSetId)""".format(dataset_name) + + return query + + +@timer +def perf_simple_query(): + """initial simple query test""" + + query = """select * from ProbeSetData limit 1""" + + _results = query_executor(query) + + return {} + + +@timer +def perf_hc_m2_dataset(): + """test the default dataset HC_M2_0606_P""" + + query = fetch_probeset_query("HC_M2_0606_P") + + _results = query_executor(query) + + return {} + + +@timer +def perf_umutaffyexon_dataset(): + """largest dataset in gn""" + + query = fetch_probeset_query("UMUTAffyExon_0209_RMA") + _results = query_executor(query) + return {} + + +def fetch_perf_functions(): + """function to filter all functions strwith perf_""" + name_func_dict = {name: obj for name, obj in + getmembers(sys.modules[__name__], isfunction)if isfunction( + obj) and obj.__module__ == __name__ and name.startswith('perf_')} + + return name_func_dict + + +def fetch_cmd_args(): + """function to fetch cmd args""" + cmd_args = sys.argv[1:] + + name_func_dict = fetch_perf_functions() + + if len(cmd_args) > 0: + callables = [func_call for name, + func_call in name_func_dict.items() if name in cmd_args] + + return callables + + return list(name_func_dict.values()) + + +if __name__ == '__main__': + func_list = fetch_cmd_args() + for call_func in func_list: + call_func() -- cgit v1.2.3 From 70888fb5fb78e7d77ca19c1a015d6c0ae494c722 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 8 Jun 2021 03:00:35 +0300 Subject: add types and dataset names --- tests/performance/test_query.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) (limited to 'tests') diff --git a/tests/performance/test_query.py b/tests/performance/test_query.py index c0a3210..222e8d6 100644 --- a/tests/performance/test_query.py +++ b/tests/performance/test_query.py @@ -6,6 +6,7 @@ import sys from inspect import getmembers from inspect import isfunction +from typing import Optional from functools import wraps from gn3.db_utils import database_connector @@ -25,9 +26,12 @@ def timer(func): return wrapper_time -def query_executor(query, fetch_all=True): +def query_executor(query: str, + dataset_name: Optional[str] = "dataset_name", + fetch_all: bool = True): """function to execute a query""" conn, _ = database_connector() + print(f"Performance tests for {dataset_name}") with conn: cursor = conn.cursor() @@ -38,7 +42,7 @@ def query_executor(query, fetch_all=True): return cursor.fetchone() -def fetch_probeset_query(dataset_name): +def fetch_probeset_query(dataset_name: str): """contains queries for datasets""" query = """SELECT * from ProbeSetData @@ -75,7 +79,7 @@ def perf_hc_m2_dataset(): query = fetch_probeset_query("HC_M2_0606_P") - _results = query_executor(query) + _results = query_executor(query, "HC_M2_0606_P") return {} @@ -85,21 +89,24 @@ def perf_umutaffyexon_dataset(): """largest dataset in gn""" query = fetch_probeset_query("UMUTAffyExon_0209_RMA") - _results = query_executor(query) + _results = query_executor(query, "UMUTAffyExon_0209_RMA") return {} def fetch_perf_functions(): """function to filter all functions strwith perf_""" - name_func_dict = {name: obj for name, obj in + name_func_dict = {name: func_obj for name, func_obj in getmembers(sys.modules[__name__], isfunction)if isfunction( - obj) and obj.__module__ == __name__ and name.startswith('perf_')} + func_obj) + and func_obj.__module__ == __name__ and name.startswith('perf_')} return name_func_dict def fetch_cmd_args(): - """function to fetch cmd args""" + """function to fetch cmd args\ + for example python file.py perf_hc_m2_dataset\ + output [perf_hc_m2_dataset obj]""" cmd_args = sys.argv[1:] name_func_dict = fetch_perf_functions() @@ -115,5 +122,5 @@ def fetch_cmd_args(): if __name__ == '__main__': func_list = fetch_cmd_args() - for call_func in func_list: - call_func() + for func in func_list: + func() -- cgit v1.2.3 From c8b5a6f407c00847ee28984f7cc18bc391c45120 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 8 Jun 2021 03:48:04 +0300 Subject: pep8 formatting --- tests/performance/test_query.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/tests/performance/test_query.py b/tests/performance/test_query.py index 222e8d6..2d05d26 100644 --- a/tests/performance/test_query.py +++ b/tests/performance/test_query.py @@ -31,7 +31,7 @@ def query_executor(query: str, fetch_all: bool = True): """function to execute a query""" conn, _ = database_connector() - print(f"Performance tests for {dataset_name}") + print(f"Performance test for {dataset_name}") with conn: cursor = conn.cursor() @@ -122,5 +122,5 @@ def fetch_cmd_args(): if __name__ == '__main__': func_list = fetch_cmd_args() - for func in func_list: - func() + for func_obj in func_list: + func_obj() -- cgit v1.2.3 From e176297bbca642f5125b5b01ae07141ebc96425d Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 9 Jun 2021 20:23:58 +0300 Subject: rename perf query file --- tests/performance/test_query.py | 126 ---------------------------------------- 1 file changed, 126 deletions(-) delete mode 100644 tests/performance/test_query.py (limited to 'tests') diff --git a/tests/performance/test_query.py b/tests/performance/test_query.py deleted file mode 100644 index 2d05d26..0000000 --- a/tests/performance/test_query.py +++ /dev/null @@ -1,126 +0,0 @@ -"""module contains performance tests for queries""" - -import time -import sys - -from inspect import getmembers -from inspect import isfunction - -from typing import Optional -from functools import wraps -from gn3.db_utils import database_connector - - -def timer(func): - """time function""" - @wraps(func) - def wrapper_time(*args, **kwargs): - """time wrapper""" - start_time = time.perf_counter() - results = func(*args, **kwargs) - end_time = time.perf_counter() - run_time = end_time - start_time - print(f"the time taken is {run_time:.3f} seconds") - return results - - return wrapper_time - - -def query_executor(query: str, - dataset_name: Optional[str] = "dataset_name", - fetch_all: bool = True): - """function to execute a query""" - conn, _ = database_connector() - print(f"Performance test for {dataset_name}") - - with conn: - cursor = conn.cursor() - cursor.execute(query) - - if fetch_all: - return cursor.fetchall() - return cursor.fetchone() - - -def fetch_probeset_query(dataset_name: str): - """contains queries for datasets""" - - query = """SELECT * from ProbeSetData - where StrainID in (4, 5, 6, 7, 8, 9, 10, 11, 12, - 14, 15, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, - 29, 30, 31, 35, 36, 37, 39, 98, 99, 100, 103, - 487, 105, 106, 110, 115,116, 117, 118, 119, - 120, 919, 147, 121, 40, 41, 124, 125, 128, 135, - 129, 130, 131, 132, 134, 138, 139, 140, 141, 142, - 144, 145, 148, 149, 920, 922, 2, 3, 1, 1100) - and id in (SELECT ProbeSetXRef.DataId - FROM (ProbeSet, ProbeSetXRef, ProbeSetFreeze) - WHERE ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id - and ProbeSetFreeze.Name = '{}' - and ProbeSet.Id = ProbeSetXRef.ProbeSetId)""".format(dataset_name) - - return query - - -@timer -def perf_simple_query(): - """initial simple query test""" - - query = """select * from ProbeSetData limit 1""" - - _results = query_executor(query) - - return {} - - -@timer -def perf_hc_m2_dataset(): - """test the default dataset HC_M2_0606_P""" - - query = fetch_probeset_query("HC_M2_0606_P") - - _results = query_executor(query, "HC_M2_0606_P") - - return {} - - -@timer -def perf_umutaffyexon_dataset(): - """largest dataset in gn""" - - query = fetch_probeset_query("UMUTAffyExon_0209_RMA") - _results = query_executor(query, "UMUTAffyExon_0209_RMA") - return {} - - -def fetch_perf_functions(): - """function to filter all functions strwith perf_""" - name_func_dict = {name: func_obj for name, func_obj in - getmembers(sys.modules[__name__], isfunction)if isfunction( - func_obj) - and func_obj.__module__ == __name__ and name.startswith('perf_')} - - return name_func_dict - - -def fetch_cmd_args(): - """function to fetch cmd args\ - for example python file.py perf_hc_m2_dataset\ - output [perf_hc_m2_dataset obj]""" - cmd_args = sys.argv[1:] - - name_func_dict = fetch_perf_functions() - - if len(cmd_args) > 0: - callables = [func_call for name, - func_call in name_func_dict.items() if name in cmd_args] - - return callables - - return list(name_func_dict.values()) - - -if __name__ == '__main__': - func_list = fetch_cmd_args() - for func_obj in func_list: - func_obj() -- cgit v1.2.3 From 5d4b24f0f2bbe72c2b2f57cc0e8596399b412199 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 9 Jun 2021 20:24:36 +0300 Subject: refactor perf query functions --- tests/performance/perf_query.py | 113 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 tests/performance/perf_query.py (limited to 'tests') diff --git a/tests/performance/perf_query.py b/tests/performance/perf_query.py new file mode 100644 index 0000000..594b9ea --- /dev/null +++ b/tests/performance/perf_query.py @@ -0,0 +1,113 @@ +"""module contains performance tests for queries""" + +import time +import sys + +from inspect import getmembers +from inspect import isfunction + +from typing import Optional +from functools import wraps +from gn3.db_utils import database_connector + + +def timer(func): + """time function""" + @wraps(func) + def wrapper_time(*args, **kwargs): + """time wrapper""" + start_time = time.perf_counter() + results = func(*args, **kwargs) + end_time = time.perf_counter() + run_time = end_time - start_time + print(f"the time taken is {run_time:.3f} seconds") + return results + + return wrapper_time + + +def query_executor(query: str, + dataset_name: Optional[str] = "dataset_name", + fetch_all: bool = True): + """function to execute a query""" + conn, _ = database_connector() + + with conn: + cursor = conn.cursor() + cursor.execute(query) + + if fetch_all: + return cursor.fetchall() + return cursor.fetchone() + + +def fetch_probeset_query(dataset_name: str): + """contains queries for datasets""" + + query = """SELECT * from ProbeSetData + where StrainID in (4, 5, 6, 7, 8, 9, 10, 11, 12, + 14, 15, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, + 29, 30, 31, 35, 36, 37, 39, 98, 99, 100, 103, + 487, 105, 106, 110, 115,116, 117, 118, 119, + 120, 919, 147, 121, 40, 41, 124, 125, 128, 135, + 129, 130, 131, 132, 134, 138, 139, 140, 141, 142, + 144, 145, 148, 149, 920, 922, 2, 3, 1, 1100) + and id in (SELECT ProbeSetXRef.DataId + FROM (ProbeSet, ProbeSetXRef, ProbeSetFreeze) + WHERE ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id + and ProbeSetFreeze.Name = '{}' + and ProbeSet.Id = ProbeSetXRef.ProbeSetId)""".format(dataset_name) + + return query + + +@timer +def perf_hc_m2_dataset(): + """test the default dataset HC_M2_0606_P""" + + dataset_name = "HC_M2_0606_P" + print(f"Performance test for {dataset_name}") + + query_executor(fetch_probeset_query(dataset_name=dataset_name), + dataset_name=dataset_name) + + +@timer +def perf_umutaffyexon_dataset(): + """largest dataset in gn""" + + dataset_name = "UMUTAffyExon_0209_RMA" + print(f"Performance test for {dataset_name}") + query_executor(fetch_probeset_query(dataset_name=dataset_name), + dataset_name=dataset_name) + + +def fetch_perf_functions(): + """function to filter all functions strwith perf_""" + name_func_dict = {name: func_obj for name, func_obj in + getmembers(sys.modules[__name__], isfunction)if isfunction( + func_obj) + and func_obj.__module__ == __name__ and name.startswith('perf_')} + + return name_func_dict + + +def fetch_cmd_args(): + """function to fetch cmd args""" + cmd_args = sys.argv[1:] + + name_func_dict = fetch_perf_functions() + + if len(cmd_args) > 0: + callables = [func_call for name, + func_call in name_func_dict.items() if name in cmd_args] + + return callables + + return list(name_func_dict.values()) + + +if __name__ == '__main__': + func_list = fetch_cmd_args() + for func_obj in func_list: + func_obj() -- cgit v1.2.3 From 78e90b41c0577f724121ae2156b9c1759d30812d Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 9 Jun 2021 20:27:20 +0300 Subject: remove ununsed variables --- tests/performance/perf_query.py | 1 - 1 file changed, 1 deletion(-) (limited to 'tests') diff --git a/tests/performance/perf_query.py b/tests/performance/perf_query.py index 594b9ea..93bb3b2 100644 --- a/tests/performance/perf_query.py +++ b/tests/performance/perf_query.py @@ -27,7 +27,6 @@ def timer(func): def query_executor(query: str, - dataset_name: Optional[str] = "dataset_name", fetch_all: bool = True): """function to execute a query""" conn, _ = database_connector() -- cgit v1.2.3 From 14be11dea2844750eee5029197f82d9e0b67123e Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 9 Jun 2021 20:31:43 +0300 Subject: minor fixes --- tests/performance/perf_query.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'tests') diff --git a/tests/performance/perf_query.py b/tests/performance/perf_query.py index 93bb3b2..12cb944 100644 --- a/tests/performance/perf_query.py +++ b/tests/performance/perf_query.py @@ -6,7 +6,6 @@ import sys from inspect import getmembers from inspect import isfunction -from typing import Optional from functools import wraps from gn3.db_utils import database_connector @@ -67,8 +66,7 @@ def perf_hc_m2_dataset(): dataset_name = "HC_M2_0606_P" print(f"Performance test for {dataset_name}") - query_executor(fetch_probeset_query(dataset_name=dataset_name), - dataset_name=dataset_name) + query_executor(fetch_probeset_query(dataset_name=dataset_name)) @timer @@ -77,8 +75,7 @@ def perf_umutaffyexon_dataset(): dataset_name = "UMUTAffyExon_0209_RMA" print(f"Performance test for {dataset_name}") - query_executor(fetch_probeset_query(dataset_name=dataset_name), - dataset_name=dataset_name) + query_executor(fetch_probeset_query(dataset_name=dataset_name)) def fetch_perf_functions(): -- cgit v1.2.3 From 12089cb2eae2201d3b348d21dd0e93a61e16e7c7 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 18 Jun 2021 21:27:38 +0000 Subject: Change test_compose_rqtl_command in test_commands.py to have different arguments to hopefully avoid github's pylint 'duplicate code' complaint Changed some parameters in test_compose_rqtl_cmd to avoid pylint complaining about duplicate code --- tests/unit/test_commands.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) (limited to 'tests') diff --git a/tests/unit/test_commands.py b/tests/unit/test_commands.py index a3d0273..aaefb23 100644 --- a/tests/unit/test_commands.py +++ b/tests/unit/test_commands.py @@ -59,24 +59,23 @@ class TestCommands(unittest.TestCase): self.assertEqual( compose_rqtl_cmd(rqtl_wrapper_cmd="rqtl-wrapper", rqtl_wrapper_kwargs={ - "g": "genofile", - "p": "phenofile", - "model": "normal", - "method": "hk", - "nperm": 1000, + "g": "the_genofile", + "p": "the_phenofile", + "model": "np", + "method": "ehk", + "nperm": 2000, "scale": "Mb", - "control": "rs123456" + "control": "rs234567" }, rqtl_wrapper_bool_kwargs=[ - "addcovar", - "interval" + "addcovar" ]), ("rqtl-wrapper " - "--g genofile --p phenofile " - "--model normal --method hk " - "--nperm 1000 --scale Mb " - "--control rs123456 " - "--addcovar --interval") + "--g the_genofile --p the_phenofile " + "--model np --method ehk " + "--nperm 2000 --scale Mb " + "--control rs234567 " + "--addcovar") ) def test_queue_cmd_exception_raised_when_redis_is_down(self): -- cgit v1.2.3 From efe73f2f142bd2a12b89ad638c9387cf9c6ded18 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 18 Jun 2021 22:01:21 +0000 Subject: Fixed test_compose_rqtl_command test; forgot to include Rscript in the command --- tests/unit/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/unit/test_commands.py b/tests/unit/test_commands.py index aaefb23..f36ba55 100644 --- a/tests/unit/test_commands.py +++ b/tests/unit/test_commands.py @@ -70,7 +70,7 @@ class TestCommands(unittest.TestCase): rqtl_wrapper_bool_kwargs=[ "addcovar" ]), - ("rqtl-wrapper " + ("Rscript rqtl-wrapper " "--g the_genofile --p the_phenofile " "--model np --method ehk " "--nperm 2000 --scale Mb " -- cgit v1.2.3 From d8c4b1a0188574cdd70638a99e2984b54bec6033 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 18 Jun 2021 22:16:26 +0000 Subject: Fixed test_rqtl.py to include Rscript in the command --- tests/unit/computations/test_rqtl.py | 82 ++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 41 deletions(-) (limited to 'tests') diff --git a/tests/unit/computations/test_rqtl.py b/tests/unit/computations/test_rqtl.py index b16f136..3bf8f81 100644 --- a/tests/unit/computations/test_rqtl.py +++ b/tests/unit/computations/test_rqtl.py @@ -1,41 +1,41 @@ -"""Test cases for procedures defined in computations.rqtl""" -import unittest - -from unittest import mock -from gn3.computations.rqtl import generate_rqtl_cmd - -class TestRqtl(unittest.TestCase): - """Test cases for computations.rqtl module""" - @mock.patch("gn3.computations.rqtl.generate_hash_of_string") - @mock.patch("gn3.computations.rqtl.get_hash_of_files") - def test_generate_rqtl_command(self, mock_get_hash_files, mock_generate_hash_string): - """Test computing mapping results with R/qtl""" - mock_get_hash_files.return_value = "my-hash1" - mock_generate_hash_string.return_value = "my-hash2" - - self.assertEqual( - generate_rqtl_cmd(rqtl_wrapper_cmd="rqtl-wrapper", - rqtl_wrapper_kwargs={ - "g": "genofile", - "p": "phenofile", - "model": "normal", - "method": "hk", - "nperm": 1000, - "scale": "Mb", - "control": "rs123456" - }, - rqtl_wrapper_bool_kwargs=[ - "addcovar", - "interval" - ]), { - "output_file": - "my-hash1my-hash2my-hash2-output.json", - "rqtl_cmd": ( - "rqtl-wrapper " - "--g genofile --p phenofile " - "--model normal --method hk " - "--nperm 1000 --scale Mb " - "--control rs123456 " - "--addcovar --interval" - ) - }) +"""Test cases for procedures defined in computations.rqtl""" +import unittest + +from unittest import mock +from gn3.computations.rqtl import generate_rqtl_cmd + +class TestRqtl(unittest.TestCase): + """Test cases for computations.rqtl module""" + @mock.patch("gn3.computations.rqtl.generate_hash_of_string") + @mock.patch("gn3.computations.rqtl.get_hash_of_files") + def test_generate_rqtl_command(self, mock_get_hash_files, mock_generate_hash_string): + """Test computing mapping results with R/qtl""" + mock_get_hash_files.return_value = "my-hash1" + mock_generate_hash_string.return_value = "my-hash2" + + self.assertEqual( + generate_rqtl_cmd(rqtl_wrapper_cmd="rqtl-wrapper", + rqtl_wrapper_kwargs={ + "g": "genofile", + "p": "phenofile", + "model": "normal", + "method": "hk", + "nperm": 1000, + "scale": "Mb", + "control": "rs123456" + }, + rqtl_wrapper_bool_kwargs=[ + "addcovar", + "interval" + ]), { + "output_file": + "my-hash1my-hash2my-hash2-output.json", + "rqtl_cmd": ( + "Rscript rqtl-wrapper " + "--g genofile --p phenofile " + "--model normal --method hk " + "--nperm 1000 --scale Mb " + "--control rs123456 " + "--addcovar --interval" + ) + }) -- cgit v1.2.3 From 9dca7551ff6c1fa8c3b26ded2c1b63cfab001eca Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 18 Jun 2021 22:21:46 +0000 Subject: Fixed file type from json to csv for test_generate_rqtl_command --- tests/unit/computations/test_rqtl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/unit/computations/test_rqtl.py b/tests/unit/computations/test_rqtl.py index 3bf8f81..09790b7 100644 --- a/tests/unit/computations/test_rqtl.py +++ b/tests/unit/computations/test_rqtl.py @@ -29,7 +29,7 @@ class TestRqtl(unittest.TestCase): "interval" ]), { "output_file": - "my-hash1my-hash2my-hash2-output.json", + "my-hash1my-hash2my-hash2-output.csv", "rqtl_cmd": ( "Rscript rqtl-wrapper " "--g genofile --p phenofile " -- cgit v1.2.3 From f7becfa11ca857104ecc1b668b4bd3d0a721083c Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 18 Jun 2021 22:28:37 +0000 Subject: Fixed another error where test_generate_rqtl_command didn't include the filename argument (not sure why running unit tests locally doesn't detect this) --- tests/unit/computations/test_rqtl.py | 1 + 1 file changed, 1 insertion(+) (limited to 'tests') diff --git a/tests/unit/computations/test_rqtl.py b/tests/unit/computations/test_rqtl.py index 09790b7..955d0ab 100644 --- a/tests/unit/computations/test_rqtl.py +++ b/tests/unit/computations/test_rqtl.py @@ -36,6 +36,7 @@ class TestRqtl(unittest.TestCase): "--model normal --method hk " "--nperm 1000 --scale Mb " "--control rs123456 " + "--filename my-hash1my-hash2my-hash2-output.csv " "--addcovar --interval" ) }) -- cgit v1.2.3