Save numeric/computational data for phenotypes.

1 files changed, 139 insertions, 15 deletions

diff --git a/scripts/load_phenotypes_to_db.py b/scripts/load_phenotypes_to_db.py
index 82324fc..4d90291 100644
--- a/scripts/load_phenotypes_to_db.py
+++ b/scripts/load_phenotypes_to_db.py
@@ -10,6 +10,7 @@ from MySQLdb.cursors import Cursor, DictCursor
 from gn_libs import jobs, mysqldb, sqlite3
 
 from r_qtl import r_qtl2 as rqtl2
+from uploader.phenotypes.modes import save_phenotypes_data
 from uploader.publications.models import create_new_publications
 from uploader.samples.models import samples_by_species_and_population
 
@@ -63,9 +64,9 @@ def save_phenotypes(
     if control_data["phenocovar_transposed"]:
         logger.info("Undoing transposition of the files rows and columns.")
         phenofiles = (
-            transpose_csv_with_rename(
-                _file
-                build_line_splitter(control_data)
+            rqtl2.transpose_csv_with_rename(
+                _file,
+                build_line_splitter(control_data),
                 build_line_joiner(control_data))
             for _file in control_data["phenocovar"])
 
@@ -89,19 +90,145 @@ def __fetch_next_dataid__(conn: mysqldb.Connection) -> int:
         return int(cursor.fetchone()) + 1
 
 
-def save_phenotypes_data(conn: mysqldb.Connection, dataidmap, samples, datafiles):
+def save_pheno_data(
+        conn: mysqldb.Connection,
+        dataidmap: dict,
+        samples: tuple[dict, ...],
+        control_data: dict
+):
     """Read the `datafiles` and save the data in the database."""
-    pass
+    phenofiles = control_data["pheno"]
+    if len(phenofiles) <= 0:
+        return tuple()
+
+    if control_data["pheno_transposed"]:
+        logger.info("Undoing transposition of the files rows and columns.")
+        phenofiles = (
+            rqtl2.transpose_csv_with_rename(
+                _file,
+                build_line_splitter(control_data),
+                build_line_joiner(control_data))
+            for _file in control_data["pheno"])
+
+    _headers = rqtl2.read_csv_file_headers(control_data["pheno"][0],
+                                           control_data["pheno_transposed"],
+                                           control_data["sep"],
+                                           control_data["comment.char"])
 
+    def __row_to_data_items__(row):
+        return tuple(
+            {
+                "phenotype_id": row["phenotype_id"],
+                "data_id": dataidmap[row["phenotype_id"]]["data_id"],
+                "sample_name": samplename,
+                "sample_id": samples[samplename]["Id"],
+                "value": value
+            }
+            for samplename, value in row.items()
+            if samplename in samples.keys())
+
+    return save_phenotypes_data(
+        conn,
+        "PublishData"
+        (item for item in
+         (row_to_dataitems(dict(zip(_headers, line))) for filecontent
+          in (rqtl2.read_csv_file(path) for path in phenofiles)
+         for idx, line in enumerate(filecontent)
+         if idx != 0)))
 
-def save_phenotype_se(conn: mysqldb.Connection, dataidmap, samples, sefiles):
+
+def save_phenotype_se(
+        conn: mysqldb.Connection,
+        dataidmap: dict,
+        samples: tuple[dict, ...],
+        control_data: dict
+):
     """Read the `sefiles` and save the data in the database."""
-    pass
+    sefiles = control_data["phenose"]
+    if len(sefiles) <= 0:
+        return tuple()
 
+    if control_data["phenose_transposed"]:
+        logger.info("Undoing transposition of the files rows and columns.")
+        sefiles = (
+            rqtl2.transpose_csv_with_rename(
+                _file,
+                build_line_splitter(control_data),
+                build_line_joiner(control_data))
+            for _file in control_data["phenose"])
 
-def save_phenotype_n(conn: mysqldb.Connection, dataidmap, samples, nfiles):
+    _headers = rqtl2.read_csv_file_headers(control_data["phenose"][0],
+                                           control_data["phenose_transposed"],
+                                           control_data["sep"],
+                                           control_data["comment.char"])
+
+    def __row_to_data_items__(row):
+        return tuple(
+            {
+                "phenotype_id": row["phenotype_id"],
+                "data_id": dataidmap[row["phenotype_id"]]["data_id"],
+                "sample_name": samplename,
+                "sample_id": samples[samplename]["Id"],
+                "error": value
+            }
+            for samplename, value in row.items()
+            if samplename in samples.keys())
+
+    return save_phenotypes_data(
+        conn,
+        "PublishSE"
+        (item for item in
+         (row_to_dataitems(dict(zip(_headers, line))) for filecontent
+          in (rqtl2.read_csv_file(path) for path in sefiles)
+         for idx, line in enumerate(filecontent)
+         if idx != 0)))
+
+
+def save_phenotype_n(
+        conn: mysqldb.Connection,
+        dataidmap: dict,
+        samples: tuple[dict, ...],
+        control_data: dict
+):
     """Read the `nfiles` and save the data in the database."""
-    pass
+    sefiles = control_data["phenonum"]
+    if len(sefiles) <= 0:
+        return tuple()
+
+    if control_data["phenonum_transposed"]:
+        logger.info("Undoing transposition of the files rows and columns.")
+        sefiles = (
+            rqtl2.transpose_csv_with_rename(
+                _file,
+                build_line_splitter(control_data),
+                build_line_joiner(control_data))
+            for _file in control_data["phenonum"])
+
+    _headers = rqtl2.read_csv_file_headers(control_data["phenonum"][0],
+                                           control_data["phenonum_transposed"],
+                                           control_data["sep"],
+                                           control_data["comment.char"])
+
+    def __row_to_data_items__(row):
+        return tuple(
+            {
+                "phenotype_id": row["phenotype_id"],
+                "data_id": dataidmap[row["phenotype_id"]]["data_id"],
+                "sample_name": samplename,
+                "sample_id": samples[samplename]["Id"],
+                "count": value
+            }
+            for samplename, value in row.items()
+            if samplename in samples.keys())
+
+    return save_phenotypes_data(
+        conn,
+        "NStrain"
+        (item for item in
+         (row_to_dataitems(dict(zip(_headers, line))) for filecontent
+          in (rqtl2.read_csv_file(path) for path in sefiles)
+         for idx, line in enumerate(filecontent)
+         if idx != 0)))
 
 
 def cross_reference_phenotypes_publications_and_data(
@@ -145,17 +272,14 @@ def load_data(conn, job):
         for row in samples_by_species_and_population(
                 conn, species["SpeciesId"], population["PopulationId"])}
     #    b. Save all the data items (DataIds are vibes), return new IDs
-    data = save_phenotypes_data(
-        cursor, dataidmap, samples, , _control_data["pheno"])
+    data = save_pheno_data(conn, dataidmap, samples, _control_data)
     # 4. Cross-reference Phenotype, Publication, and PublishData in PublishXRef
     xrefs = cross_reference_phenotypes_publications_and_data(
         cursor, __merge_map_with_publications__(dataidmap))
     # 5. If standard errors and N exist, save them too
     #    (use IDs returned in `3. b.` above).
-    data_se = save_phenotypes_data(
-        cursor, dataidmap, samples, , _control_data["phenose"])
-    data_n = save_phenotypes_n(
-        cursor, dataidmap, samples, , _control_data["phenonum"])
+    data_se = save_phenotypes_se(conn, dataidmap, samples, _control_data)
+    data_n = save_phenotypes_n(conn, dataidmap, samples, _control_data)
     # 6. If entirely new data, update authorisations (break this down)
     update_auth(_user, _species, _population, _dataset, _phenos)
     return 0