diff options
author | Frederick Muriuki Muriithi | 2025-06-03 07:00:40 -0500 |
---|---|---|
committer | Frederick Muriuki Muriithi | 2025-06-03 07:29:31 -0500 |
commit | 70de586b590e824d7bcc3777830fda323a0e7983 (patch) | |
tree | 6a7bf06ff0e902da2e28bc43413d209d8a3aa09e /uploader/phenotypes/models.py | |
parent | 733d7e89612a7324856611030c0e8faa0b51538c (diff) | |
download | gn-uploader-70de586b590e824d7bcc3777830fda323a0e7983.tar.gz |
Memory saving: Don't reload saved data
We already have all the data in memory, so we do not need to reload it
and keep yet another copy.
Diffstat (limited to 'uploader/phenotypes/models.py')
-rw-r--r-- | uploader/phenotypes/models.py | 18 |
1 files changed, 3 insertions, 15 deletions
diff --git a/uploader/phenotypes/models.py b/uploader/phenotypes/models.py index f4d3529..7861427 100644 --- a/uploader/phenotypes/models.py +++ b/uploader/phenotypes/models.py @@ -324,7 +324,7 @@ def save_phenotypes_data( conn: mdb.Connection, table: str, data: Iterable[dict] -) -> tuple[dict, ...]: +) -> int: """Save new phenotypes data into the database.""" _table_details = { "PublishData": { @@ -334,7 +334,6 @@ def save_phenotypes_data( "NStrain": { "table": "PublishData", "valueCol": "count", "DataIdCol": "DataId"} }[table] - saved_data = tuple() with conn.cursor(cursorclass=DictCursor) as cursor: _count = 0 while True: @@ -348,21 +347,10 @@ def save_phenotypes_data( (f"INSERT INTO {_table_details['table']}" f"({_table_details['DataIdCol']}, StrainId, {_table_details['valueCol']}) " "VALUES " - f"(%(data_id)s, %(sample_id)s, %({_table_details['valueCol']})s) " - "RETURNING *"), + f"(%(data_id)s, %(sample_id)s, %({_table_details['valueCol']})s) "), tuple(batch)) - - paramstr = ", ".join(["(%s, %s)"] * len(batch)) - cursor.execute(f"SELECT * FROM {_table_details['table']} " - f"WHERE ({_table_details['DataIdCol']}, StrainId) " - f"IN ({paramstr})", - tuple(single for items in - ((item["data_id"], item["sample_id"]) - for item in batch) - for single in items)) - saved_data = saved_data + tuple(dict(row) for row in cursor.fetchall()) _count = _count + len(batch) logger.debug("Saved a total of %s data rows", _count) - return saved_data + return _count |