read to generate datasets from todolist file

1 files changed, 14 insertions, 3 deletions

diff --git a/wqflask/scripts/textfiles_generator.py b/wqflask/scripts/textfiles_generator.py
index 59bbdb29..b7c94eb0 100644
--- a/wqflask/scripts/textfiles_generator.py
+++ b/wqflask/scripts/textfiles_generator.py
@@ -253,8 +253,7 @@ def argument_parser():
 
 def run_textfiles_generator(args):
     try:
-
-        for (d_type, dataset_name, _species) in DATASET_NAMES:
+        for (d_type, dataset_name, _species) in fetch_to_generate_dataset("ProbeSet", "textfile"):
             file_name = __generate_file_name__(dataset_name, args.SQL_URI)
             if not check_file_expiry(os.path.join(
                     args.TMPDIR, "Probesets"), file_name):
@@ -266,7 +265,7 @@ def run_textfiles_generator(args):
 
 
 def run_metadata_files_generator(args):
-    for (dataset_type, dataset_name, species) in DATASET_NAMES:
+    for (dataset_type, dataset_name, species) in fetch_to_generate_dataset("ProbeSet", "metadata"):
         try:
             if not check_file_expiry(os.path.join(TMPDIR, f"metadata_{dataset_type}"), dataset_name):
                 return
@@ -305,6 +304,18 @@ def check_file_expiry(target_file_path, dataset_name, max_days=20):
         return True
 
 
+def fetch_to_generate_dataset(dataset_type, gen_type):
+    try:
+        with lmdb.open(os.path.join("/tmp", "todolist_generate"), readonly=True, lock=False) as env:
+            with env.begin() as txn:
+                data = txn.get(f"{gen_type}:{dataset_type}".encode())
+                if data:
+                    return [result for result in pickle.loads(data).values()]
+                return DATASET_NAMES
+    except Exception as err:
+        return DATASET_NAMES
+
+
 if __name__ == '__main__':
     args = argument_parser()
     TMPDIR = args.TMPDIR