about summary refs log tree commit diff
path: root/aisearch/scripts/rdf2partial_text.py
diff options
context:
space:
mode:
authorJohannes Medagbe <johanmedagbe@gmail.com>2026-09-21 10:03:20 +0000
committerJohannes Medagbe <johanmedagbe@gmail.com>2026-09-22 12:18:32 +0000
commitdce53892085c02c8571d8a6495e39c449294aa5f (patch)
treeae64d32cc845d30ec2a4cc3514a23dc5683a09a1 /aisearch/scripts/rdf2partial_text.py
downloadgenenetwork_ai_search-master.tar.gz
New: commit aisearch repo master
Diffstat (limited to 'aisearch/scripts/rdf2partial_text.py')
-rw-r--r--aisearch/scripts/rdf2partial_text.py70
1 files changed, 70 insertions, 0 deletions
diff --git a/aisearch/scripts/rdf2partial_text.py b/aisearch/scripts/rdf2partial_text.py
new file mode 100644
index 0000000..632b5ea
--- /dev/null
+++ b/aisearch/scripts/rdf2partial_text.py
@@ -0,0 +1,70 @@
+import json
+import os
+import string
+from copy import copy
+
+dir = os.getenv("DIR")
+if dir is None:
+    raise ValueError("DIR not specified")
+
+collection = {}
+
+files = [os.path.join(dir, file) for file in os.listdir(dir) if "ttl" in file]
+
+
+def clean(text) -> str:
+    return text.strip(string.punctuation).strip()
+
+
+for file in files:
+    with open(file) as f:
+        lines = f.readlines()
+        mem = ""
+        for ind, line in enumerate(lines):
+            if (not line.startswith("@")) and len(line) > 1:
+                if line.startswith("<"):
+                    line = line.strip()
+                    contents = line.split(" ")
+                    key = (
+                        clean(contents[0].split("/")[-1])
+                        if "/" in contents[0]
+                        else clean(contents[0].split(":")[-1])
+                    )
+                    mem = copy(key)
+                    value = " ".join(
+                        (
+                            clean(content.split("/")[-1])
+                            if "/" in content
+                            else clean(content.split(":")[-1])
+                        )
+                        for content in contents[1:]
+                    )
+                    key = clean(key)
+                    value = clean(value)
+                    if key not in collection:
+                        collection[key] = [value]
+                    else:
+                        if value not in collection[key]:
+                            collection[key].append(value)
+                else:
+                    line = line.strip()
+                    key = copy(mem)
+                    contents = line.split(" ")
+                    value = " ".join(
+                        (
+                            clean(content.split("/")[-1])
+                            if "/" in content
+                            else clean(content.split(":")[-1])
+                        )
+                        for content in contents
+                    )
+                    key = clean(key)
+                    value = clean(value)
+                    if key not in collection:
+                        collection[key] = [value]
+                    else:
+                        if value not in collection[key]:
+                            collection[key].append(value)
+
+with open(f"{dir}aggr_rdf.txt", "w") as new:
+    new.write(json.dumps(collection))