about summary refs log tree commit diff
path: root/aisearch/scripts/new_rdf2partial_text.py
diff options
context:
space:
mode:
Diffstat (limited to 'aisearch/scripts/new_rdf2partial_text.py')
-rw-r--r--aisearch/scripts/new_rdf2partial_text.py34
1 files changed, 34 insertions, 0 deletions
diff --git a/aisearch/scripts/new_rdf2partial_text.py b/aisearch/scripts/new_rdf2partial_text.py
new file mode 100644
index 0000000..29b99f8
--- /dev/null
+++ b/aisearch/scripts/new_rdf2partial_text.py
@@ -0,0 +1,34 @@
+import json
+import os
+import string
+
+dir = os.getenv("DIR")
+if dir is None:
+    raise ValueError("DIR not specified")
+
+collection = {}
+
+files = [os.path.join(dir, file) for file in os.listdir(dir) if "ttl" in file]
+
+
+def clean(text) -> str:
+    return text.strip(string.punctuation).strip()
+
+
+for file in files:
+    with open(file) as f:
+        lines = f.readlines()
+        for ind, line in enumerate(lines):
+            line = line.strip()
+            if len(line) > 1:
+                contents = line.split(" ")
+                key = clean(contents[0])
+                value = " ".join(clean(content) for content in contents[1:])
+                if key not in collection:
+                    collection[key] = [value]
+                else:
+                    if value not in collection[key]:
+                        collection[key].append(value)
+
+with open(f"{dir}aggr_rdf.txt", "w") as new:
+    new.write(json.dumps(collection))