diff options
Diffstat (limited to 'aisearch/scripts/new_rdf2partial_text.py')
| -rw-r--r-- | aisearch/scripts/new_rdf2partial_text.py | 34 |
1 files changed, 34 insertions, 0 deletions
diff --git a/aisearch/scripts/new_rdf2partial_text.py b/aisearch/scripts/new_rdf2partial_text.py new file mode 100644 index 0000000..29b99f8 --- /dev/null +++ b/aisearch/scripts/new_rdf2partial_text.py @@ -0,0 +1,34 @@ +import json +import os +import string + +dir = os.getenv("DIR") +if dir is None: + raise ValueError("DIR not specified") + +collection = {} + +files = [os.path.join(dir, file) for file in os.listdir(dir) if "ttl" in file] + + +def clean(text) -> str: + return text.strip(string.punctuation).strip() + + +for file in files: + with open(file) as f: + lines = f.readlines() + for ind, line in enumerate(lines): + line = line.strip() + if len(line) > 1: + contents = line.split(" ") + key = clean(contents[0]) + value = " ".join(clean(content) for content in contents[1:]) + if key not in collection: + collection[key] = [value] + else: + if value not in collection[key]: + collection[key].append(value) + +with open(f"{dir}aggr_rdf.txt", "w") as new: + new.write(json.dumps(collection)) |
