From 7f2a1f9d1b91b9d0d24da56afcfe26197a434af3 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 16 Aug 2021 11:13:40 +0300 Subject: Add "Description" column to CaseAttribute See: eecaad2d --- scripts/add_missing_columns.sh | 3 +++ 1 file changed, 3 insertions(+) (limited to 'scripts') diff --git a/scripts/add_missing_columns.sh b/scripts/add_missing_columns.sh index 70d5fdeb..611e2dd6 100644 --- a/scripts/add_missing_columns.sh +++ b/scripts/add_missing_columns.sh @@ -13,6 +13,9 @@ ALTER TABLE PublishXRef ADD mean double AFTER DataId; + ALTER TABLE CaseAttribute + ADD Description varchar(255) AFTER Name; + -- This takes some time ALTER TABLE ProbeSet ADD UniProtID varchar(20) AFTER ProteinName; -- cgit v1.2.3 From 6e0ea75ca427721aed0a5f394b501b2cde9bf769 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 23 Sep 2021 13:17:55 +0300 Subject: Add script for creating/ updating groups during authorisation --- scripts/authentication/group.py | 130 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 scripts/authentication/group.py (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py new file mode 100644 index 00000000..b89bc3ec --- /dev/null +++ b/scripts/authentication/group.py @@ -0,0 +1,130 @@ +"""A script for adding users to a specific group. + +Example: + +Assuming there are no groups and 'test@bonfacemunyoki.com' does not +exist in Redis: + +.. code-block:: bash + python group.py -g "editors" -m "test@bonfacemunyoki.com" + +results in:: + + Successfully created the group: 'editors' + Data: '{"admins": [], "members": []}' + +If 'me@bonfacemunyoki.com' exists in 'users' in Redis and we run: + +.. code-block:: bash + python group.py -g "editors" -m "me@bonfacemunyoki.com" + +now results in:: + + No new group was created. + Updated Data: {'admins': [], 'members': ['me@bonfacemunyoki.com']} + +""" + +import argparse +import redis +import json + +from typing import Dict, List, Optional, Set +from glom import glom # type: ignore + + +def create_group_data(users: Dict, target_group: str, + members: Optional[str] = None, + admins: Optional[str] = None) -> Dict: + """Return a dictionary that contains the following keys: "key", + "field", and "value" that can be used in a redis hash as follows: + HSET key field value + + Parameters: + + - `users`: a list of users for example: + + {'8ad942fe-490d-453e-bd37-56f252e41603': + '{"email_address": "me@test.com", + "full_name": "John Doe", + "organization": "Genenetwork", + "password": {"algorithm": "pbkdf2", + "hashfunc": "sha256", + "salt": "gJrd1HnPSSCmzB5veMPaVk2ozzDlS1Z7Ggcyl1+pciA=", + "iterations": 100000, "keylength": 32, + "created_timestamp": "2021-09-22T11:32:44.971912", + "password": "edcdaa60e84526c6"}, + "user_id": "8ad942fe", "confirmed": 1, + "registration_info": { + "timestamp": "2021-09-22T11:32:45.028833", + "ip_address": "127.0.0.1", + "user_agent": "Mozilla/5.0"}}'} + + - `target_group`: the group name that will be stored inside the + "groups" hash in Redis. + + - `members`: a comma-separated list of values that contain members + of the `target_group` e.g. "me@test1.com, me@test2.com, + me@test3.com" + + - `admins`: a comma-separated list of values that contain + administrators of the `target_group` e.g. "me@test1.com, + me@test2.com, me@test3.com" + + """ + _members = "".join(members.split()).split(",") if members else [] + _admins: List = "".join(admins.split()).split(",") if admins else [] + + user_emails: Set = {glom(json.loads(user_details), "email_address") + for _, user_details in users.items()} + + return {"key": "groups", + "field": target_group, + "value": json.dumps({ + "admins": [admin for admin in _admins + if admin in user_emails], + "members": [member for member in _members + if member in user_emails] + })} + + +if __name__ == "__main__": + # Initialising the parser CLI arguments + parser = argparse.ArgumentParser() + parser.add_argument("-g", "--group-name", + help="This is the name of the GROUP mask") + parser.add_argument("-m", "--members", + help="Members of the GROUP mask") + parser.add_argument("-a", "--admins", + help="Admins of the GROUP mask") + args = parser.parse_args() + + if not args.group_name: + exit("\nExiting. Please specify a group name to use!\n") + + members = args.members if args.members else None + admins = args.admins if args.admins else None + REDIS_CONN = redis.Redis() + USERS = {key.decode(): val.decode() + for key, val in REDIS_CONN.hgetall("users").items()} + + if not any([members, admins]): + exit("\nExiting. Please provide a value for " + "MEMBERS(-m) or ADMINS(-a)!\n") + + data = create_group_data( + users=USERS, + target_group=args.group_name, + members=members, + admins=admins) + created_p = REDIS_CONN.hset(data.get("key", ""), + data.get("field", ""), + data.get("value", "")) + + groups = json.loads(REDIS_CONN.hget("groups", + args.group_name)) # type: ignore + if created_p: + exit(f"\nSuccessfully created the group: '{args.group_name}'\n" + f"Data: {groups}\n") + exit("\nNo new group was created.\n" + f"Updated Data: {groups}\n") -- cgit v1.2.3 From c585945c5516092b362efecc16325ad9ecc54291 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 23 Sep 2021 15:39:32 +0300 Subject: Add script that adds "editors" group to all resources in Redis --- scripts/authentication/resource.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 scripts/authentication/resource.py (limited to 'scripts') diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py new file mode 100644 index 00000000..75ef9e93 --- /dev/null +++ b/scripts/authentication/resource.py @@ -0,0 +1,25 @@ +"""A script that adds the group: 'editors' to every +resource. 'editors' should have the right to edit both metadata and +data. + +To use this script, simply run: + +.. code-block:: python + python resource.py + +""" +import json +import redis + + +if __name__ == "__main__": + REDIS_CONN = redis.Redis() + resources = REDIS_CONN.hgetall("resources_clone") + for resource_id, resource in resources.items(): + deserialized_resource = json.loads(resource) + deserialized_resource["group_masks"] = { + "editors": {"metadata": "edit", + "data": "edit"}} + REDIS_CONN.hset("resources_clone", + resource_id, + json.dumps(deserialized_resource)) -- cgit v1.2.3 From df487791c91a5aa1a9a3b4e1a6c9ce17a58eafe6 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 4 Oct 2021 12:21:16 +0300 Subject: Modify resource editing script to enable data backups & restoration --- scripts/authentication/resource.py | 101 ++++++++++++++++++++++++++++++++----- 1 file changed, 88 insertions(+), 13 deletions(-) (limited to 'scripts') diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py index 75ef9e93..8fcf09d7 100644 --- a/scripts/authentication/resource.py +++ b/scripts/authentication/resource.py @@ -1,25 +1,100 @@ -"""A script that adds the group: 'editors' to every -resource. 'editors' should have the right to edit both metadata and -data. +"""A script that: -To use this script, simply run: +- Optionally restores data from a json file. + +- By default, without any args provided, adds the group: 'editors' to +every resource. 'editors' should have the right to edit both metadata +and data. + +- Optionally creates a back-up every time you edit a resource. + + +To restore a back-up: + +.. code-block:: python + python resource.py --restore + +To add editors to every resource without creating a back-up: .. code-block:: python python resource.py +To add editors to every resource while creating a back-up before any +destructive edits: + +.. code-block:: python + python resource.py --enable-backup + """ +import argparse import json import redis +import os + +from datetime import datetime + + +def recover_hash(name: str, file_path: str, set_function) -> bool: + """Recover back-ups using the `set_function` + + Parameters: + + - `name`: Redis hash where `file_path` will be restored + + - `file_path`: File path where redis hash is sourced from + + - `set_function`: Function used to do the Redis backup for + example: HSET + + """ + try: + with open(file_path, "r") as f: + resources = json.load(f) + for resource_id, resource in resources.items(): + set_function(name=name, + key=resource_id, + value=resource) + return True + except Exception as e: + print(e) + return False if __name__ == "__main__": - REDIS_CONN = redis.Redis() - resources = REDIS_CONN.hgetall("resources_clone") - for resource_id, resource in resources.items(): - deserialized_resource = json.loads(resource) - deserialized_resource["group_masks"] = { - "editors": {"metadata": "edit", - "data": "edit"}} - REDIS_CONN.hset("resources_clone", + # Initialising the parser CLI arguments + parser = argparse.ArgumentParser() + parser.add_argument("--restore", + help="Restore from a given backup") + parser.add_argument("--enable-backup", action="store_true", + help="Create a back up before edits") + args = parser.parse_args() + + if args.restore: + if recover_hash(name="resources", + file_path=args.back_up, + set_function=redis.Redis(decode_responses=True).hset): + exit(f"\n Done restoring {args.back_up}!\n") + else: + exit(f"\n There was an error restoring {args.back_up}!\n") + + REDIS_CONN = redis.Redis(decode_responses=True) + RESOURCES = REDIS_CONN.hgetall("resources") + BACKUP_DIR = os.path.join(os.getenv("HOME"), "redis") + if args.enable_backup: + FILENAME = ("resources-" + f"{datetime.now().strftime('%Y-%m-%d-%I:%M:%S-%p')}" + ".json") + if not os.path.exists(BACKUP_DIR): + os.mkdir(BACKUP_DIR) + with open(os.path.join(BACKUP_DIR, FILENAME), "w") as f: + json.dump(RESOURCES, f, indent=4) + print(f"\nDone backing upto {FILENAME}") + + for resource_id, resource in RESOURCES.items(): + _resource = json.loads(resource) # str -> dict conversion + _resource["group_masks"] = {"editors": {"metadata": "edit", + "data": "edit"}} + REDIS_CONN.hset("resources", resource_id, - json.dumps(deserialized_resource)) + json.dumps(_resource)) + exit("Done updating `resources`\n") -- cgit v1.2.3 From c5215d1ed224480a274476933beded9d2ba7f7dc Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 4 Oct 2021 13:40:06 +0300 Subject: Decode redis response by default --- scripts/authentication/group.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index b89bc3ec..265e8664 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -104,9 +104,8 @@ if __name__ == "__main__": members = args.members if args.members else None admins = args.admins if args.admins else None - REDIS_CONN = redis.Redis() - USERS = {key.decode(): val.decode() - for key, val in REDIS_CONN.hgetall("users").items()} + REDIS_CONN = redis.Redis(decode_responses=True) + USERS = REDIS_CONN.hgetall("users") if not any([members, admins]): exit("\nExiting. Please provide a value for " -- cgit v1.2.3 From 8fae92c83d49042da68638319385df02061df44b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 4 Oct 2021 13:45:59 +0300 Subject: scripts: group.py: Remove "glom" dependency --- scripts/authentication/group.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index 265e8664..02f782b3 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -30,7 +30,6 @@ import redis import json from typing import Dict, List, Optional, Set -from glom import glom # type: ignore def create_group_data(users: Dict, target_group: str, @@ -74,10 +73,12 @@ def create_group_data(users: Dict, target_group: str, """ _members = "".join(members.split()).split(",") if members else [] _admins: List = "".join(admins.split()).split(",") if admins else [] - - user_emails: Set = {glom(json.loads(user_details), "email_address") - for _, user_details in users.items()} - + user_emails: Set = set() + for _, user_details in users.items(): + _details = json.loads(user_details) + if _details.get("email_address"): + user_emails.add(_details.get("email_address")) + print(user_emails) return {"key": "groups", "field": target_group, "value": json.dumps({ -- cgit v1.2.3 From 7c1dd1211f96ca1021debc27a80d1700e70b9c6b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 4 Oct 2021 13:49:12 +0300 Subject: scripts: group.py: Modify exit message when displaying updated data --- scripts/authentication/group.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index 02f782b3..cc0b037e 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -125,6 +125,6 @@ if __name__ == "__main__": args.group_name)) # type: ignore if created_p: exit(f"\nSuccessfully created the group: '{args.group_name}'\n" - f"Data: {groups}\n") + f"`HGETALL groups {args.group_name}`: {groups}\n") exit("\nNo new group was created.\n" - f"Updated Data: {groups}\n") + f"`HGETALL groups {args.group_name}`: {groups}\n") -- cgit v1.2.3 From 35a970adba5ee1d60769a81b446122a60eac9494 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 4 Oct 2021 21:03:52 +0000 Subject: Changed the group.py script to replace user e-mails with IDs and to include id, name, changed_timestamp, and created_timestamp in group details --- scripts/authentication/group.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index cc0b037e..a5f75aad 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -26,11 +26,13 @@ now results in:: """ import argparse +import datetime import redis import json from typing import Dict, List, Optional, Set +REDIS_CONN = redis.Redis(decode_responses=True) def create_group_data(users: Dict, target_group: str, members: Optional[str] = None, @@ -71,21 +73,26 @@ def create_group_data(users: Dict, target_group: str, me@test2.com, me@test3.com" """ - _members = "".join(members.split()).split(",") if members else [] + + _members: List = "".join(members.split()).split(",") if members else [] _admins: List = "".join(admins.split()).split(",") if admins else [] - user_emails: Set = set() - for _, user_details in users.items(): + + user_ids: Dict = dict() + for user_id, user_details in users.items(): _details = json.loads(user_details) if _details.get("email_address"): - user_emails.add(_details.get("email_address")) - print(user_emails) + user_ids[_details.get("email_address")] = user_id + print(user_ids) return {"key": "groups", "field": target_group, "value": json.dumps({ - "admins": [admin for admin in _admins - if admin in user_emails], - "members": [member for member in _members - if member in user_emails] + "id": target_group, + "name": target_group, + "admins": [user_ids[admin] for admin in _admins + if admin in user_ids], + "members": [user_ids[member] for member in _members + if member in user_ids], + "changed_timestamp": datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') })} @@ -105,7 +112,6 @@ if __name__ == "__main__": members = args.members if args.members else None admins = args.admins if args.admins else None - REDIS_CONN = redis.Redis(decode_responses=True) USERS = REDIS_CONN.hgetall("users") if not any([members, admins]): @@ -117,6 +123,12 @@ if __name__ == "__main__": target_group=args.group_name, members=members, admins=admins) + + if not REDIS_CONN.hget("groups", data.get("field", "")): + updated_data = json.loads(data["value"]) + updated_data["created_timestamp"] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + data["value"] = json.dumps(updated_data) + created_p = REDIS_CONN.hset(data.get("key", ""), data.get("field", ""), data.get("value", "")) -- cgit v1.2.3 From 4c6a7e46dd7afe311c0bed38c4a69ddadf3cb416 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 4 Oct 2021 21:09:15 +0000 Subject: Moved REDIS_CONN back into if __name__ == '__main__' since it doesn't need to be globally accessed anymore (I think I intiially moved it because I was calling it in create_group_data, but that ended up being unnecessary --- scripts/authentication/group.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index a5f75aad..76c7fb4f 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -32,8 +32,6 @@ import json from typing import Dict, List, Optional, Set -REDIS_CONN = redis.Redis(decode_responses=True) - def create_group_data(users: Dict, target_group: str, members: Optional[str] = None, admins: Optional[str] = None) -> Dict: @@ -112,6 +110,8 @@ if __name__ == "__main__": members = args.members if args.members else None admins = args.admins if args.admins else None + + REDIS_CONN = redis.Redis(decode_responses=True) USERS = REDIS_CONN.hgetall("users") if not any([members, admins]): -- cgit v1.2.3 From 949789a00d8e6e901cc18b939737cd42e14c0236 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 6 Oct 2021 16:12:22 +0300 Subject: scripts: group: Use a unique key to identify a group --- scripts/authentication/group.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index 76c7fb4f..eea13efe 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -29,6 +29,7 @@ import argparse import datetime import redis import json +import uuid from typing import Dict, List, Optional, Set @@ -71,26 +72,31 @@ def create_group_data(users: Dict, target_group: str, me@test2.com, me@test3.com" """ + # Emails + _members: Set = set("".join(members.split()).split(",") + if members else []) + _admins: Set = set("".join(admins.split()).split(",") + if admins else []) - _members: List = "".join(members.split()).split(",") if members else [] - _admins: List = "".join(admins.split()).split(",") if admins else [] + # Unique IDs + member_ids: Set = set() + admin_ids: Set = set() - user_ids: Dict = dict() for user_id, user_details in users.items(): _details = json.loads(user_details) - if _details.get("email_address"): - user_ids[_details.get("email_address")] = user_id - print(user_ids) + if _details.get("email_address") in _members: + member_ids.add(user_id) + if _details.get("email_address") in _admins: + admin_ids.add(user_id) + + timestamp: str = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') return {"key": "groups", - "field": target_group, + "field": str(uuid.uuid4()), "value": json.dumps({ - "id": target_group, "name": target_group, - "admins": [user_ids[admin] for admin in _admins - if admin in user_ids], - "members": [user_ids[member] for member in _members - if member in user_ids], - "changed_timestamp": datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + "admins": list(admin_ids), + "members": list(member_ids), + "changed_timestamp": timestamp, })} -- cgit v1.2.3 From 931c7eb07cc995118ba808df760fd74de036853f Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 6 Oct 2021 16:15:31 +0300 Subject: scripts: group: Remove unused import --- scripts/authentication/group.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index eea13efe..d3f9a1e4 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -31,7 +31,8 @@ import redis import json import uuid -from typing import Dict, List, Optional, Set +from typing import Dict, Optional, Set + def create_group_data(users: Dict, target_group: str, members: Optional[str] = None, -- cgit v1.2.3 From 870edaf2cf8ce8588ee7c58d08fc1f307f7198ec Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 6 Oct 2021 16:19:04 +0300 Subject: scripts: group: Remove empty `""` value for data.get data.get("field") will default to None if there is no value; and None is falsy. --- scripts/authentication/group.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index d3f9a1e4..1919d9db 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -131,10 +131,10 @@ if __name__ == "__main__": members=members, admins=admins) - if not REDIS_CONN.hget("groups", data.get("field", "")): updated_data = json.loads(data["value"]) updated_data["created_timestamp"] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') data["value"] = json.dumps(updated_data) + if not REDIS_CONN.hget("groups", data.get("field")): created_p = REDIS_CONN.hset(data.get("key", ""), data.get("field", ""), -- cgit v1.2.3 From 40dddc1a78a7808b480d26594ced689cdcc08c24 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 6 Oct 2021 21:23:08 +0300 Subject: scripts: group: Fix indentation --- scripts/authentication/group.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index 1919d9db..7e73be15 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -131,10 +131,10 @@ if __name__ == "__main__": members=members, admins=admins) - updated_data = json.loads(data["value"]) - updated_data["created_timestamp"] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') - data["value"] = json.dumps(updated_data) if not REDIS_CONN.hget("groups", data.get("field")): + updated_data = json.loads(data["value"]) + updated_data["created_timestamp"] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + data["value"] = json.dumps(updated_data) created_p = REDIS_CONN.hset(data.get("key", ""), data.get("field", ""), -- cgit v1.2.3 From 67222a6cb11995eb5a4af58f63cc9385ccfb9226 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 6 Oct 2021 21:24:16 +0300 Subject: scripts: group: Break up long line --- scripts/authentication/group.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index 7e73be15..ed17f260 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -133,7 +133,8 @@ if __name__ == "__main__": if not REDIS_CONN.hget("groups", data.get("field")): updated_data = json.loads(data["value"]) - updated_data["created_timestamp"] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + updated_data["created_timestamp"] = timestamp data["value"] = json.dumps(updated_data) created_p = REDIS_CONN.hset(data.get("key", ""), -- cgit v1.2.3 From dc378d26c003a8f0503ad69235d1685d66e4d611 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 6 Oct 2021 21:26:03 +0300 Subject: scripts: group: Update docstrings for "create_group_data" --- scripts/authentication/group.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index ed17f260..08a4a2bc 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -41,6 +41,9 @@ def create_group_data(users: Dict, target_group: str, "field", and "value" that can be used in a redis hash as follows: HSET key field value + The "field" return value is a unique-id that is used to + distinguish the groups. + Parameters: - `users`: a list of users for example: -- cgit v1.2.3 From d5f6670836cbed804a00e02ec0258d0c87564006 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 6 Oct 2021 21:40:35 +0300 Subject: scripts: group: Replace args.group_name with data["field"] --- scripts/authentication/group.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index 08a4a2bc..c8c2caad 100644 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -145,7 +145,7 @@ if __name__ == "__main__": data.get("value", "")) groups = json.loads(REDIS_CONN.hget("groups", - args.group_name)) # type: ignore + data.get("field"))) # type: ignore if created_p: exit(f"\nSuccessfully created the group: '{args.group_name}'\n" f"`HGETALL groups {args.group_name}`: {groups}\n") -- cgit v1.2.3 From 70f8ed53f85cfb42ca81ed6c3b4c9cf1060940e5 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 6 Oct 2021 21:44:51 +0300 Subject: scripts: resource: Add option for specifying a groups uuid --- scripts/authentication/resource.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py index 8fcf09d7..4996f34c 100644 --- a/scripts/authentication/resource.py +++ b/scripts/authentication/resource.py @@ -63,12 +63,16 @@ def recover_hash(name: str, file_path: str, set_function) -> bool: if __name__ == "__main__": # Initialising the parser CLI arguments parser = argparse.ArgumentParser() + parser.add_argument("--group-id", + help="Add the group id to all resources") parser.add_argument("--restore", help="Restore from a given backup") parser.add_argument("--enable-backup", action="store_true", help="Create a back up before edits") args = parser.parse_args() + if not args.group_id: + exit("Please specify the group-id!\n") if args.restore: if recover_hash(name="resources", file_path=args.back_up, @@ -92,8 +96,8 @@ if __name__ == "__main__": for resource_id, resource in RESOURCES.items(): _resource = json.loads(resource) # str -> dict conversion - _resource["group_masks"] = {"editors": {"metadata": "edit", - "data": "edit"}} + _resource["group_masks"] = {args.group_id: {"metadata": "edit", + "data": "edit"}} REDIS_CONN.hset("resources", resource_id, json.dumps(_resource)) -- cgit v1.2.3 From 7805a48172ada364d3783db043dbcf637445a7fe Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 8 Oct 2021 22:07:43 +0000 Subject: Adding convert_dol_genotypes.py to scripts; everything is hard-coded in it since I was only writing it to generate a specific file and it probably won't be re-used --- scripts/convert_dol_genotypes.py | 68 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 scripts/convert_dol_genotypes.py (limited to 'scripts') diff --git a/scripts/convert_dol_genotypes.py b/scripts/convert_dol_genotypes.py new file mode 100644 index 00000000..353f1b53 --- /dev/null +++ b/scripts/convert_dol_genotypes.py @@ -0,0 +1,68 @@ +# This is just to convert the Rqtl2 format genotype files for DOL into a .geno file +# Everything is hard-coded since I doubt this will be re-used and I just wanted to generate the file quickly + +import os + +geno_dir = "/home/zas1024/gn2-zach/DO_genotypes/" +markers_file = "/home/zas1024/gn2-zach/DO_genotypes/SNP_Map.txt" +gn_geno_path = "/home/zas1024/gn2-zach/DO_genotypes/DOL.geno" + +marker_data = {} +with open(markers_file, "r") as markers_fh: + for i, line in enumerate(markers_fh): + if i == 0: + continue + else: + line_items = line.split("\t") + this_marker = {} + this_marker['chr'] = line_items[2] if line_items[2] != "0" else "M" + this_marker['pos'] = f'{float(line_items[3])/1000000:.6f}' + marker_data[line_items[1]] = this_marker + +sample_names = [] +for filename in os.listdir(geno_dir): + if "gm4qtl2_geno" in filename: + with open(geno_dir + "/" + filename, "r") as rqtl_geno_fh: + for i, line in enumerate(rqtl_geno_fh): + line_items = line.split(",") + if i < 3: + continue + elif not len(sample_names) and i == 3: + sample_names = [item.replace("TLB", "TB") for item in line_items[1:]] + elif i > 3: + marker_data[line_items[0]]['genotypes'] = ["X" if item.strip() == "-" else item.strip() for item in line_items[1:]] + +def sort_func(e): + try: + return int(e['chr']) + except: + if e['chr'] == "X": + return 20 + elif e['chr'] == "Y": + return 21 + elif e['chr'] == "M": + return 22 + +marker_list = [] +for key, value in marker_data.items(): + if 'genotypes' in value: + this_marker = { + 'chr': value['chr'], + 'locus': key, + 'pos': value['pos'], + 'genotypes': value['genotypes'] + } + marker_list.append(this_marker) + +marker_list.sort(key=sort_func) + +with open(gn_geno_path, "w") as gn_geno_fh: + gn_geno_fh.write("\t".join((["Chr", "Locus", "cM", "Mb"] + sample_names))) + for marker in marker_list: + row_contents = [ + marker['chr'], + marker['locus'], + marker['pos'], + marker['pos'] + ] + marker['genotypes'] + gn_geno_fh.write("\t".join(row_contents) + "\n") -- cgit v1.2.3 From b37a9c6c495d142852d0cee54d83f5c9e815e37b Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 8 Oct 2021 22:19:07 +0000 Subject: Fixed the sort to account for both chr and pos in a kind of hack-y way + added some comments + changed EOL to LF because the file suddenly started including EOL characters --- scripts/convert_dol_genotypes.py | 142 ++++++++++++++++++++------------------- 1 file changed, 74 insertions(+), 68 deletions(-) (limited to 'scripts') diff --git a/scripts/convert_dol_genotypes.py b/scripts/convert_dol_genotypes.py index 353f1b53..81b3bd6d 100644 --- a/scripts/convert_dol_genotypes.py +++ b/scripts/convert_dol_genotypes.py @@ -1,68 +1,74 @@ -# This is just to convert the Rqtl2 format genotype files for DOL into a .geno file -# Everything is hard-coded since I doubt this will be re-used and I just wanted to generate the file quickly - -import os - -geno_dir = "/home/zas1024/gn2-zach/DO_genotypes/" -markers_file = "/home/zas1024/gn2-zach/DO_genotypes/SNP_Map.txt" -gn_geno_path = "/home/zas1024/gn2-zach/DO_genotypes/DOL.geno" - -marker_data = {} -with open(markers_file, "r") as markers_fh: - for i, line in enumerate(markers_fh): - if i == 0: - continue - else: - line_items = line.split("\t") - this_marker = {} - this_marker['chr'] = line_items[2] if line_items[2] != "0" else "M" - this_marker['pos'] = f'{float(line_items[3])/1000000:.6f}' - marker_data[line_items[1]] = this_marker - -sample_names = [] -for filename in os.listdir(geno_dir): - if "gm4qtl2_geno" in filename: - with open(geno_dir + "/" + filename, "r") as rqtl_geno_fh: - for i, line in enumerate(rqtl_geno_fh): - line_items = line.split(",") - if i < 3: - continue - elif not len(sample_names) and i == 3: - sample_names = [item.replace("TLB", "TB") for item in line_items[1:]] - elif i > 3: - marker_data[line_items[0]]['genotypes'] = ["X" if item.strip() == "-" else item.strip() for item in line_items[1:]] - -def sort_func(e): - try: - return int(e['chr']) - except: - if e['chr'] == "X": - return 20 - elif e['chr'] == "Y": - return 21 - elif e['chr'] == "M": - return 22 - -marker_list = [] -for key, value in marker_data.items(): - if 'genotypes' in value: - this_marker = { - 'chr': value['chr'], - 'locus': key, - 'pos': value['pos'], - 'genotypes': value['genotypes'] - } - marker_list.append(this_marker) - -marker_list.sort(key=sort_func) - -with open(gn_geno_path, "w") as gn_geno_fh: - gn_geno_fh.write("\t".join((["Chr", "Locus", "cM", "Mb"] + sample_names))) - for marker in marker_list: - row_contents = [ - marker['chr'], - marker['locus'], - marker['pos'], - marker['pos'] - ] + marker['genotypes'] - gn_geno_fh.write("\t".join(row_contents) + "\n") +# This is just to convert the Rqtl2 format genotype files for DOL into a .geno file +# Everything is hard-coded since I doubt this will be re-used and I just wanted to generate the file quickly + +import os + +geno_dir = "/home/zas1024/gn2-zach/DO_genotypes/" +markers_file = "/home/zas1024/gn2-zach/DO_genotypes/SNP_Map.txt" +gn_geno_path = "/home/zas1024/gn2-zach/DO_genotypes/DOL.geno" + +# Iterate through the SNP_Map.txt file to get marker positions +marker_data = {} +with open(markers_file, "r") as markers_fh: + for i, line in enumerate(markers_fh): + if i == 0: + continue + else: + line_items = line.split("\t") + this_marker = {} + this_marker['chr'] = line_items[2] if line_items[2] != "0" else "M" + this_marker['pos'] = f'{float(line_items[3])/1000000:.6f}' + marker_data[line_items[1]] = this_marker + +# Iterate through R/qtl2 format genotype files and pull out the samplelist and genotypes for each marker +sample_names = [] +for filename in os.listdir(geno_dir): + if "gm4qtl2_geno" in filename: + with open(geno_dir + "/" + filename, "r") as rqtl_geno_fh: + for i, line in enumerate(rqtl_geno_fh): + line_items = line.split(",") + if i < 3: + continue + elif not len(sample_names) and i == 3: + sample_names = [item.replace("TLB", "TB") for item in line_items[1:]] + elif i > 3: + marker_data[line_items[0]]['genotypes'] = ["X" if item.strip() == "-" else item.strip() for item in line_items[1:]] + +# Generate list of marker obs to iterate through when writing to .geno file +marker_list = [] +for key, value in marker_data.items(): + if 'genotypes' in value: + this_marker = { + 'chr': value['chr'], + 'locus': key, + 'pos': value['pos'], + 'genotypes': value['genotypes'] + } + marker_list.append(this_marker) + +def sort_func(e): + """For ensuring that X/Y chromosomes/mitochondria are sorted to the end correctly""" + try: + return float((e['chr']))*1000 + float(e['pos']) + except: + if e['chr'] == "X": + return 20000 + float(e['pos']) + elif e['chr'] == "Y": + return 21000 + float(e['pos']) + elif e['chr'] == "M": + return 22000 + float(e['pos']) + +# Sort markers by chromosome +marker_list.sort(key=sort_func) + +# Write lines to .geno file +with open(gn_geno_path, "w") as gn_geno_fh: + gn_geno_fh.write("\t".join((["Chr", "Locus", "cM", "Mb"] + sample_names))) + for marker in marker_list: + row_contents = [ + marker['chr'], + marker['locus'], + marker['pos'], + marker['pos'] + ] + marker['genotypes'] + gn_geno_fh.write("\t".join(row_contents) + "\n") -- cgit v1.2.3