From e8b6b4e99f7a4ea649dab0c5a7bf7695531e97d2 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 22 Oct 2021 18:52:39 +0000 Subject: Include the admin privilege for the 'editors' group, since it wasn't being set before (which caused some problems) --- scripts/authentication/resource.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py index 4996f34c..1a2bcd8a 100644 --- a/scripts/authentication/resource.py +++ b/scripts/authentication/resource.py @@ -97,7 +97,8 @@ if __name__ == "__main__": for resource_id, resource in RESOURCES.items(): _resource = json.loads(resource) # str -> dict conversion _resource["group_masks"] = {args.group_id: {"metadata": "edit", - "data": "edit"}} + "data": "edit", + "admin": "edit-admins"}} REDIS_CONN.hset("resources", resource_id, json.dumps(_resource)) -- cgit v1.2.3 From 6bdf173ccb79e2e21cfc4e0d96377dae3bd1e6c1 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 14:55:03 +0300 Subject: scripts: group.py: Add shebang --- scripts/authentication/group.py | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 scripts/authentication/group.py (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py old mode 100644 new mode 100755 index c8c2caad..460a6329 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """A script for adding users to a specific group. Example: -- cgit v1.2.3 From 467723891d7c153c365322ea5812a714186a478d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 14:57:11 +0300 Subject: scripts: group.py: Add extra optional arg to specify uid --- scripts/authentication/group.py | 118 ++++++++++++++++++++++++---------------- 1 file changed, 71 insertions(+), 47 deletions(-) (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index 460a6329..08652454 100755 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -7,22 +7,40 @@ Assuming there are no groups and 'test@bonfacemunyoki.com' does not exist in Redis: .. code-block:: bash - python group.py -g "editors" -m "test@bonfacemunyoki.com" + python group.py -n "editors" \ + -m "me@bonfacemunyoki.com" results in:: - Successfully created the group: 'editors' - Data: '{"admins": [], "members": []}' + Successfully created the group: 'editors' + `HGET groups 0360449f-5a96-4940-8522-b22d62085da9`: {'name': 'editors', 'admins': [], 'members': ['8ad942fe-490d-453e-bd37-56f252e41603'], 'changed_timestamp': 'Oct 19 2021 09:34AM', 'created_timestamp': 'Oct 19 2021 09:34AM'} + -If 'me@bonfacemunyoki.com' exists in 'users' in Redis and we run: +Assuming we have a group's unique id: .. code-block:: bash - python group.py -g "editors" -m "me@bonfacemunyoki.com" + python group.py -n "editors" \ + -m "me@bonfacemunyoki.com" \ + -g "8ad942fe-490d-453e-bd37-56f252e41603" now results in:: - No new group was created. - Updated Data: {'admins': [], 'members': ['me@bonfacemunyoki.com']} + Successfully created the group: 'editors' + `HGET groups 8ad942fe-490d-453e-bd37-56f252e41603`: {'name': 'editors', 'admins': [], 'members': ['8ad942fe-490d-453e-bd37-56f252e41603'], 'changed_timestamp': 'Oct 19 2021 09:38AM', 'created_timestamp': 'Oct 19 2021 09:38AM'} + +If 'me@bonfacemunyoki.com' exists in 'users' in Redis for the above +command and we run: + +.. code-block:: bash + python group.py -n "editors" \ + -m "me@bonfacemunyoki.com" \ + -g "8ad942fe-490d-453e-bd37-56f252e41603" + +now results in:: + + No new group was created. + `HGET groups 8ad942fe-490d-453e-bd37-56f252e41603`: {'name': 'editors', 'admins': [], 'members': ['8ad942fe-490d-453e-bd37-56f252e41603'], 'changed_timestamp': 'Oct 19 2021 09:40AM'} + """ @@ -37,44 +55,48 @@ from typing import Dict, Optional, Set def create_group_data(users: Dict, target_group: str, members: Optional[str] = None, - admins: Optional[str] = None) -> Dict: - """Return a dictionary that contains the following keys: "key", - "field", and "value" that can be used in a redis hash as follows: - HSET key field value + admins: Optional[str] = None, + group_id: Optional[str] = None) -> Dict: + """Create group data which is isomorphic to a redis HSET i.e.: KEY, + FIELD and VALUE. If the group_id is not specified, a unique hash + will be generated. The "field" return value is a unique-id that is used to distinguish the groups. - Parameters: - - - `users`: a list of users for example: - - {'8ad942fe-490d-453e-bd37-56f252e41603': - '{"email_address": "me@test.com", - "full_name": "John Doe", - "organization": "Genenetwork", - "password": {"algorithm": "pbkdf2", - "hashfunc": "sha256", - "salt": "gJrd1HnPSSCmzB5veMPaVk2ozzDlS1Z7Ggcyl1+pciA=", - "iterations": 100000, "keylength": 32, - "created_timestamp": "2021-09-22T11:32:44.971912", - "password": "edcdaa60e84526c6"}, - "user_id": "8ad942fe", "confirmed": 1, - "registration_info": { - "timestamp": "2021-09-22T11:32:45.028833", - "ip_address": "127.0.0.1", - "user_agent": "Mozilla/5.0"}}'} - - - `target_group`: the group name that will be stored inside the - "groups" hash in Redis. - - - `members`: a comma-separated list of values that contain members - of the `target_group` e.g. "me@test1.com, me@test2.com, - me@test3.com" - - - `admins`: a comma-separated list of values that contain - administrators of the `target_group` e.g. "me@test1.com, - me@test2.com, me@test3.com" + Args: + - users: a list of users for example: + {'8ad942fe-490d-453e-bd37-56f252e41603': + '{"email_address": "me@test.com", + "full_name": "John Doe", + "organization": "Genenetwork", + "password": {"algorithm": "pbkdf2", + "hashfunc": "sha256", + "salt": "gJrd1HnPSSCmzB5veMPaVk2ozzDlS1Z7Ggcyl1+pciA=", + "iterations": 100000, "keylength": 32, + "created_timestamp": "2021-09-22T11:32:44.971912", + "password": "edcdaa60e84526c6"}, + "user_id": "8ad942fe", "confirmed": 1, + "registration_info": { + "timestamp": "2021-09-22T11:32:45.028833", + "ip_address": "127.0.0.1", + "user_agent": "Mozilla/5.0"}}'} + + - target_group: the group name that will be stored inside the + "groups" hash in Redis. + - members: an optional comma-separated list of values that + contain members of the `target_group` e.g. "me@test1.com, + me@test2.com, me@test3.com" + - admins: an optional comma-separated list of values that + contain administrators of the `target_group` + e.g. "me@test1.com, me@test2.com, me@test3.com" + - group_id: an optional unique identifier for a group. If not + set, a unique value will be auto-generated. + + Returns: + A dictionary that contains the following keys: "key", "field", + and "value" that can be used in a redis hash as follows: HSET key + field value """ # Emails @@ -96,7 +118,7 @@ def create_group_data(users: Dict, target_group: str, timestamp: str = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') return {"key": "groups", - "field": str(uuid.uuid4()), + "field": (group_id or str(uuid.uuid4())), "value": json.dumps({ "name": target_group, "admins": list(admin_ids), @@ -108,8 +130,10 @@ def create_group_data(users: Dict, target_group: str, if __name__ == "__main__": # Initialising the parser CLI arguments parser = argparse.ArgumentParser() - parser.add_argument("-g", "--group-name", + parser.add_argument("-n", "--group-name", help="This is the name of the GROUP mask") + parser.add_argument("-g", "--group-id", + help="[Optional] This is the name of the GROUP mask") parser.add_argument("-m", "--members", help="Members of the GROUP mask") parser.add_argument("-a", "--admins", @@ -133,7 +157,8 @@ if __name__ == "__main__": users=USERS, target_group=args.group_name, members=members, - admins=admins) + admins=admins, + group_id=args.group_id) if not REDIS_CONN.hget("groups", data.get("field")): updated_data = json.loads(data["value"]) @@ -144,11 +169,10 @@ if __name__ == "__main__": created_p = REDIS_CONN.hset(data.get("key", ""), data.get("field", ""), data.get("value", "")) - groups = json.loads(REDIS_CONN.hget("groups", data.get("field"))) # type: ignore if created_p: exit(f"\nSuccessfully created the group: '{args.group_name}'\n" - f"`HGETALL groups {args.group_name}`: {groups}\n") + f"`HGET groups {data.get('field')}`: {groups}\n") exit("\nNo new group was created.\n" - f"`HGETALL groups {args.group_name}`: {groups}\n") + f"`HGET groups {data.get('field')}`: {groups}\n") -- cgit v1.2.3 From 5d6559cdc0f9251a90b351bf440b930a000f8354 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 16:17:32 +0300 Subject: scripts: resource.py: Add shebang --- scripts/authentication/resource.py | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 scripts/authentication/resource.py (limited to 'scripts') diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py old mode 100644 new mode 100755 index 1a2bcd8a..1f5f31d8 --- a/scripts/authentication/resource.py +++ b/scripts/authentication/resource.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """A script that: - Optionally restores data from a json file. -- cgit v1.2.3 From 182327254fe2964c3dd41aabe49ab99748800a64 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 19 Oct 2021 16:47:38 +0300 Subject: scripts: resource: Add value for "admin" when updating resources --- scripts/authentication/resource.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/authentication/resource.py b/scripts/authentication/resource.py index 1f5f31d8..4b8d801a 100755 --- a/scripts/authentication/resource.py +++ b/scripts/authentication/resource.py @@ -38,14 +38,14 @@ from datetime import datetime def recover_hash(name: str, file_path: str, set_function) -> bool: """Recover back-ups using the `set_function` - Parameters: + Args: + - name: Redis hash where `file_path` will be restored + - file_path: File path where redis hash is sourced from + - set_function: Function used to do the Redis backup for + example: HSET - - `name`: Redis hash where `file_path` will be restored - - - `file_path`: File path where redis hash is sourced from - - - `set_function`: Function used to do the Redis backup for - example: HSET + Returns: + A boolean indicating whether the function ran successfully. """ try: @@ -99,7 +99,7 @@ if __name__ == "__main__": _resource = json.loads(resource) # str -> dict conversion _resource["group_masks"] = {args.group_id: {"metadata": "edit", "data": "edit", - "admin": "edit-admins"}} + "admin": "not-admin"}} REDIS_CONN.hset("resources", resource_id, json.dumps(_resource)) -- cgit v1.2.3 From f75ffe73e592bc79f4decf268d50b225ade43488 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 5 Nov 2021 13:40:44 +0300 Subject: Add script for manually adding "editors" to each resource --- scripts/authentication/editors.py | 45 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 scripts/authentication/editors.py (limited to 'scripts') diff --git a/scripts/authentication/editors.py b/scripts/authentication/editors.py new file mode 100755 index 00000000..12f48872 --- /dev/null +++ b/scripts/authentication/editors.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Manually add editors users""" +import redis +import json +import uuid +import datetime + +if __name__ == "__main__": + conn = redis.Redis(decode_responses=True) + group_uid = "" + for guid in conn.hgetall("groups"): + group_details = json.loads(conn.hget("groups", guid)) + if group_details.get("name") == "editors": + group_uid = guid + break + + if not group_uid: + group_uid = str(uuid.uuid4()) + timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + conn.hset( + "groups", + group_uid, + json.dumps( + { + "name": "editors", + "admins": [], + "members": ["labwilliams@gmail.com"], + "changed_timestamp": timestamp, + })) + + for resource in conn.hgetall("resources"): + _resource = json.loads(conn.hget("resources", resource)) + _resource["default_mask"] = { + 'data': 'view', + 'metadata': 'view', + 'admin': 'not-admin', + } + _resource["group_masks"] = { + group_uid: { + 'metadata': 'edit', + 'data': 'edit', + 'admin': 'edit-admins', + }} + conn.hset("resources", resource, json.dumps(_resource)) + print("Done adding editor's group to resources!") -- cgit v1.2.3 From 5c8429d6365e0a03ea2686633df5fd555c9b8ae5 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 5 Nov 2021 13:44:01 +0300 Subject: Use members uuid --- scripts/authentication/editors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/authentication/editors.py b/scripts/authentication/editors.py index 12f48872..750dc309 100755 --- a/scripts/authentication/editors.py +++ b/scripts/authentication/editors.py @@ -24,7 +24,7 @@ if __name__ == "__main__": { "name": "editors", "admins": [], - "members": ["labwilliams@gmail.com"], + "members": ["8ad942fe-490d-453e-bd37-56f252e41603"], "changed_timestamp": timestamp, })) -- cgit v1.2.3 From 178fe53c907b8857865236d9066d7f6f3cf3d9a7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 5 Nov 2021 17:35:13 +0300 Subject: Add created_timestamp when creating a new group --- scripts/authentication/editors.py | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/authentication/editors.py b/scripts/authentication/editors.py index 750dc309..dc3b1075 100755 --- a/scripts/authentication/editors.py +++ b/scripts/authentication/editors.py @@ -25,6 +25,7 @@ if __name__ == "__main__": "name": "editors", "admins": [], "members": ["8ad942fe-490d-453e-bd37-56f252e41603"], + "created_timestamp": timestamp, "changed_timestamp": timestamp, })) -- cgit v1.2.3 From 990d2e4305ed89c2bc78ff370b71a3a4f68e5f01 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 5 Nov 2021 17:40:52 +0300 Subject: Add created_timestamp when creating a new group --- scripts/authentication/group.py | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/authentication/group.py b/scripts/authentication/group.py index 08652454..dd9ba808 100755 --- a/scripts/authentication/group.py +++ b/scripts/authentication/group.py @@ -123,6 +123,7 @@ def create_group_data(users: Dict, target_group: str, "name": target_group, "admins": list(admin_ids), "members": list(member_ids), + "created_timestamp": timestamp, "changed_timestamp": timestamp, })} -- cgit v1.2.3 From 69676ecd49a779350ef7bd3faa348c7f27602a20 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 17 Dec 2021 20:10:33 +0000 Subject: Some updates to the DOL genotypes conversion script --- scripts/convert_dol_genotypes.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/convert_dol_genotypes.py b/scripts/convert_dol_genotypes.py index 81b3bd6d..5cda2e9c 100644 --- a/scripts/convert_dol_genotypes.py +++ b/scripts/convert_dol_genotypes.py @@ -1,6 +1,8 @@ # This is just to convert the Rqtl2 format genotype files for DOL into a .geno file # Everything is hard-coded since I doubt this will be re-used and I just wanted to generate the file quickly +# This is to be used on the files generated as described by Karl Broman here - https://kbroman.org/qtl2/pages/prep_do_data.html + import os geno_dir = "/home/zas1024/gn2-zach/DO_genotypes/" @@ -30,9 +32,13 @@ for filename in os.listdir(geno_dir): if i < 3: continue elif not len(sample_names) and i == 3: - sample_names = [item.replace("TLB", "TB") for item in line_items[1:]] + sample_names_positions = [[item.replace("TLB", "TB").strip(), i] for i, item in enumerate(line_items[1:])] + sample_names_positions.sort(key = lambda x: x[0][2:]) + sample_names = [sample[0] for sample in sample_names_positions] elif i > 3: - marker_data[line_items[0]]['genotypes'] = ["X" if item.strip() == "-" else item.strip() for item in line_items[1:]] + genotypes = ["X" if item.strip() == "-" else item.strip() for item in line_items[1:]] + ordered_genotypes = [genotypes[i].strip() for i in [pos[1] for pos in sample_names_positions]] + marker_data[line_items[0]]['genotypes'] = ordered_genotypes # Generate list of marker obs to iterate through when writing to .geno file marker_list = [] @@ -46,6 +52,7 @@ for key, value in marker_data.items(): } marker_list.append(this_marker) + def sort_func(e): """For ensuring that X/Y chromosomes/mitochondria are sorted to the end correctly""" try: @@ -63,7 +70,7 @@ marker_list.sort(key=sort_func) # Write lines to .geno file with open(gn_geno_path, "w") as gn_geno_fh: - gn_geno_fh.write("\t".join((["Chr", "Locus", "cM", "Mb"] + sample_names))) + gn_geno_fh.write("\t".join((["Chr", "Locus", "cM", "Mb"] + sample_names)) + "\n") for marker in marker_list: row_contents = [ marker['chr'], @@ -72,3 +79,4 @@ with open(gn_geno_path, "w") as gn_geno_fh: marker['pos'] ] + marker['genotypes'] gn_geno_fh.write("\t".join(row_contents) + "\n") + -- cgit v1.2.3