From 795ba2ffb5ed5150004785768b8b8c479b24b197 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 2 Sep 2022 07:22:08 -0500 Subject: Collapsed P2 resolving --- topics/systems/migrate-p2.gmi | 12 ++++++++++++ topics/systems/orchestration.gmi | 31 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 topics/systems/migrate-p2.gmi create mode 100644 topics/systems/orchestration.gmi (limited to 'topics/systems') diff --git a/topics/systems/migrate-p2.gmi b/topics/systems/migrate-p2.gmi new file mode 100644 index 0000000..c7fcb90 --- /dev/null +++ b/topics/systems/migrate-p2.gmi @@ -0,0 +1,12 @@ +* Penguin2 crash + +This week the boot partition of P2 crashed. We have a few lessons here, not least having a fallback for all services ;) + +* Tasks + +- [ ] setup space.uthsc.edu for GN2 development +- [ ] update DNS to tux02 128.169.4.52 and space 128.169.5.175 +- [ ] move CI/CD to tux02 + + +* Notes diff --git a/topics/systems/orchestration.gmi b/topics/systems/orchestration.gmi new file mode 100644 index 0000000..336dbbd --- /dev/null +++ b/topics/systems/orchestration.gmi @@ -0,0 +1,31 @@ +* Orchestration and fallbacks + +After the Penguin2 crash in Aug. 2022 it has become increasingly clear how hard it is to deploy GeneNetwork. GNU Guix helps a great deal with dependencies, but it does not handle orchestration between machines/services well. Also we need to look at the future. + +What is GN today in terms of services + + 1. Main GN2 server (Python, 20+ processes, 3+ instances: depends on all below) + 2. Matching GN3 server and REST endpoint (Python: less dependencies) + 3. Mariadb + 4. redis + 5. virtuoso + 6. GN-proxy (Racket, authentication handler: redis, mariadb) + 7. Alias proxy (Racket, gene aliases wikidata) + 8. Jupyter R and Julia notebooks + 9. BNW server (Octave) +10. UCSC browser +11. GN1 instances (older python, 12 instances in principle, 2 running today) +12. Access to HPC for GEMMA (coming) +13. Backup services +14. monitoring services + +I am still missing a few! All run by a man and his diligent dog. + +For the future the orchestration needs to be more robust and resilient. This means: + + 1. A fallback for every service on a separate machine + 2. Improved privacy protection for (future) human data + 3. Separate servers serving different data sources + 4. Partial synchronization between data sources + +The only way we *can* scale is by adding machines. But the system is not yet ready for that. Also getting rid of monolithic primary databases in favor of files helps synchronization. -- cgit v1.2.3 From c216fc75dbffe3e9ace4369a59256f0c93f72368 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 3 Sep 2022 04:42:45 -0500 Subject: orchestration: adding services --- topics/systems/orchestration.gmi | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'topics/systems') diff --git a/topics/systems/orchestration.gmi b/topics/systems/orchestration.gmi index 336dbbd..4f4c877 100644 --- a/topics/systems/orchestration.gmi +++ b/topics/systems/orchestration.gmi @@ -16,8 +16,11 @@ What is GN today in terms of services 10. UCSC browser 11. GN1 instances (older python, 12 instances in principle, 2 running today) 12. Access to HPC for GEMMA (coming) -13. Backup services -14. monitoring services +13. Backup services (sheepdog, rsync, borg) +14. monitoring services (incl. systemd, gunicorn, shepherd, sheepdog) +15. mail server +16. https certificates +17. http(s) proxy (nginx) I am still missing a few! All run by a man and his diligent dog. -- cgit v1.2.3 From cfe8b6a84d55949e13c92c622053d6b83468e681 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 3 Sep 2022 10:47:07 -0500 Subject: DNS update --- topics/systems/dns-changes.gmi | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'topics/systems') diff --git a/topics/systems/dns-changes.gmi b/topics/systems/dns-changes.gmi index 7c42589..a535cab 100644 --- a/topics/systems/dns-changes.gmi +++ b/topics/systems/dns-changes.gmi @@ -9,15 +9,22 @@ We are moving thing to a new DNS hosting service. We have accounts on both. To m * Sign in to your GoDaddy account. * Export the DNS record to a file * Print the DNS settings to a PDF -* Start a transfer from DNSsimple to get an auth code +* On GoDaddy disable WHOIS privacy protection (on the domains table) +* On GoDaddy start a transfer from DNSsimple to get an auth code + Click your username at the top right of the page. + Select My Products. + Click Manage next to the relevant domain. + Scroll down to Additional Settings. + Click Get authorization code. Note: If you have more than 6 domains in your account, click Email my code - + Set transfer on DNSsimple - tick DNS box - + Check DNS on switch - it may not be completely automatic - + Cherk record on DNSsimple - + Check transfer with `dig systemsgenetics.org NS` * On DNSimple add the authorisation code under Tamara -* Import DNS settings on DNSimple + + Set transfer on DNSimple - tick DNS box + + Check the `DNS on' switch - it may not be completely automatic + + Cherk record on DNSimple + + Check transfer with `dig systemsgenetics.org NS` +* Import DNS settings on DNSimple (cut-N-paste) + + Edit delegation - make sure the delegation box is set +=> https://support.dnsimple.com/articles/delegating-dnsimple-registered +* Test + + dig systemsgenetics.org [NS] + + dig systemsgenetics.org @ns1.dnsimple.com NS + + whois systemsgenetics.org -- cgit v1.2.3 From f52cfbb325ad28cd743ea94b83859977f0063230 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 5 Sep 2022 08:57:16 -0500 Subject: orchestration --- topics/systems/orchestration.gmi | 1 + 1 file changed, 1 insertion(+) (limited to 'topics/systems') diff --git a/topics/systems/orchestration.gmi b/topics/systems/orchestration.gmi index 4f4c877..5e0a298 100644 --- a/topics/systems/orchestration.gmi +++ b/topics/systems/orchestration.gmi @@ -21,6 +21,7 @@ What is GN today in terms of services 15. mail server 16. https certificates 17. http(s) proxy (nginx) +18. CI/CD server (with github webhooks) I am still missing a few! All run by a man and his diligent dog. -- cgit v1.2.3