Update Gemma docs

1 files changed, 112 insertions, 35 deletions

diff --git a/docs/gemma.ipynb b/docs/gemma.ipynb
index a66df47..956d1d3 100644
--- a/docs/gemma.ipynb
+++ b/docs/gemma.ipynb
@@ -21,7 +21,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -40,7 +40,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 2,
    "metadata": {
     "scrolled": true
    },
@@ -66,6 +66,27 @@
     "\n",
     "Before you perform any computation, you need to ensure you have your data uploaded. For these examples, I'll use data provided [here](https://github.com/genetics-statistics/gemma-wrapper/tree/master/test/data/input)\n",
     "\n",
+    "An example of a metadata.json file:\n",
+    "\n",
+    "```\n",
+    "{\n",
+    "    \"title\": \"This is my dataset for testing the REST API\",\n",
+    "    \"description\": \"Longer description\",\n",
+    "    \"date\": \"20210127\",\n",
+    "    \"authors\": [\n",
+    "        \"R. W. Williams\"\n",
+    "    ],\n",
+    "    \"cross\": \"BXD\",\n",
+    "    \"geno\": \"/ipfs/QmakcPHuxKouUvuNZ5Gna1pyXSAPB5fFSkqFt5pDydd9A4/GN638/BXH.geno\",\n",
+    "    \"pheno\": \"BXD_pheno.txt\",\n",
+    "    \"snps\": \"BXD_snps.txt\",\n",
+    "    \"covar\": \"BXD_covariates.txt\"\n",
+    "}\n",
+    "\n",
+    "```\n",
+    "\n",
+    "Note that you provide the genotype file with an ipfs valid address.\n",
+    "\n",
     "##### POST /api/metadata/upload/:token\n",
     "\n",
     "If token is provided, your user directory will be overridden with the new upload data!\n"
@@ -73,14 +94,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 22,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'status': 0, 'token': 'VROD4N-XVW1L0'}\n"
+      "PEERO1-WE4BBX\n"
      ]
     }
    ],
@@ -88,39 +109,49 @@
     "# I intentionally upload the data file without having the metadata file so that I can upload it later.\n",
     "r = requests.post(\"http://gn3-test.genenetwork.org/api/metadata/upload/\",\n",
     "                  files=[('file', ('file.tar.gz',\n",
-    "                                   open('/tmp/file.tar.gz',\n",
+    "                                   open('/tmp/file-no-metadata.tar.gz',\n",
     "                                        'rb'), 'application/octet-stream'))])\n",
     "\n",
-    "print(r.json())"
+    "token_no_metadata = r.json()[\"token\"]\n",
+    "print(token_no_metadata)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Now we overrlde the upload with the relevant metadatafile:"
+    "Now we can override the upload with the relevant metadatafile if we want. If the contents are the same, we get the same token!"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "r = requests.post(\"http://gn3-test.genenetwork.org/api/metadata/upload/VROD4N-XVW1L0\",\n",
+    "                  files=[('file', ('file.tar.gz',\n",
+    "                                   open('/tmp/file.tar.gz',\n",
+    "                                        'rb'), 'application/octet-stream'))])\n",
+    "\n",
+    "token = r.json()[\"token\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'status': 0, 'token': 'VROD4N-XVW1L0'}\n"
+      "VROD4N-XVW1L0\n"
      ]
     }
    ],
    "source": [
-    "r = requests.post(\"http://gn3-test.genenetwork.org/api/metadata/upload/VROD4N-XVW1L0\",\n",
-    "                  files=[('file', ('file-with-metadata.tar.gz',\n",
-    "                                   open('/tmp/file-with-metadata.tar.gz',\n",
-    "                                        'rb'), 'application/octet-stream'))])\n",
-    "\n",
-    "print(r.json())"
+    "print(token)"
    ]
   },
   {
@@ -157,19 +188,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 66,
+   "execution_count": 43,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'output_file': '8f4906862459e59dcb452fd8162d2cc1-output.json', 'status': 'queued', 'unique_id': 'cmd::2021-03-1005-1727-1727-7006dfef-8e60-4bb2-8390-0c174d15a17c'}\n"
+      "{'output_file': '5e9b1e32053f3d456418af2119a0a9e0-output.json', 'status': 'queued', 'unique_id': 'cmd::2021-03-2209-1901-1901-aef4973e-6d07-49f3-8216-ea883c6442b3'}\n"
      ]
     }
    ],
    "source": [
-    "r = requests.post(\"http://gn3-test.genenetwork.org/api/gemma/k-compute/VROD4N-XVW1L0\")\n",
+    "r = requests.post(\"http://gn3-test.genenetwork.org/api/\"\n",
+    "                  f\"gemma/k-compute/{token}\")\n",
     "print(r.json())"
    ]
   },
@@ -180,7 +212,7 @@
     "This generates the command:\n",
     "\n",
     "```\n",
-    "gemma-wrapper --json -- -g /tmp/VROD4N-XVW1L0/BXD_geno.txt.gz -p /tmp/VROD4N-XVW1L0/BXD_pheno.txt -a /tmp/VROD4N-XVW1L0/BXD_snps.txt -gk > /tmp/VROD4N-XVW1L0/8f4906862459e59dcb452fd8162d2cc1-output.json\n",
+    "gemma-wrapper --json -- -g /tmp/cache/QmRZpiWCPxn6d1sCaPLpFQBpDmaPaPygvZ165oK4B9myjy/GN602/BXD.geno -p /tmp/VROD4N-XVW1L0/BXD_pheno.txt -a /tmp/VROD4N-XVW1L0/BXD_snps.txt -gk > /tmp/VROD4N-XVW1L0/5e9b1e32053f3d456418af2119a0a9e0-output.json\n",
     "```"
    ]
   },
@@ -193,25 +225,52 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 44,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'status': 'queued'}\n"
+      "{'status': 'error'}\n"
      ]
     }
    ],
    "source": [
-    "r = requests.get(\"http://gn3-test.genenetwork.org/api/gemma/status/cmd%3A%3A2021-03-1005-1727-1727-7006dfef-8e60-4bb2-8390-0c174d15a17c\")\n",
+    "r = requests.get(\"http://gn3-test.genenetwork.org/api/gemma/\"\n",
+    "                 \"status/cmd%3A%3A2021-03-2209-1901-1901-aef4973e-6d07-\"\n",
+    "                 \"49f3-8216-ea883c6442b3\")\n",
     "print(r.json())"
    ]
   },
   {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Work as expected with the error log:\n",
+    "\n",
+    "```\n",
+    "Using GEMMA 0.98.4 (2020-12-15) by Xiang Zhou and team (C) 2012-2020\n",
+    "\n",
+    "Found 0.98.4, comparing against expected v0.98.0\n",
+    "\n",
+    "GEMMA 0.98.4 (2020-12-15) by Xiang Zhou and team (C) 2012-2020\n",
+    "Reading Files ...\n",
+    "**** FAILED: Parsing input file '/tmp/cache/QmRZpiWCPxn6d1sCaPLpFQBpDmaPaPygvZ165oK4B9myjy/GN602/BXD.geno' failed in function ReadFile_geno in src/gemma_io.cpp at line 744                                                              \n",
+    "{\"warnings\":[],\"errno\":2,\"debug\":[],\"type\":\"K\",\"files\":[[null,\"/tmp/605220a4d963542392fb44c15060cf6a8cee659b.log.txt\",\"/tmp/605220a4d963542392fb44c15060cf6a8cee659b.cXX.txt\"]],\"gemma_command\":\"/gnu/store/yyrfmg0i18w190l4lb21cv86fqclalsk-gemma-gn2-git-0.98.3-47221d6/bin/gemma -g /tmp/cache/QmRZpiWCPxn6d1sCaPLpFQBpDmaPaPygvZ165oK4B9myjy/GN602/BXD.geno -p /tmp/VROD4N-XVW1L0/BXD_pheno.txt -a /tmp/VROD4N-XVW1L0/BXD_snps.txt -gk -outdir /tmp -o 605220a4d963542392fb44c15060cf6a8cee659b\"}Traceback (most recent call last):                          \n",
+    "        6: from /gnu/store/86df7mjr3y1mrz62k4zipm6bznj10faj-profile/bin/gemma-wrapper:4:in `<main>'                                                        \n",
+    "        5: from /gnu/store/86df7mjr3y1mrz62k4zipm6bznj10faj-profile/bin/gemma-wrapper:4:in `load'                                                          \n",
+    "        4: from /gnu/store/p1alkfcsw3m24vlgxfb6gk24zn67h8n2-gemma-wrapper-0.98.1/bin/.real/gemma-wrapper:23:in `<top (required)>'                          \n",
+    "        3: from /gnu/store/p1alkfcsw3m24vlgxfb6gk24zn67h8n2-gemma-wrapper-0.98.1/bin/.real/gemma-wrapper:23:in `load'                                      \n",
+    "        2: from /gnu/store/p1alkfcsw3m24vlgxfb6gk24zn67h8n2-gemma-wrapper-0.98.1/lib/ruby/vendor_ruby/gems/bio-gemma-wrapper-0.98.1/bin/gemma-wrapper:345:in `<top (required)>'                                                          \n",
+    "        1: from /gnu/store/p1alkfcsw3m24vlgxfb6gk24zn67h8n2-gemma-wrapper-0.98.1/lib/ruby/vendor_ruby/gems/bio-gemma-wrapper-0.98.1/bin/gemma-wrapper:316:in `block in <top (required)>'                                                 \n",
+    "/gnu/store/p1alkfcsw3m24vlgxfb6gk24zn67h8n2-gemma-wrapper-0.98.1/lib/ruby/vendor_ruby/gems/bio-gemma-wrapper-0.98.1/bin/gemma-wrapper:278:in `block in <top (required)>': exit on GEMMA error 2 (RuntimeError) \n",
+    "```"
+   ]
+  },
+  {
    "cell_type": "code",
-   "execution_count": 68,
+   "execution_count": 31,
    "metadata": {
     "scrolled": true
    },
@@ -220,13 +279,17 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'status': 'success'}\n"
+      "{'status': 'error'}\n"
      ]
     }
    ],
    "source": [
     "# After a short while:\n",
-    "r = requests.get(\"http://gn3-test.genenetwork.org/api/gemma/status/cmd%3A%3A2021-03-1005-1727-1727-7006dfef-8e60-4bb2-8390-0c174d15a17c\")\n",
+    "# Commands will throw error since I didn't know where the right\n",
+    "# genotype files for BXD are :(\n",
+    "r = requests.get(\"http://gn3-test.genenetwork.org/api/gemma/status/\"\n",
+    "                 \"cmd%3A%3A2021-03-2209-0015-\"\n",
+    "                 \"0015-bb5c765f-93a3-45d1-a50d-b817cd7192e7\")\n",
     "print(r.json())"
    ]
   },
@@ -259,7 +322,8 @@
     }
    ],
    "source": [
-    "r = requests.post(\"http://gn3-test.genenetwork.org/api/gemma/k-compute/loco/1%2C2%2C3%2C4/VROD4N-XVW1L0\")\n",
+    "r = requests.post(\"http://gn3-test.genenetwork.org/api/gemma/\"\n",
+    "                  \"k-compute/loco/1%2C2%2C3%2C4/VROD4N-XVW1L0\")\n",
     "print(r.json())"
    ]
   },
@@ -288,7 +352,9 @@
     }
    ],
    "source": [
-    "r = requests.post(\"http://gn3-test.genenetwork.org/api/gemma/gwa-compute/8f4906862459e59dcb452fd8162d2cc1-output.json/VROD4N-XVW1L0\")\n",
+    "r = requests.post(\"http://gn3-test.genenetwork.org/api/\"\n",
+    "                  \"gemma/gwa-compute/8f4906862459e59dcb\"\n",
+    "                  \"452fd8162d2cc1-output.json/VROD4N-XVW1L0\")\n",
     "print(r.json())"
    ]
   },
@@ -315,7 +381,10 @@
     }
    ],
    "source": [
-    "r = requests.post(\"http://gn3-test.genenetwork.org/api/gemma/gwa-compute/covars/8f4906862459e59dcb452fd8162d2cc1-output.json/VROD4N-XVW1L0\")\n",
+    "r = requests.post(\"http://gn3-test.genenetwork.org/api/gemma/\"\n",
+    "                  \"gwa-compute/covars/\"\n",
+    "                  \"8f4906862459e59dcb452fd8162d2cc1-output.json/\"\n",
+    "                  \"VROD4N-XVW1L0\")\n",
     "print(r.json())"
    ]
   },
@@ -342,7 +411,10 @@
     }
    ],
    "source": [
-    "r = requests.post(\"http://gn3-test.genenetwork.org/api/gemma/gwa-compute/8f4906862459e59dcb452fd8162d2cc1-output.json/loco/maf/9/VROD4N-XVW1L0\")\n",
+    "r = requests.post(\"http://gn3-test.genenetwork.org/api/\"\n",
+    "                  \"gemma/gwa-compute/\"\n",
+    "                  \"8f4906862459e59dcb452fd8162d2cc1-output.json/\"\n",
+    "                  \"loco/maf/9/VROD4N-XVW1L0\")\n",
     "print(r.json())"
    ]
   },
@@ -369,7 +441,9 @@
     }
    ],
    "source": [
-    "r = requests.post(\"http://gn3-test.genenetwork.org/api/gemma/gwa-compute/8f4906862459e59dcb452fd8162d2cc1-output.json/loco/covars/maf/9/VROD4N-XVW1L0\")\n",
+    "r = requests.post(\"http://gn3-test.genenetwork.org/api/gemma/\"\n",
+    "                  \"gwa-compute/8f4906862459e59dcb452fd8162d2cc1\"\n",
+    "                  \"-output.json/loco/covars/maf/9/VROD4N-XVW1L0\")\n",
     "print(r.json())"
    ]
   },
@@ -403,7 +477,8 @@
     }
    ],
    "source": [
-    "r = requests.post(\"http://gn3-test.genenetwork.org/api/gemma/k-gwa-compute/covars/VROD4N-XVW1L0\")\n",
+    "r = requests.post(\"http://gn3-test.genenetwork.org/api/gemma/\"\n",
+    "                  \"k-gwa-compute/covars/VROD4N-XVW1L0\")\n",
     "print(r.json())"
    ]
   },
@@ -428,7 +503,8 @@
     }
    ],
    "source": [
-    "r = requests.post(\"http://gn3-test.genenetwork.org/api/gemma/k-gwa-compute/loco/1%2C2%2C3/maf/9/VROD4N-XVW1L0\")\n",
+    "r = requests.post(\"http://gn3-test.genenetwork.org/api/gemma/\"\n",
+    "                  \"k-gwa-compute/loco/1%2C2%2C3/maf/9/VROD4N-XVW1L0\")\n",
     "print(r.json())"
    ]
   },
@@ -453,7 +529,8 @@
     }
    ],
    "source": [
-    "r = requests.post(\"http://gn3-test.genenetwork.org/api/gemma//k-gwa-compute/covars/loco/1%2C2%2C3/maf/9/VROD4N-XVW1L0\")\n",
+    "r = requests.post(\"http://gn3-test.genenetwork.org/api/gemma/\"\n",
+    "                  \"k-gwa-compute/covars/loco/1%2C2%2C3/maf/9/VROD4N-XVW1L0\")\n",
     "print(r.json())"
    ]
   }
@@ -461,7 +538,7 @@
  "metadata": {
   "kernelspec": {
    "display_name": "Python 3",
-   "language": "/gnu/store/bvd09gb8ka642jzgxd2lpqlpdp160gn0-python-wrapper-3.8.2/bin/python",
+   "language": "python",
    "name": "python3"
   },
   "language_info": {
@@ -474,7 +551,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,