Merge master and testingmaster

4 files changed, 448 insertions, 71 deletions

diff --git a/doc/API_readme.md b/doc/API_readme.md
new file mode 100644
index 00000000..be6668dc
--- /dev/null
+++ b/doc/API_readme.md
@@ -0,0 +1,169 @@
+# API Query Documentation #
+---
+# Fetching Dataset/Trait info/data #
+---
+## Fetch Species List ##
+
+To get a list of species with data available in GN (and their associated names and ids):
+```
+curl http://genenetwork.org/api/v_pre1/species
+[ { "FullName": "Mus musculus", "Id": 1, "Name": "mouse", "TaxonomyId": 10090 }, ... { "FullName": "Populus trichocarpa", "Id": 10, "Name": "poplar", "TaxonomyId": 3689 } ]
+```
+
+Or to get a single species info:
+```
+curl http://genenetwork.org/api/v_pre1/species/mouse
+``` 
+OR 
+```
+curl http://genenetwork.org/api/v_pre1/species/mouse.json
+```
+
+*For all queries where the last field is a user-specified name/ID, there will be the option to append a file format type. Currently there is only JSON (and it will default to JSON if none is provided), but other formats will be added later*
+
+## Fetch Groups/RISets ##
+
+This query can optionally filter by species:
+
+```
+curl http://genenetwork.org/api/v_pre1/groups (for all species)
+```
+OR
+```
+curl http://genenetwork.org/api/v_pre1/groups/mouse (for just mouse groups/RISets)
+[ { "DisplayName": "BXD", "FullName": "BXD RI Family", "GeneticType": "riset", "Id": 1, "MappingMethodId": "1", "Name": "BXD", "SpeciesId": 1, "public": 2 }, ... { "DisplayName": "AIL LGSM F34 and F39-43 (GBS)", "FullName": "AIL LGSM F34 and F39-43 (GBS)", "GeneticType": "intercross", "Id": 72, "MappingMethodId": "2", "Name": "AIL-LGSM-F34-F39-43-GBS", "SpeciesId": 1, "public": 2 } ]
+```
+
+## Fetch Genotypes for Group/RISet ##
+```
+curl http://genenetwork.org/api/v_pre1/genotypes/bimbam/BXD
+curl http://genenetwork.org/api/v_pre1/genotypes/BXD.bimbam
+```
+Returns a group's genotypes in one of several formats - bimbam, rqtl2, or geno (a format used by qtlreaper which is just a CSV file consisting of marker positions and genotypes)
+
+Rqtl2 genotype queries can also include the dataset name and will return a zip of the genotypes, phenotypes, and gene map (marker names/positions). For example:
+```
+curl http://genenetwork.org/api/v_pre1/genotypes/rqtl2/BXD/HC_M2_0606_P.zip
+```
+
+## Fetch Datasets ##
+```
+curl http://genenetwork.org/api/v_pre1/datasets/bxd
+```
+OR
+```
+curl http://genenetwork.org/api/v_pre1/datasets/mouse/bxd
+[ { "AvgID": 1, "CreateTime": "Fri, 01 Aug 2003 00:00:00 GMT", "DataScale": "log2", "FullName": "UTHSC/ETHZ/EPFL BXD Liver Polar Metabolites Extraction A, CD Cohorts (Mar 2017) log2", "Id": 1, "Long_Abbreviation": "BXDMicroArray_ProbeSet_August03", "ProbeFreezeId": 3, "ShortName": "Brain U74Av2 08/03 MAS5", "Short_Abbreviation": "Br_U_0803_M", "confidentiality": 0, "public": 0 }, ... { "AvgID": 3, "CreateTime": "Tue, 14 Aug 2018 00:00:00 GMT", "DataScale": "log2", "FullName": "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Aug18) RMA", "Id": 859, "Long_Abbreviation": "EPFLMouseLiverCDRMAApr18", "ProbeFreezeId": 181, "ShortName": "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Aug18) RMA", "Short_Abbreviation": "EPFLMouseLiverCDRMA0818", "confidentiality": 0, "public": 1 } ]
+```
+(I added the option to specify species just in case we end up with the same group name across multiple species at some point, though it's currently unnecessary)
+
+## Fetch Individual Dataset Info ##
+### For mRNA Assay/"ProbeSet" ###
+
+```
+curl http://genenetwork.org/api/v_pre1/dataset/HC_M2_0606_P
+```
+OR
+```
+curl http://genenetwork.org/api/v_pre1/dataset/bxd/HC_M2_0606_P```
+{ "confidential": 0, "data_scale": "log2", "dataset_type": "mRNA expression", "full_name": "Hippocampus Consortium M430v2 (Jun06) PDNN", "id": 112, "name": "HC_M2_0606_P", "public": 2, "short_name": "Hippocampus M430v2 BXD 06/06 PDNN", "tissue": "Hippocampus mRNA", "tissue_id": 9 }
+```
+(This also has the option to specify group/riset)
+
+### For "Phenotypes" (basically non-mRNA Expression; stuff like weight, sex, etc) ###
+For these traits, the query fetches publication info and takes the group and phenotype 'ID' as input. For example:
+```
+curl http://genenetwork.org/api/v_pre1/dataset/bxd/10001
+{ "dataset_type": "phenotype", "description": "Central nervous system, morphology: Cerebellum weight, whole, bilateral in adults of both sexes [mg]", "id": 10001, "name": "CBLWT2", "pubmed_id": 11438585, "title": "Genetic control of the mouse cerebellum: identification of quantitative trait loci modulating size and architecture", "year": "2001" }
+```
+
+## Fetch Sample Data for Dataset ##
+``` 
+curl http://genenetwork.org/api/v_pre1/sample_data/HSNIH-PalmerPublish.csv
+```
+
+Returns a CSV file with sample/strain names as the columns and trait IDs as rows
+
+## Fetch Sample Data for Single Trait ##
+```
+curl http://genenetwork.org/api/v_pre1/sample_data/HC_M2_0606_P/1436869_at
+[ { "data_id": 23415463, "sample_name": "129S1/SvImJ", "sample_name_2": "129S1/SvImJ", "se": 0.123, "value": 8.201 }, { "data_id": 23415463, "sample_name": "A/J", "sample_name_2": "A/J", "se": 0.046, "value": 8.413 }, { "data_id": 23415463, "sample_name": "AKR/J", "sample_name_2": "AKR/J", "se": 0.134, "value": 8.856 }, ... ]
+```
+
+## Fetch Trait List for Dataset ##
+```
+curl http://genenetwork.org/api/v_pre1/traits/HXBBXHPublish.json
+[ { "Additive": 0.0499967532467532, "Id": 10001, "LRS": 16.2831307029479, "Locus": "rs106114574", "PhenotypeId": 1449, "PublicationId": 319, "Sequence": 1 }, ... ]
+```
+
+Both JSON and CSV formats can be specified, with JSON as default. There is also an optional "ids_only" and "names_only" parameter that will only return a list of trait IDs or names, respectively.
+
+## Fetch Trait Info (Name, Description, Location, etc) ##
+### For mRNA Expression/"ProbeSet" ###
+```
+curl http://genenetwork.org/api/v_pre1/trait/HC_M2_0606_P/1436869_at
+{ "additive": -0.214087568058076, "alias": "HHG1; HLP3; HPE3; SMMCI; Dsh; Hhg1", "chr": "5", "description": "sonic hedgehog (hedgehog)", "id": 99602, "locus": "rs8253327", "lrs": 12.7711275309832, "mb": 28.457155, "mean": 9.27909090909091, "name": "1436869_at", "p_value": 0.306, "se": null, "symbol": "Shh" }
+```
+
+### For "Phenotypes" ###
+For phenotypes this just gets the  max LRS, its location, and additive effect (as calculated by qtlreaper)
+
+Since each group/riset only has one phenotype "dataset", this query takes either the group/riset name or the group/riset name + "Publish" (for example "BXDPublish", which is the dataset name in the DB) as input
+```
+curl http://genenetwork.org/api/v_pre1/trait/BXD/10001
+{ "additive": 2.39444435069444, "id": 4, "locus": "rs48756159", "lrs": 13.4974911471087 }
+```
+
+---
+
+# Analyses #
+---
+## Mapping ##
+Currently two mapping tools can be used - GEMMA and R/qtl. qtlreaper will be added later with Christian Fischer's RUST implementation - https://github.com/chfi/rust-qtlreaper
+
+Each method's query takes the following parameters respectively (more will be added):
+### GEMMA ###
+* trait_id (*required*) - ID for trait being mapped
+* db (*required*) - DB name for trait above (Short_Abbreviation listed when you query for datasets)
+* use_loco - Whether to use LOCO (leave one chromosome out) method (default = false)
+* maf - minor allele frequency (default = 0.01)
+
+Example query:
+```
+curl http://genenetwork.org/api/v_pre1/mapping?trait_id=10015&db=BXDPublish&method=gemma&use_loco=true
+```
+
+### R/qtl ###
+(See the R/qtl guide for information on some of these options - http://www.rqtl.org/manual/qtl-manual.pdf)
+* trait_id (*required*) - ID for trait being mapped
+* db (*required*) - DB name for trait above (Short_Abbreviation listed when you query for datasets)
+* rqtl_method - hk (default) | ehk | em | imp | mr | mr-imp | mr-argmax ; Corresponds to the "method" option for the R/qtl scanone function.
+* rqtl_model - normal (default) | binary | 2-part | np ; corresponds to the "model" option for the R/qtl scanone function
+* num_perm - number of permutations; 0 by default
+* control_marker - Name of marker to use as control; this relies on the user knowing the name of the marker they want to use as a covariate
+* interval_mapping - Whether to use interval mapping; "false" by default
+* pair_scan - *NYI*
+
+Example query:
+```
+curl http://genenetwork.org/api/v_pre1/mapping?trait_id=1418701_at&db=HC_M2_0606_P&method=rqtl&num_perm=100
+```
+
+Some combinations of methods/models may not make sense. The R/qtl manual should be referred to for any questions on its use (specifically the scanone function in this case)
+
+## Calculate Correlation ##
+Currently only Sample and Tissue correlations are implemented
+
+This query currently takes the following parameters (though more will be added):
+* trait_id (*required*) - ID for trait used for correlation
+* db (*required*) - DB name for the trait above (this is the Short_Abbreviation listed when you query for datasets)
+* target_db (*required*) - Target DB name to be correlated against
+* type - sample (default) | tissue
+* method - pearson (default) | spearman
+* return - Number of results to return (default = 500)
+
+Example query:
+```
+curl http://genenetwork.org/api/v_pre1/correlation?trait_id=1427571_at&db=HC_M2_0606_P&target_db=BXDPublish&type=sample&return_count=100
+[ { "#_strains": 6, "p_value": 0.004804664723032055, "sample_r": -0.942857142857143, "trait": 20511 }, { "#_strains": 6, "p_value": 0.004804664723032055, "sample_r": -0.942857142857143, "trait": 20724 }, { "#_strains": 12, "p_value": 1.8288943424888848e-05, "sample_r": -0.9233615170820528, "trait": 13536 }, { "#_strains": 7, "p_value": 0.006807187408935392, "sample_r": 0.8928571428571429, "trait": 10157 }, { "#_strains": 7, "p_value": 0.006807187408935392, "sample_r": -0.8928571428571429, "trait": 20392 }, ... ]
+```
diff --git a/doc/GUIX-Reproducible-from-source.org b/doc/GUIX-Reproducible-from-source.org
index 83adce99..19e4d14f 100644
--- a/doc/GUIX-Reproducible-from-source.org
+++ b/doc/GUIX-Reproducible-from-source.org
@@ -33,7 +33,6 @@ GNU Guix has to be installed as root. I tested this recipe on a fresh
 install of Debian 8.3.0 (in KVM) though it should work on any modern
 Linux distribution (including CentOS).
 
-
 Note that GN2 consists of an approx. 5 GB installation including
 database. If you use a virtual machine we recommend to use at least
 double.
diff --git a/doc/README.org b/doc/README.org
index f290480d..828ed2cd 100644
--- a/doc/README.org
+++ b/doc/README.org
@@ -3,23 +3,22 @@
 * Table of Contents                                                     :TOC:
  - [[#introduction][Introduction]]
  - [[#install][Install]]
-   - [[#tarball][Tarball]]
-   - [[#docker][Docker]]
-   - [[#with-source][With source]]
  - [[#running-gn2][Running GN2]]
- - [[#run-mysql-server][Run MySQL server]]
-   - [[#install-mysql-with-gnu-guix][Install MySQL with GNU GUIx]]
+ - [[#run-mariadb-server][Run MariaDB server]]
+   - [[#install-mariadb-with-gnu-guix][Install MariaDB with GNU GUIx]]
    - [[#load-the-small-database-in-mysql][Load the small database in MySQL]]
  - [[#gn2-dependency-graph][GN2 Dependency Graph]]
  - [[#working-with-the-gn2-source-code][Working with the GN2 source code]]
+ - [[#read-more][Read more]]
  - [[#trouble-shooting][Trouble shooting]]
    - [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]]
-   - [[#error-can-not-find-directory-homegn2_data][ERROR: can not find directory $HOME/gn2_data]]
+   - [[#error-can-not-find-directory-homegn2_data-or-can-not-find-directory-homegenotype_filesgenotype][ERROR: 'can not find directory $HOME/gn2_data' or 'can not find directory $HOME/genotype_files/genotype']]
    - [[#cant-run-a-module][Can't run a module]]
    - [[#rpy2-error-show-now-found][Rpy2 error 'show' now found]]
    - [[#mysql-cant-connect-server-through-socket-error][Mysql can't connect server through socket ERROR]]
- - [[#read-more][Read more]]
  - [[#irc-session][IRC session]]
+ - [[#notes][NOTES]]
+   - [[#deploying-gn2-official][Deploying GN2 official]]
 
 * Introduction
 
@@ -38,44 +37,10 @@ an example of the [[#gn2-dependency-graph][GN2 Dependency Graph]].
 
 * Install
 
-The quickest way to install GN2 is by using a binary installation
-(tarball or Docker image).  These installations are bundled by GNU
-Guix and include all dependencies. You can install GeneNetwork on most
-Linux distributions, including Debian, Ubuntu, Fedora and CentOS,
-provided you have administrator privileges (root). The alternative is
-a Docker installation.
-
-** Tarball
-
-Download the ~800Mb tarball from
-[[http://files.genenetwork.org/software/binary_tarball/]]. Validate the checksum and
-unpack to root, for example
-
-: tar xvzf genenetwork2-2.10rc3-1538ffd-tarball-pack.tar.gz
-: mv /gnu /
-: mv /opt/genenetwork2 /opt/
-
-Now you shoud be able to start the server with
-
-: /opt/genenetwork2/bin/genenetwork2
-
-When the server stops with a MySQL error [[#run-mysql-server][Run MySQL server]]
-and set SQL_URI to point at it. For example:
-
-: export SQL_URI=mysql://gn2:mysql_password@127.0.0.1/db_webqtl_s
-
-See also [[#mysql-cant-connect-server-through-socket-error][Mysql can't connect server through socket ERROR]].
-
-** Docker
-
-Docker images are also available through
-[[http://files.genenetwork.org/software/]]. Validate the checksum and run
-with [[https://docs.docker.com/engine/reference/commandline/load/][Docker load]].
-
-** With source
-
-For more elaborate installation instructions on deploying GeneNetwork from
-source see [[#source-deployment][Source deployment]].
+Make sure to install GNU Guix using the binary download instructions
+on the main website. Follow the instructions on
+[[GUIX-Reproducible-from-source.org]] to download pre-built binaries. Note
+the download amounts to several GBs of data.
 
 * Running GN2
 
@@ -92,37 +57,48 @@ or you can set environment variables to override individual parameters, e.g.
 the debug and logging switches can be particularly useful when
 developing GN2.
 
-* Run MySQL server
-** Install MySQL with GNU GUIx
+* Run MariaDB server
+** Install MariaDB with GNU GUIx
+
+/Note: we moved to MariaDB/
 
 These are the steps you can take to install a fresh installation of
-mysql (which comes as part of the GNU Guix genenetwork2 install).
+mariadb (which comes as part of the GNU Guix genenetwork2 install).
 
 As root configure and run
 
-:  adduser mysql && addgroup mysql
-:  mysqld --datadir=/var/mysql --initialize-insecure
-:  mkdir -p /var/run/mysqld
-:  chown mysql.mysql ~/mysql /var/run/mysqld
-:  mysqld -u mysql --datadir=/var/mysql --explicit_defaults_for_timestamp -P 12048"
+#+BEGIN_SRC bash
+adduser mariadb && addgroup mariadb
+mysqld --datadir=/home/mariadb/database --initialize-insecure
+mkdir -p /var/run/mariadbd
+chown mariadb.mariadb /var/run/mariadbd
+mysqld -u mariadb --datadir=/home/mariadb/database/mariadb --explicit_defaults_for_timestamp -P 12048"
+#+END_SRC
 
 If you want to run as root you may have to set
 
 : /etc/my.cnf
-: [mysqld]
+: [mariadbd]
 : user=root
 
+You also need to set
+
+: ft_min_word_len = 3
+
+To make sure word text searches (shh) work and rebuild the tables if
+required.
+
 To check error output in a file on start-up run with something like
 
-: mysqld -u mysql --console  --explicit_defaults_for_timestamp  --datadir=/gnu/mysql --log-error=~/test.log
+: mariadbd -u mariadb --console  --explicit_defaults_for_timestamp  --datadir=/gnu/mariadb --log-error=~/test.log
 
-Other tips are that Guix installs mysqld in your profile, so this may work
+Other tips are that Guix installs mariadbd in your profile, so this may work
 
-: /home/user/.guix-profile/bin/mysqld -u mysql --explicit_defaults_for_timestamp  --datadir=/gnu/mysql
+: /home/user/.guix-profile/bin/mariadbd -u mariadb --explicit_defaults_for_timestamp  --datadir=/gnu/mariadb
 
 When you get errors like:
 
-: qlalchemy.exc.IntegrityError: (_mysql_exceptions.IntegrityError) (1215, 'Cannot add foreign key constraint')
+: qlalchemy.exc.IntegrityError: (_mariadb_exceptions.IntegrityError) (1215, 'Cannot add foreign key constraint')
 
 you may need to set
 
@@ -192,6 +168,12 @@ http://biogems.info/contrib/genenetwork/gn2.svg
 
 See [[development.org]].
 
+* Read more
+
+If you want to understand the architecture of GN2 read
+[[Architecture.org]].  The rest of this document is mostly on deployment
+of GN2.
+
 * Trouble shooting
 
 ** ImportError: No module named jinja2
@@ -210,13 +192,17 @@ On one system:
 : export GEM_PATH="$HOME/.guix-profile/lib/ruby/gems/2.2.0"
 
 and perhaps a few more.
-** ERROR: can not find directory $HOME/gn2_data
+** ERROR: 'can not find directory $HOME/gn2_data' or 'can not find directory $HOME/genotype_files/genotype'
 
 The default settings file looks in your $HOME/gn2_data. Since these
 files come with a Guix installation you should take a hint from the
 values in the installed version of default_settings.py (see above in
 this document).
 
+You can use the GENENETWORK_FILES switch to set the datadir, for example
+
+: env GN2_PROFILE=~/opt/gn-latest GENENETWORK_FILES=/gnu/data/gn2_data ./bin/genenetwork2
+
 ** Can't run a module
 
 In rare cases, development modules are not brought in with Guix
@@ -257,17 +243,6 @@ if that works run genenetwork after setting SQL_URI to something like
 
 : export SQL_URI=mysql://gn2:mysql_password@127.0.0.1/db_webqtl_s
 
-* Running ElasticSearch
-
-In order to start up elasticsearch, use the following command:
-
-: env JAVA_HOME=/opt/jdk-9.0.4 /opt/elasticsearch-6.2.1/bin/elasticsearch
-
-* Read more
-
-If you want to understand the architecture of GN2 read
-[[Architecture.org]].  The rest of this document is mostly on deployment
-of GN2.
 
 * IRC session
 
@@ -676,3 +651,31 @@ The following derivations would be built:
 <pjotrp> I am building it on guix.genenetwork.org right now  [10:09]
 <user01> nice  [10:10]
 #+end_src
+
+* NOTES
+
+** Deploying GN2 official
+
+Let's see how fast we can deploy a second copy of GN2.
+
+- [-] Base install
+  + [X] First install a Debian server with GNU Guix on board
+  + [X] Get Guix build going
+    - [X] Build the correct version of Guix
+    - [X] Check out the correct gn-stable version of guix-bioinformatics http://git.genenetwork.org/pjotrp/guix-bioinformatics
+    - [X] guix package -i genenetwork2 -p /usr/local/guix-profiles/gn2-stable
+  + [X] Create a gn2 user and home with space
+  + [X] Install redis (currently debian)
+    - [X] add to systemd
+    - [X] update redis.cnf
+    - [X] update database
+  + [X] Install mariadb (currently debian mariadb-server)
+    - [X] add to systemd
+    - [X] system stop mysql
+    - [X] update mysql.cnf
+    - [X] update database (see gn-services/services/mariadb.md)
+    - [X] check tables
+  + [ ] run gn2 (rust-qtlreaper not working)
+  + [X] update nginx
+  + [ ] install genenetwork3
+    - [ ] add to systemd
diff --git a/doc/elasticsearch.org b/doc/elasticsearch.org
index 18adfc8b..864a8363 100644
--- a/doc/elasticsearch.org
+++ b/doc/elasticsearch.org
@@ -1,5 +1,12 @@
 * Elasticsearch
 
+** Introduction
+
+GeneNetwork uses elasticsearch (ES) for all things considered
+'state'. One example is user collections, another is user management.
+
+** Example
+
 To get the right environment, first you can get a python REPL with something like
 
 : env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ../etc/default_settings.py -cli python
@@ -39,3 +46,202 @@ record['hits']['hits'][0][u'_source']['email_address']
 u"myname@email.com"
 
 #+END_SRC
+
+** Health
+
+ES provides support for checking its health:
+
+: curl -XGET http://localhost:9200/_cluster/health?pretty=true
+
+#+BEGIN_SRC json
+
+
+    {
+      "cluster_name" : "asgard",
+      "status" : "yellow",
+      "timed_out" : false,
+      "number_of_nodes" : 1,
+      "number_of_data_nodes" : 1,
+      "active_primary_shards" : 5,
+      "active_shards" : 5,
+      "relocating_shards" : 0,
+      "initializing_shards" : 0,
+      "unassigned_shards" : 5
+    }
+
+#+END_SRC
+
+Yellow means just one instance is running (no worries).
+
+To get full cluster info
+
+: curl -XGET "localhost:9200/_cluster/stats?human&pretty"
+
+#+BEGIN_SRC json
+{
+  "_nodes" : {
+    "total" : 1,
+    "successful" : 1,
+    "failed" : 0
+  },
+  "cluster_name" : "elasticsearch",
+  "timestamp" : 1529050366452,
+  "status" : "yellow",
+  "indices" : {
+    "count" : 3,
+    "shards" : {
+      "total" : 15,
+      "primaries" : 15,
+      "replication" : 0.0,
+      "index" : {
+        "shards" : {
+          "min" : 5,
+          "max" : 5,
+          "avg" : 5.0
+        },
+        "primaries" : {
+          "min" : 5,
+          "max" : 5,
+          "avg" : 5.0
+        },
+        "replication" : {
+          "min" : 0.0,
+          "max" : 0.0,
+          "avg" : 0.0
+        }
+      }
+    },
+    "docs" : {
+      "count" : 14579,
+      "deleted" : 0
+    },
+    "store" : {
+      "size" : "44.7mb",
+      "size_in_bytes" : 46892794
+    },
+    "fielddata" : {
+      "memory_size" : "0b",
+      "memory_size_in_bytes" : 0,
+      "evictions" : 0
+    },
+    "query_cache" : {
+      "memory_size" : "0b",
+      "memory_size_in_bytes" : 0,
+      "total_count" : 0,
+      "hit_count" : 0,
+      "miss_count" : 0,
+      "cache_size" : 0,
+      "cache_count" : 0,
+      "evictions" : 0
+    },
+    "completion" : {
+      "size" : "0b",
+      "size_in_bytes" : 0
+    },
+    "segments" : {
+      "count" : 24,
+      "memory" : "157.3kb",
+      "memory_in_bytes" : 161112,
+      "terms_memory" : "122.6kb",
+      "terms_memory_in_bytes" : 125569,
+      "stored_fields_memory" : "15.3kb",
+      "stored_fields_memory_in_bytes" : 15728,
+      "term_vectors_memory" : "0b",
+      "term_vectors_memory_in_bytes" : 0,
+      "norms_memory" : "10.8kb",
+      "norms_memory_in_bytes" : 11136,
+      "points_memory" : "111b",
+      "points_memory_in_bytes" : 111,
+      "doc_values_memory" : "8.3kb",
+      "doc_values_memory_in_bytes" : 8568,
+      "index_writer_memory" : "0b",
+      "index_writer_memory_in_bytes" : 0,
+      "version_map_memory" : "0b",
+      "version_map_memory_in_bytes" : 0,
+      "fixed_bit_set" : "0b",
+      "fixed_bit_set_memory_in_bytes" : 0,
+      "max_unsafe_auto_id_timestamp" : -1,
+      "file_sizes" : { }
+    }
+  },
+  "nodes" : {
+    "count" : {
+      "total" : 1,
+      "data" : 1,
+      "coordinating_only" : 0,
+      "master" : 1,
+      "ingest" : 1
+    },
+    "versions" : [
+      "6.2.1"
+    ],
+    "os" : {
+      "available_processors" : 16,
+      "allocated_processors" : 16,
+      "names" : [
+        {
+          "name" : "Linux",
+          "count" : 1
+        }
+      ],
+      "mem" : {
+        "total" : "125.9gb",
+        "total_in_bytes" : 135189286912,
+        "free" : "48.3gb",
+        "free_in_bytes" : 51922628608,
+        "used" : "77.5gb",
+        "used_in_bytes" : 83266658304,
+        "free_percent" : 38,
+        "used_percent" : 62
+      }
+    },
+    "process" : {
+      "cpu" : {
+        "percent" : 0
+      },
+      "open_file_descriptors" : {
+        "min" : 415,
+        "max" : 415,
+        "avg" : 415
+      }
+    },
+    "jvm" : {
+      "max_uptime" : "1.9d",
+      "max_uptime_in_millis" : 165800616,
+      "versions" : [
+        {
+          "version" : "9.0.4",
+          "vm_name" : "OpenJDK 64-Bit Server VM",
+          "vm_version" : "9.0.4+11",
+          "vm_vendor" : "Oracle Corporation",
+          "count" : 1
+        }
+      ],
+      "mem" : {
+        "heap_used" : "1.1gb",
+        "heap_used_in_bytes" : 1214872032,
+        "heap_max" : "23.8gb",
+        "heap_max_in_bytes" : 25656426496
+      },
+      "threads" : 110
+    },
+    "fs" : {
+      "total" : "786.4gb",
+      "total_in_bytes" : 844400918528,
+      "free" : "246.5gb",
+      "free_in_bytes" : 264688160768,
+      "available" : "206.5gb",
+      "available_in_bytes" : 221771468800
+    },
+    "plugins" : [ ],
+    "network_types" : {
+      "transport_types" : {
+        "netty4" : 1
+      },
+      "http_types" : {
+        "netty4" : 1
+      }
+    }
+  }
+}
+#+BEGIN_SRC json