aboutsummaryrefslogtreecommitdiff
path: root/doc
diff options
context:
space:
mode:
Diffstat (limited to 'doc')
-rw-r--r--doc/API_readme.md169
-rw-r--r--doc/GUIX-Reproducible-from-source.org1
-rw-r--r--doc/README.org143
-rw-r--r--doc/elasticsearch.org206
4 files changed, 448 insertions, 71 deletions
diff --git a/doc/API_readme.md b/doc/API_readme.md
new file mode 100644
index 00000000..be6668dc
--- /dev/null
+++ b/doc/API_readme.md
@@ -0,0 +1,169 @@
+# API Query Documentation #
+---
+# Fetching Dataset/Trait info/data #
+---
+## Fetch Species List ##
+
+To get a list of species with data available in GN (and their associated names and ids):
+```
+curl http://genenetwork.org/api/v_pre1/species
+[ { "FullName": "Mus musculus", "Id": 1, "Name": "mouse", "TaxonomyId": 10090 }, ... { "FullName": "Populus trichocarpa", "Id": 10, "Name": "poplar", "TaxonomyId": 3689 } ]
+```
+
+Or to get a single species info:
+```
+curl http://genenetwork.org/api/v_pre1/species/mouse
+```
+OR
+```
+curl http://genenetwork.org/api/v_pre1/species/mouse.json
+```
+
+*For all queries where the last field is a user-specified name/ID, there will be the option to append a file format type. Currently there is only JSON (and it will default to JSON if none is provided), but other formats will be added later*
+
+## Fetch Groups/RISets ##
+
+This query can optionally filter by species:
+
+```
+curl http://genenetwork.org/api/v_pre1/groups (for all species)
+```
+OR
+```
+curl http://genenetwork.org/api/v_pre1/groups/mouse (for just mouse groups/RISets)
+[ { "DisplayName": "BXD", "FullName": "BXD RI Family", "GeneticType": "riset", "Id": 1, "MappingMethodId": "1", "Name": "BXD", "SpeciesId": 1, "public": 2 }, ... { "DisplayName": "AIL LGSM F34 and F39-43 (GBS)", "FullName": "AIL LGSM F34 and F39-43 (GBS)", "GeneticType": "intercross", "Id": 72, "MappingMethodId": "2", "Name": "AIL-LGSM-F34-F39-43-GBS", "SpeciesId": 1, "public": 2 } ]
+```
+
+## Fetch Genotypes for Group/RISet ##
+```
+curl http://genenetwork.org/api/v_pre1/genotypes/bimbam/BXD
+curl http://genenetwork.org/api/v_pre1/genotypes/BXD.bimbam
+```
+Returns a group's genotypes in one of several formats - bimbam, rqtl2, or geno (a format used by qtlreaper which is just a CSV file consisting of marker positions and genotypes)
+
+Rqtl2 genotype queries can also include the dataset name and will return a zip of the genotypes, phenotypes, and gene map (marker names/positions). For example:
+```
+curl http://genenetwork.org/api/v_pre1/genotypes/rqtl2/BXD/HC_M2_0606_P.zip
+```
+
+## Fetch Datasets ##
+```
+curl http://genenetwork.org/api/v_pre1/datasets/bxd
+```
+OR
+```
+curl http://genenetwork.org/api/v_pre1/datasets/mouse/bxd
+[ { "AvgID": 1, "CreateTime": "Fri, 01 Aug 2003 00:00:00 GMT", "DataScale": "log2", "FullName": "UTHSC/ETHZ/EPFL BXD Liver Polar Metabolites Extraction A, CD Cohorts (Mar 2017) log2", "Id": 1, "Long_Abbreviation": "BXDMicroArray_ProbeSet_August03", "ProbeFreezeId": 3, "ShortName": "Brain U74Av2 08/03 MAS5", "Short_Abbreviation": "Br_U_0803_M", "confidentiality": 0, "public": 0 }, ... { "AvgID": 3, "CreateTime": "Tue, 14 Aug 2018 00:00:00 GMT", "DataScale": "log2", "FullName": "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Aug18) RMA", "Id": 859, "Long_Abbreviation": "EPFLMouseLiverCDRMAApr18", "ProbeFreezeId": 181, "ShortName": "EPFL/LISP BXD CD Liver Affy Mouse Gene 1.0 ST (Aug18) RMA", "Short_Abbreviation": "EPFLMouseLiverCDRMA0818", "confidentiality": 0, "public": 1 } ]
+```
+(I added the option to specify species just in case we end up with the same group name across multiple species at some point, though it's currently unnecessary)
+
+## Fetch Individual Dataset Info ##
+### For mRNA Assay/"ProbeSet" ###
+
+```
+curl http://genenetwork.org/api/v_pre1/dataset/HC_M2_0606_P
+```
+OR
+```
+curl http://genenetwork.org/api/v_pre1/dataset/bxd/HC_M2_0606_P```
+{ "confidential": 0, "data_scale": "log2", "dataset_type": "mRNA expression", "full_name": "Hippocampus Consortium M430v2 (Jun06) PDNN", "id": 112, "name": "HC_M2_0606_P", "public": 2, "short_name": "Hippocampus M430v2 BXD 06/06 PDNN", "tissue": "Hippocampus mRNA", "tissue_id": 9 }
+```
+(This also has the option to specify group/riset)
+
+### For "Phenotypes" (basically non-mRNA Expression; stuff like weight, sex, etc) ###
+For these traits, the query fetches publication info and takes the group and phenotype 'ID' as input. For example:
+```
+curl http://genenetwork.org/api/v_pre1/dataset/bxd/10001
+{ "dataset_type": "phenotype", "description": "Central nervous system, morphology: Cerebellum weight, whole, bilateral in adults of both sexes [mg]", "id": 10001, "name": "CBLWT2", "pubmed_id": 11438585, "title": "Genetic control of the mouse cerebellum: identification of quantitative trait loci modulating size and architecture", "year": "2001" }
+```
+
+## Fetch Sample Data for Dataset ##
+```
+curl http://genenetwork.org/api/v_pre1/sample_data/HSNIH-PalmerPublish.csv
+```
+
+Returns a CSV file with sample/strain names as the columns and trait IDs as rows
+
+## Fetch Sample Data for Single Trait ##
+```
+curl http://genenetwork.org/api/v_pre1/sample_data/HC_M2_0606_P/1436869_at
+[ { "data_id": 23415463, "sample_name": "129S1/SvImJ", "sample_name_2": "129S1/SvImJ", "se": 0.123, "value": 8.201 }, { "data_id": 23415463, "sample_name": "A/J", "sample_name_2": "A/J", "se": 0.046, "value": 8.413 }, { "data_id": 23415463, "sample_name": "AKR/J", "sample_name_2": "AKR/J", "se": 0.134, "value": 8.856 }, ... ]
+```
+
+## Fetch Trait List for Dataset ##
+```
+curl http://genenetwork.org/api/v_pre1/traits/HXBBXHPublish.json
+[ { "Additive": 0.0499967532467532, "Id": 10001, "LRS": 16.2831307029479, "Locus": "rs106114574", "PhenotypeId": 1449, "PublicationId": 319, "Sequence": 1 }, ... ]
+```
+
+Both JSON and CSV formats can be specified, with JSON as default. There is also an optional "ids_only" and "names_only" parameter that will only return a list of trait IDs or names, respectively.
+
+## Fetch Trait Info (Name, Description, Location, etc) ##
+### For mRNA Expression/"ProbeSet" ###
+```
+curl http://genenetwork.org/api/v_pre1/trait/HC_M2_0606_P/1436869_at
+{ "additive": -0.214087568058076, "alias": "HHG1; HLP3; HPE3; SMMCI; Dsh; Hhg1", "chr": "5", "description": "sonic hedgehog (hedgehog)", "id": 99602, "locus": "rs8253327", "lrs": 12.7711275309832, "mb": 28.457155, "mean": 9.27909090909091, "name": "1436869_at", "p_value": 0.306, "se": null, "symbol": "Shh" }
+```
+
+### For "Phenotypes" ###
+For phenotypes this just gets the max LRS, its location, and additive effect (as calculated by qtlreaper)
+
+Since each group/riset only has one phenotype "dataset", this query takes either the group/riset name or the group/riset name + "Publish" (for example "BXDPublish", which is the dataset name in the DB) as input
+```
+curl http://genenetwork.org/api/v_pre1/trait/BXD/10001
+{ "additive": 2.39444435069444, "id": 4, "locus": "rs48756159", "lrs": 13.4974911471087 }
+```
+
+---
+
+# Analyses #
+---
+## Mapping ##
+Currently two mapping tools can be used - GEMMA and R/qtl. qtlreaper will be added later with Christian Fischer's RUST implementation - https://github.com/chfi/rust-qtlreaper
+
+Each method's query takes the following parameters respectively (more will be added):
+### GEMMA ###
+* trait_id (*required*) - ID for trait being mapped
+* db (*required*) - DB name for trait above (Short_Abbreviation listed when you query for datasets)
+* use_loco - Whether to use LOCO (leave one chromosome out) method (default = false)
+* maf - minor allele frequency (default = 0.01)
+
+Example query:
+```
+curl http://genenetwork.org/api/v_pre1/mapping?trait_id=10015&db=BXDPublish&method=gemma&use_loco=true
+```
+
+### R/qtl ###
+(See the R/qtl guide for information on some of these options - http://www.rqtl.org/manual/qtl-manual.pdf)
+* trait_id (*required*) - ID for trait being mapped
+* db (*required*) - DB name for trait above (Short_Abbreviation listed when you query for datasets)
+* rqtl_method - hk (default) | ehk | em | imp | mr | mr-imp | mr-argmax ; Corresponds to the "method" option for the R/qtl scanone function.
+* rqtl_model - normal (default) | binary | 2-part | np ; corresponds to the "model" option for the R/qtl scanone function
+* num_perm - number of permutations; 0 by default
+* control_marker - Name of marker to use as control; this relies on the user knowing the name of the marker they want to use as a covariate
+* interval_mapping - Whether to use interval mapping; "false" by default
+* pair_scan - *NYI*
+
+Example query:
+```
+curl http://genenetwork.org/api/v_pre1/mapping?trait_id=1418701_at&db=HC_M2_0606_P&method=rqtl&num_perm=100
+```
+
+Some combinations of methods/models may not make sense. The R/qtl manual should be referred to for any questions on its use (specifically the scanone function in this case)
+
+## Calculate Correlation ##
+Currently only Sample and Tissue correlations are implemented
+
+This query currently takes the following parameters (though more will be added):
+* trait_id (*required*) - ID for trait used for correlation
+* db (*required*) - DB name for the trait above (this is the Short_Abbreviation listed when you query for datasets)
+* target_db (*required*) - Target DB name to be correlated against
+* type - sample (default) | tissue
+* method - pearson (default) | spearman
+* return - Number of results to return (default = 500)
+
+Example query:
+```
+curl http://genenetwork.org/api/v_pre1/correlation?trait_id=1427571_at&db=HC_M2_0606_P&target_db=BXDPublish&type=sample&return_count=100
+[ { "#_strains": 6, "p_value": 0.004804664723032055, "sample_r": -0.942857142857143, "trait": 20511 }, { "#_strains": 6, "p_value": 0.004804664723032055, "sample_r": -0.942857142857143, "trait": 20724 }, { "#_strains": 12, "p_value": 1.8288943424888848e-05, "sample_r": -0.9233615170820528, "trait": 13536 }, { "#_strains": 7, "p_value": 0.006807187408935392, "sample_r": 0.8928571428571429, "trait": 10157 }, { "#_strains": 7, "p_value": 0.006807187408935392, "sample_r": -0.8928571428571429, "trait": 20392 }, ... ]
+```
diff --git a/doc/GUIX-Reproducible-from-source.org b/doc/GUIX-Reproducible-from-source.org
index 83adce99..19e4d14f 100644
--- a/doc/GUIX-Reproducible-from-source.org
+++ b/doc/GUIX-Reproducible-from-source.org
@@ -33,7 +33,6 @@ GNU Guix has to be installed as root. I tested this recipe on a fresh
install of Debian 8.3.0 (in KVM) though it should work on any modern
Linux distribution (including CentOS).
-
Note that GN2 consists of an approx. 5 GB installation including
database. If you use a virtual machine we recommend to use at least
double.
diff --git a/doc/README.org b/doc/README.org
index f290480d..828ed2cd 100644
--- a/doc/README.org
+++ b/doc/README.org
@@ -3,23 +3,22 @@
* Table of Contents :TOC:
- [[#introduction][Introduction]]
- [[#install][Install]]
- - [[#tarball][Tarball]]
- - [[#docker][Docker]]
- - [[#with-source][With source]]
- [[#running-gn2][Running GN2]]
- - [[#run-mysql-server][Run MySQL server]]
- - [[#install-mysql-with-gnu-guix][Install MySQL with GNU GUIx]]
+ - [[#run-mariadb-server][Run MariaDB server]]
+ - [[#install-mariadb-with-gnu-guix][Install MariaDB with GNU GUIx]]
- [[#load-the-small-database-in-mysql][Load the small database in MySQL]]
- [[#gn2-dependency-graph][GN2 Dependency Graph]]
- [[#working-with-the-gn2-source-code][Working with the GN2 source code]]
+ - [[#read-more][Read more]]
- [[#trouble-shooting][Trouble shooting]]
- [[#importerror-no-module-named-jinja2][ImportError: No module named jinja2]]
- - [[#error-can-not-find-directory-homegn2_data][ERROR: can not find directory $HOME/gn2_data]]
+ - [[#error-can-not-find-directory-homegn2_data-or-can-not-find-directory-homegenotype_filesgenotype][ERROR: 'can not find directory $HOME/gn2_data' or 'can not find directory $HOME/genotype_files/genotype']]
- [[#cant-run-a-module][Can't run a module]]
- [[#rpy2-error-show-now-found][Rpy2 error 'show' now found]]
- [[#mysql-cant-connect-server-through-socket-error][Mysql can't connect server through socket ERROR]]
- - [[#read-more][Read more]]
- [[#irc-session][IRC session]]
+ - [[#notes][NOTES]]
+ - [[#deploying-gn2-official][Deploying GN2 official]]
* Introduction
@@ -38,44 +37,10 @@ an example of the [[#gn2-dependency-graph][GN2 Dependency Graph]].
* Install
-The quickest way to install GN2 is by using a binary installation
-(tarball or Docker image). These installations are bundled by GNU
-Guix and include all dependencies. You can install GeneNetwork on most
-Linux distributions, including Debian, Ubuntu, Fedora and CentOS,
-provided you have administrator privileges (root). The alternative is
-a Docker installation.
-
-** Tarball
-
-Download the ~800Mb tarball from
-[[http://files.genenetwork.org/software/binary_tarball/]]. Validate the checksum and
-unpack to root, for example
-
-: tar xvzf genenetwork2-2.10rc3-1538ffd-tarball-pack.tar.gz
-: mv /gnu /
-: mv /opt/genenetwork2 /opt/
-
-Now you shoud be able to start the server with
-
-: /opt/genenetwork2/bin/genenetwork2
-
-When the server stops with a MySQL error [[#run-mysql-server][Run MySQL server]]
-and set SQL_URI to point at it. For example:
-
-: export SQL_URI=mysql://gn2:mysql_password@127.0.0.1/db_webqtl_s
-
-See also [[#mysql-cant-connect-server-through-socket-error][Mysql can't connect server through socket ERROR]].
-
-** Docker
-
-Docker images are also available through
-[[http://files.genenetwork.org/software/]]. Validate the checksum and run
-with [[https://docs.docker.com/engine/reference/commandline/load/][Docker load]].
-
-** With source
-
-For more elaborate installation instructions on deploying GeneNetwork from
-source see [[#source-deployment][Source deployment]].
+Make sure to install GNU Guix using the binary download instructions
+on the main website. Follow the instructions on
+[[GUIX-Reproducible-from-source.org]] to download pre-built binaries. Note
+the download amounts to several GBs of data.
* Running GN2
@@ -92,37 +57,48 @@ or you can set environment variables to override individual parameters, e.g.
the debug and logging switches can be particularly useful when
developing GN2.
-* Run MySQL server
-** Install MySQL with GNU GUIx
+* Run MariaDB server
+** Install MariaDB with GNU GUIx
+
+/Note: we moved to MariaDB/
These are the steps you can take to install a fresh installation of
-mysql (which comes as part of the GNU Guix genenetwork2 install).
+mariadb (which comes as part of the GNU Guix genenetwork2 install).
As root configure and run
-: adduser mysql && addgroup mysql
-: mysqld --datadir=/var/mysql --initialize-insecure
-: mkdir -p /var/run/mysqld
-: chown mysql.mysql ~/mysql /var/run/mysqld
-: mysqld -u mysql --datadir=/var/mysql --explicit_defaults_for_timestamp -P 12048"
+#+BEGIN_SRC bash
+adduser mariadb && addgroup mariadb
+mysqld --datadir=/home/mariadb/database --initialize-insecure
+mkdir -p /var/run/mariadbd
+chown mariadb.mariadb /var/run/mariadbd
+mysqld -u mariadb --datadir=/home/mariadb/database/mariadb --explicit_defaults_for_timestamp -P 12048"
+#+END_SRC
If you want to run as root you may have to set
: /etc/my.cnf
-: [mysqld]
+: [mariadbd]
: user=root
+You also need to set
+
+: ft_min_word_len = 3
+
+To make sure word text searches (shh) work and rebuild the tables if
+required.
+
To check error output in a file on start-up run with something like
-: mysqld -u mysql --console --explicit_defaults_for_timestamp --datadir=/gnu/mysql --log-error=~/test.log
+: mariadbd -u mariadb --console --explicit_defaults_for_timestamp --datadir=/gnu/mariadb --log-error=~/test.log
-Other tips are that Guix installs mysqld in your profile, so this may work
+Other tips are that Guix installs mariadbd in your profile, so this may work
-: /home/user/.guix-profile/bin/mysqld -u mysql --explicit_defaults_for_timestamp --datadir=/gnu/mysql
+: /home/user/.guix-profile/bin/mariadbd -u mariadb --explicit_defaults_for_timestamp --datadir=/gnu/mariadb
When you get errors like:
-: qlalchemy.exc.IntegrityError: (_mysql_exceptions.IntegrityError) (1215, 'Cannot add foreign key constraint')
+: qlalchemy.exc.IntegrityError: (_mariadb_exceptions.IntegrityError) (1215, 'Cannot add foreign key constraint')
you may need to set
@@ -192,6 +168,12 @@ http://biogems.info/contrib/genenetwork/gn2.svg
See [[development.org]].
+* Read more
+
+If you want to understand the architecture of GN2 read
+[[Architecture.org]]. The rest of this document is mostly on deployment
+of GN2.
+
* Trouble shooting
** ImportError: No module named jinja2
@@ -210,13 +192,17 @@ On one system:
: export GEM_PATH="$HOME/.guix-profile/lib/ruby/gems/2.2.0"
and perhaps a few more.
-** ERROR: can not find directory $HOME/gn2_data
+** ERROR: 'can not find directory $HOME/gn2_data' or 'can not find directory $HOME/genotype_files/genotype'
The default settings file looks in your $HOME/gn2_data. Since these
files come with a Guix installation you should take a hint from the
values in the installed version of default_settings.py (see above in
this document).
+You can use the GENENETWORK_FILES switch to set the datadir, for example
+
+: env GN2_PROFILE=~/opt/gn-latest GENENETWORK_FILES=/gnu/data/gn2_data ./bin/genenetwork2
+
** Can't run a module
In rare cases, development modules are not brought in with Guix
@@ -257,17 +243,6 @@ if that works run genenetwork after setting SQL_URI to something like
: export SQL_URI=mysql://gn2:mysql_password@127.0.0.1/db_webqtl_s
-* Running ElasticSearch
-
-In order to start up elasticsearch, use the following command:
-
-: env JAVA_HOME=/opt/jdk-9.0.4 /opt/elasticsearch-6.2.1/bin/elasticsearch
-
-* Read more
-
-If you want to understand the architecture of GN2 read
-[[Architecture.org]]. The rest of this document is mostly on deployment
-of GN2.
* IRC session
@@ -676,3 +651,31 @@ The following derivations would be built:
<pjotrp> I am building it on guix.genenetwork.org right now [10:09]
<user01> nice [10:10]
#+end_src
+
+* NOTES
+
+** Deploying GN2 official
+
+Let's see how fast we can deploy a second copy of GN2.
+
+- [-] Base install
+ + [X] First install a Debian server with GNU Guix on board
+ + [X] Get Guix build going
+ - [X] Build the correct version of Guix
+ - [X] Check out the correct gn-stable version of guix-bioinformatics http://git.genenetwork.org/pjotrp/guix-bioinformatics
+ - [X] guix package -i genenetwork2 -p /usr/local/guix-profiles/gn2-stable
+ + [X] Create a gn2 user and home with space
+ + [X] Install redis (currently debian)
+ - [X] add to systemd
+ - [X] update redis.cnf
+ - [X] update database
+ + [X] Install mariadb (currently debian mariadb-server)
+ - [X] add to systemd
+ - [X] system stop mysql
+ - [X] update mysql.cnf
+ - [X] update database (see gn-services/services/mariadb.md)
+ - [X] check tables
+ + [ ] run gn2 (rust-qtlreaper not working)
+ + [X] update nginx
+ + [ ] install genenetwork3
+ - [ ] add to systemd
diff --git a/doc/elasticsearch.org b/doc/elasticsearch.org
index 18adfc8b..864a8363 100644
--- a/doc/elasticsearch.org
+++ b/doc/elasticsearch.org
@@ -1,5 +1,12 @@
* Elasticsearch
+** Introduction
+
+GeneNetwork uses elasticsearch (ES) for all things considered
+'state'. One example is user collections, another is user management.
+
+** Example
+
To get the right environment, first you can get a python REPL with something like
: env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ../etc/default_settings.py -cli python
@@ -39,3 +46,202 @@ record['hits']['hits'][0][u'_source']['email_address']
u"myname@email.com"
#+END_SRC
+
+** Health
+
+ES provides support for checking its health:
+
+: curl -XGET http://localhost:9200/_cluster/health?pretty=true
+
+#+BEGIN_SRC json
+
+
+ {
+ "cluster_name" : "asgard",
+ "status" : "yellow",
+ "timed_out" : false,
+ "number_of_nodes" : 1,
+ "number_of_data_nodes" : 1,
+ "active_primary_shards" : 5,
+ "active_shards" : 5,
+ "relocating_shards" : 0,
+ "initializing_shards" : 0,
+ "unassigned_shards" : 5
+ }
+
+#+END_SRC
+
+Yellow means just one instance is running (no worries).
+
+To get full cluster info
+
+: curl -XGET "localhost:9200/_cluster/stats?human&pretty"
+
+#+BEGIN_SRC json
+{
+ "_nodes" : {
+ "total" : 1,
+ "successful" : 1,
+ "failed" : 0
+ },
+ "cluster_name" : "elasticsearch",
+ "timestamp" : 1529050366452,
+ "status" : "yellow",
+ "indices" : {
+ "count" : 3,
+ "shards" : {
+ "total" : 15,
+ "primaries" : 15,
+ "replication" : 0.0,
+ "index" : {
+ "shards" : {
+ "min" : 5,
+ "max" : 5,
+ "avg" : 5.0
+ },
+ "primaries" : {
+ "min" : 5,
+ "max" : 5,
+ "avg" : 5.0
+ },
+ "replication" : {
+ "min" : 0.0,
+ "max" : 0.0,
+ "avg" : 0.0
+ }
+ }
+ },
+ "docs" : {
+ "count" : 14579,
+ "deleted" : 0
+ },
+ "store" : {
+ "size" : "44.7mb",
+ "size_in_bytes" : 46892794
+ },
+ "fielddata" : {
+ "memory_size" : "0b",
+ "memory_size_in_bytes" : 0,
+ "evictions" : 0
+ },
+ "query_cache" : {
+ "memory_size" : "0b",
+ "memory_size_in_bytes" : 0,
+ "total_count" : 0,
+ "hit_count" : 0,
+ "miss_count" : 0,
+ "cache_size" : 0,
+ "cache_count" : 0,
+ "evictions" : 0
+ },
+ "completion" : {
+ "size" : "0b",
+ "size_in_bytes" : 0
+ },
+ "segments" : {
+ "count" : 24,
+ "memory" : "157.3kb",
+ "memory_in_bytes" : 161112,
+ "terms_memory" : "122.6kb",
+ "terms_memory_in_bytes" : 125569,
+ "stored_fields_memory" : "15.3kb",
+ "stored_fields_memory_in_bytes" : 15728,
+ "term_vectors_memory" : "0b",
+ "term_vectors_memory_in_bytes" : 0,
+ "norms_memory" : "10.8kb",
+ "norms_memory_in_bytes" : 11136,
+ "points_memory" : "111b",
+ "points_memory_in_bytes" : 111,
+ "doc_values_memory" : "8.3kb",
+ "doc_values_memory_in_bytes" : 8568,
+ "index_writer_memory" : "0b",
+ "index_writer_memory_in_bytes" : 0,
+ "version_map_memory" : "0b",
+ "version_map_memory_in_bytes" : 0,
+ "fixed_bit_set" : "0b",
+ "fixed_bit_set_memory_in_bytes" : 0,
+ "max_unsafe_auto_id_timestamp" : -1,
+ "file_sizes" : { }
+ }
+ },
+ "nodes" : {
+ "count" : {
+ "total" : 1,
+ "data" : 1,
+ "coordinating_only" : 0,
+ "master" : 1,
+ "ingest" : 1
+ },
+ "versions" : [
+ "6.2.1"
+ ],
+ "os" : {
+ "available_processors" : 16,
+ "allocated_processors" : 16,
+ "names" : [
+ {
+ "name" : "Linux",
+ "count" : 1
+ }
+ ],
+ "mem" : {
+ "total" : "125.9gb",
+ "total_in_bytes" : 135189286912,
+ "free" : "48.3gb",
+ "free_in_bytes" : 51922628608,
+ "used" : "77.5gb",
+ "used_in_bytes" : 83266658304,
+ "free_percent" : 38,
+ "used_percent" : 62
+ }
+ },
+ "process" : {
+ "cpu" : {
+ "percent" : 0
+ },
+ "open_file_descriptors" : {
+ "min" : 415,
+ "max" : 415,
+ "avg" : 415
+ }
+ },
+ "jvm" : {
+ "max_uptime" : "1.9d",
+ "max_uptime_in_millis" : 165800616,
+ "versions" : [
+ {
+ "version" : "9.0.4",
+ "vm_name" : "OpenJDK 64-Bit Server VM",
+ "vm_version" : "9.0.4+11",
+ "vm_vendor" : "Oracle Corporation",
+ "count" : 1
+ }
+ ],
+ "mem" : {
+ "heap_used" : "1.1gb",
+ "heap_used_in_bytes" : 1214872032,
+ "heap_max" : "23.8gb",
+ "heap_max_in_bytes" : 25656426496
+ },
+ "threads" : 110
+ },
+ "fs" : {
+ "total" : "786.4gb",
+ "total_in_bytes" : 844400918528,
+ "free" : "246.5gb",
+ "free_in_bytes" : 264688160768,
+ "available" : "206.5gb",
+ "available_in_bytes" : 221771468800
+ },
+ "plugins" : [ ],
+ "network_types" : {
+ "transport_types" : {
+ "netty4" : 1
+ },
+ "http_types" : {
+ "netty4" : 1
+ }
+ }
+ }
+}
+#+BEGIN_SRC json