Merge pull request #607 from Alexanderlacuna/feature/integrate-gn3-wgcna

test  template for gn3 wgcna

7 files changed, 476 insertions, 47 deletions

diff --git a/wqflask/tests/unit/wqflask/wgcna/__init__.py b/wqflask/tests/unit/wqflask/wgcna/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/wqflask/tests/unit/wqflask/wgcna/__init__.py
diff --git a/wqflask/tests/unit/wqflask/wgcna/test_wgcna.py b/wqflask/tests/unit/wqflask/wgcna/test_wgcna.py
new file mode 100644
index 00000000..8e947e2f
--- /dev/null
+++ b/wqflask/tests/unit/wqflask/wgcna/test_wgcna.py
@@ -0,0 +1,50 @@
+
+"""module contains for processing gn3 wgcna data"""
+from unittest import TestCase
+
+from wqflask.wgcna.gn3_wgcna import process_wgcna_data
+
+
+class DataProcessingTests(TestCase):
+    """class contains data processing tests"""
+
+    def test_data_processing(self):
+        """test for parsing data for datatable"""
+        output = {
+            "input": {
+                "sample_names": ["BXD1", "BXD2", "BXD3", "BXD4", "BXD5", "BXD6"],
+
+            },
+            "output": {
+                "ModEigens": {
+                    "MEturquoise": [
+                        0.0646677768085351,
+                        0.137200224277058,
+                        0.63451113720732,
+                        -0.544002665501479,
+                        -0.489487590361863,
+                        0.197111117570427
+                    ],
+                    "MEgrey": [
+                        0.213,
+                        0.214,
+                        0.3141,
+                        -0.545,
+                        -0.423,
+                        0.156,
+                    ]
+                }}}
+
+        row_data = [['BXD1', 0.065, 0.213],
+                    ['BXD2', 0.137, 0.214],
+                    ['BXD3', 0.635, 0.314],
+                    ['BXD4', -0.544, -0.545],
+                    ['BXD5', -0.489, -0.423],
+                    ['BXD6', 0.197, 0.156]]
+
+        expected_results = {
+            "col_names": ["sample_names", "MEturquoise", "MEgrey"],
+            "mod_dataset": row_data
+        }
+
+        self.assertEqual(process_wgcna_data(output), expected_results)
diff --git a/wqflask/wqflask/templates/test_wgcna_results.html b/wqflask/wqflask/templates/test_wgcna_results.html
new file mode 100644
index 00000000..1dddd393
--- /dev/null
+++ b/wqflask/wqflask/templates/test_wgcna_results.html
@@ -0,0 +1,158 @@
+{% extends "base.html" %}
+{% block title %}WCGNA results{% endblock %}
+{% block content %}
+<!-- Start of body -->
+
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/xterm/3.14.5/xterm.min.css" integrity="sha512-iLYuqv+v/P4u9erpk+KM83Ioe/l7SEmr7wB6g+Kg1qmEit8EShDKnKtLHlv2QXUp7GGJhmqDI+1PhJYLTsfb8w==" crossorigin="anonymous" referrerpolicy="no-referrer" />
+
+<link rel="stylesheet" href="https://cdn.datatables.net/1.11.3/css/jquery.dataTables.min.css">
+
+
+<style type="text/css">
+
+
+.container {
+    min-height: 100vh;
+    width: 100vw;
+    padding: 20px;
+
+}
+
+.grid_container {
+
+
+    width: 80vw;
+    margin: auto;
+    padding: 20px;
+
+
+    display: grid;
+    grid-template-columns: repeat(7, 1fr);
+    /*grid-gap: 5px;*/
+    border: 1px solid black;
+    grid-column-gap: 20px;
+
+}
+
+.control_sft_column {
+    text-align: center;
+}
+
+.grid_container div:not(:last-child) {
+    border-right: 1px solid #000;
+}
+
+.grid_container .control_sft_column h3 {
+    font-weight: bold;
+    font-size: 18px;
+}
+
+.control_net_colors {
+
+    display: flex;
+    flex-wrap: wrap;
+    justify-content: space-between;
+    align-items: center;
+    text-align: center;
+}
+
+
+.control_mod_eigens {
+    display: grid;
+    grid-template-columns: repeat(2, 200px);
+}
+
+.control-image{
+  display: block;
+  margin-left: auto;
+  margin-right: auto;
+  width: 80vw;
+}
+</style>
+<div class="container">
+    <div>
+        <div >
+            <h2 style="text-align:center">Soft Thresholds </h2>
+            <div class="grid_container">
+
+            {% for key, value in results["data"]["output"]["soft_threshold"].items()%}
+            <div class="control_sft_column">
+                <h3>{{key}}</h3>
+                {% for val in value %}
+                <p>{{val|round(3)}}</p>
+                {% endfor %}
+            </div>
+            {% endfor %}
+            </div>
+        </div>
+
+        <div>
+        
+            {% if image["image_generated"] %}
+            <div >
+                <img class="control-image" src="data:image/jpeg;base64,{{ image['image_data']| safe }}">
+            </div>
+
+            {% endif %}
+<!--                 <div >
+                <img class="control-image" src="data:image/jpeg;base64,{{ results['data']['output']['image_data2']| safe }}">
+            </div> -->
+        </div>
+
+        <div>
+            <h2 style="text-align:center;"> Module eigen genes </h2>
+            <table id="eigens" class="display" width="80vw"></table>
+        </div>
+
+        <div>
+            <h2 style="text-align:center;">Phenotype modules </h2>
+
+             <table id="phenos" class="display"  width="40vw" ></table>
+        </div>
+    </div>
+</div>
+
+{% endblock %}
+
+{% block js %}
+
+<script src="https://cdnjs.cloudflare.com/ajax/libs/xterm/3.14.5/xterm.min.js"></script>
+
+<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.js') }}"></script>
+<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.dataTables.min.js') }}"></script>
+<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/scroller/js/dataTables.scroller.min.js') }}"></script>
+
+
+<script type="text/javascript">
+
+
+let results = {{results|safe}}
+
+let phenoModules = results["data"]["output"]["net_colors"]
+let phenotypes = Object.keys(phenoModules)
+let phenoMods = Object.values(phenoModules)
+
+let {col_names,mod_dataset} = {{data|safe}}
+    $('#eigens').DataTable( {
+        data: mod_dataset,
+        columns: col_names.map((name)=>{
+            return {
+                title:name
+            }
+        })
+    } );
+    $('#phenos').DataTable( {
+        data:phenotypes.map((phenoName,idx)=>{
+            return [phenoName,phenoMods[idx]]
+        }),
+        columns: [{
+            title:"Phenotypes"
+        },
+        {
+        title: "Modules"
+        }]
+    } );
+
+
+</script>
+{% endblock %}
\ No newline at end of file
diff --git a/wqflask/wqflask/templates/tool_buttons.html b/wqflask/wqflask/templates/tool_buttons.html
index 3f9d8211..3ee5be19 100644
--- a/wqflask/wqflask/templates/tool_buttons.html
+++ b/wqflask/wqflask/templates/tool_buttons.html
@@ -18,13 +18,13 @@
   BNW
 </button>
 
-<!-- <button id="wgcna_setup" class="btn btn-primary submit_special" data-url="/wgcna_setup" title="WGCNA Analysis" >
+<button id="wgcna_setup" class="btn btn-primary submit_special" data-url="/wgcna_setup" title="WGCNA Analysis" >
     WGCNA
 </button>
 
 <button id="ctl_setup" class="btn btn-primary submit_special" data-url="/ctl_setup" title="CTL Analysis" >
     CTL Maps
-</button> -->
+</button>
 
 <button id="heatmap" class="btn btn-primary submit_special" data-url="/heatmap" title="Heatmap" >
     MultiMap
diff --git a/wqflask/wqflask/templates/wgcna_setup.html b/wqflask/wqflask/templates/wgcna_setup.html
index c5461497..86d9fa10 100644
--- a/wqflask/wqflask/templates/wgcna_setup.html
+++ b/wqflask/wqflask/templates/wgcna_setup.html
@@ -1,49 +1,161 @@
 {% extends "base.html" %}
 {% block title %}WCGNA analysis{% endblock %}
+{% block content %}
+<!-- Start of body -->
+<style type="text/css">
+
+#terminal {
+    margin-top: 10px;
+}
+
+</style>
+<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/xterm@4.14.1/css/xterm.css">
 
-{% block content %} <!-- Start of body -->
-<h1> WGCNA analysis parameters</h1>
 <div class="container">
-  {% if request.form['trait_list'].split(",")|length < 4 %}
-  <div class="alert alert-danger" role="alert">
-    <span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span>
-    <span class="sr-only">Error:</span>
-    <h2>Too few phenotypes as input</h2>
-    Please make sure you select enough phenotypes / genes to perform WGCNA. Your collection needs to contain at least 4 different phenotypes. You provided {{request.form['trait_list'].split(',')|length}} phenotypes as input.
-  </div>
-  {% else %}
-  <form action="/wgcna_results" method="post" class="form-horizontal">
-    <input type="hidden" name="trait_list" id="trait_list" value= "{{request.form['trait_list']}}">
-    <div class="form-group">
-      <label for="SoftThresholds"> Soft threshold: </label>
-      <div class="col-sm-10">
-        <input type="text" class="form-inline" name="SoftThresholds" id="SoftThresholds" value="1,2,3,4,5,6,7,8,9">
-      </div>
-    </div>
-    <div class="form-group">
-      <label for="MinModuleSize"> Minimum module size: </label>
-      <div class="col-sm-10">
-        <input type="text" class="form-inline" name="MinModuleSize" id="MinModuleSize" value="30">
-      </div>
-    </div>
-    <div class="form-group">
-      <label for="TOMtype"> TOMtype: </label>
-      <div class="col-sm-10">
-        <input type="text" class="form-inline" name="TOMtype" id="TOMtype" value="unsigned">
-      </div>
-    </div>
-    <div class="form-group">
-      <label for="mergeCutHeight"> mergeCutHeight: </label>
-      <div class="col-sm-10">
-        <input type="text" class="form-inline" name="mergeCutHeight" id="mergeCutHeight" value="0.25">
-      </div>
+    <div class="col-md-5">
+        <h1 class="mx-3 my-2 "> WGCNA analysis parameters</h1>
+        {% if request.form['trait_list'].split(",")|length < 4 %} <div class="alert alert-danger" role="alert">
+            <span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span>
+            <span class="sr-only">Error:</span>
+            <h2>Too few phenotypes as input</h2>
+            Please make sure you select enough phenotypes / genes to perform WGCNA. Your collection needs to contain at least 4 different phenotypes. You provided {{request.form['trait_list'].split(',')|length}} phenotypes as input.
     </div>
-    <div class="form-group">
-      <div class="col-sm-10">
-        <input type="submit" class="btn btn-primary" value="Run WGCNA using these settings" />
-      </div>
-    </div>
-  </form>
-  {% endif %}
+    {% else %}
+    <form class="col-md-12" action="/wgcna_results" method="post" class="form-horizontal" id="wgcna_form">
+        <input type="hidden" name="trait_list" id="trait_list" value="{{request.form['trait_list']}}">
+        <div class="form-group row ">
+            <label for="SoftThresholds" class="col-md-3 col-form-label col-form-label-sm">Soft threshhold</label>
+            <div class="col-md-9">
+                <input type="text" class="form-control form-control-md" value="1,2,3,4,5,6,7,8,9" id="SoftThresholds" name="SoftThresholds">
+            </div>
+        </div>
+        <div class="form-group row ">
+            <label for="MinModuleSize" class="col-md-3 col-form-label col-form-label-sm">Minimum module size:</label>
+            <div class="col-md-9">
+                <input type="text" class="form-control form-control-md" id="MinModuleSize" value="30" name="MinModuleSize">
+            </div>
+        </div>
+
+        <div class="form-group row">
+          <label for="TOMtype" class="col-md-3 col-form-label col-form-label-sm">TOMtype:</label>
+          <div class="col-md-9">
+            <select class="form-control" id="TOMtype" name="TOMtype">
+            <option value="unsigned">unsigned</option>
+            <option value="signed">signed</option>
+          </select>
+          </div>
+
+        </div>
+        <div class="form-group row ">
+            <label for="mergeCutHeight" class="col-md-3 col-form-label col-form-label-sm">mergeCutHeight:</label>
+            <div class="col-md-9">
+                <input type="text" class="form-control form-control-md" id="mergeCutHeight" value="0.25" name="mergeCutHeight">
+            </div>
+        </div>
+
+        <div class="form-group row">
+          <label for="corType" class="col-md-3 col-form-label col-form-label-sm">corType:</label>
+          <div class="col-md-9">
+            <select class="form-control  col-md-9" id="corType" name="corType">
+            <option value="pearson">pearson</option>
+            <option value="bicor">bicor</option>
+          </select>
+          </div>
+
+        </div>
+        <div class="form-group">
+            <div class="text-center">
+                <input type="submit" class="btn btn-primary" value="Run WGCNA using these settings" />
+            </div>
+        </div>
+
+
+
+    </form>
+    {% endif %}
 </div>
-{% endblock %}
+<div class="col-md-7">
+        <div id="terminal" class="mt-2">
+        </div>
+</div>
+</div>
+
+<script src="https://cdn.socket.io/4.2.0/socket.io.min.js" integrity="sha384-PiBR5S00EtOj2Lto9Uu81cmoyZqR57XcOna1oAuVuIEjzj0wpqDVfD0JA9eXlRsj" crossorigin="anonymous"></script>
+
+<script src="https://cdn.jsdelivr.net/npm/xterm@4.14.1/lib/xterm.min.js"></script>
+
+<script src="https://cdn.jsdelivr.net/npm/xterm-addon-attach@0.6.0/lib/xterm-addon-attach.min.js"></script>
+
+
+<script src="https://cdn.jsdelivr.net/npm/xterm-addon-fit@0.5.0/lib/xterm-addon-fit.min.js"></script>
+ <script src="https://code.jquery.com/jquery-3.5.1.js"
+          integrity="sha256-QWo7LDvxbWT2tbbQ97B53yJnYU3WhH/C8ycbRAkjPDc="
+          crossorigin="anonymous"></script>
+<script>
+// document.addEventListener('DOMContentLoaded', function() {
+let term = new Terminal({
+    cursorBlink: true,
+    lineHeight: 1.3,
+    scrollback: true,
+    macOptionIsMeta: true
+});
+
+let termDebugs = {
+    general: "Computation process to be displayed here....",
+    success: "Computation in process ......",
+    fail: "Too few phenotypes as input must be >=4"
+}
+
+const fitAddon = new FitAddon.FitAddon()
+term.loadAddon(fitAddon)
+
+term.open(document.getElementById('terminal'));
+term.setOption('theme', {
+    background: '#300a24'
+});
+term.writeln(termDebugs.general)
+
+wgcnaForm = document.querySelector("#wgcna_form")
+
+const socket = io("http://127.0.0.1:8081") //issue gn3 private
+const attachAddon = new AttachAddon.AttachAddon(socket);
+
+term.loadAddon(attachAddon);
+
+fitAddon.fit()
+term.onData((data) => {
+    term.write(data)
+})
+
+
+if (wgcnaForm) {
+} else {
+    term.writeln(termDebugs.fail)
+}
+
+socket.on("connect", () => {
+    $("#wgcna_form").append(`<input type="hidden" name="socket_id" value=${socket.id}>`);
+})
+
+socket.on("output", ({
+    data
+}) => {
+    term.writeln(data)
+})
+
+$(document).on('submit', '#wgcna_form', function(e) {
+    term.writeln(termDebugs.success)
+
+    e.preventDefault();
+    var form = $(this);
+    $.ajax({
+        type: 'POST',
+        url: '/wgcna_results',
+        data: form.serialize(),
+        success: function(data) {
+            document.write(data)
+        }
+    })
+});
+</script>
+{% endblock %}
\ No newline at end of file
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 5067ca0e..b0da1f21 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -40,8 +40,8 @@ from gn3.db.phenotypes import Probeset
 from gn3.db.phenotypes import Publication
 from gn3.db.phenotypes import PublishXRef
 from gn3.db.phenotypes import probeset_mapping
-from gn3.db.traits import get_trait_csv_sample_data
-from gn3.db.traits import update_sample_data
+# from gn3.db.traits import get_trait_csv_sample_data
+# from gn3.db.traits import update_sample_data
 
 
 from flask import current_app
@@ -79,6 +79,7 @@ from wqflask.correlation_matrix import show_corr_matrix
 from wqflask.correlation import corr_scatter_plot
 # from wqflask.wgcna import wgcna_analysis
 # from wqflask.ctl import ctl_analysis
+from wqflask.wgcna.gn3_wgcna import run_wgcna
 from wqflask.snp_browser import snp_browser
 from wqflask.search_results import SearchResultPage
 from wqflask.export_traits import export_search_results_csv
@@ -376,6 +377,12 @@ def wcgna_setup():
     return render_template("wgcna_setup.html", **request.form)
 
 
+@app.route("/wgcna_results", methods=('POST',))
+def wcgna_results():
+    """call the gn3 api to get wgcna response data"""
+    results = run_wgcna(dict(request.form))
+    return render_template("test_wgcna_results.html", **results)
+
 @app.route("/ctl_setup", methods=('POST',))
 def ctl_setup():
     # We are going to get additional user input for the analysis
diff --git a/wqflask/wqflask/wgcna/gn3_wgcna.py b/wqflask/wqflask/wgcna/gn3_wgcna.py
new file mode 100644
index 00000000..c4cc2e7f
--- /dev/null
+++ b/wqflask/wqflask/wgcna/gn3_wgcna.py
@@ -0,0 +1,102 @@
+"""module contains code to consume gn3-wgcna api
+and process data to be rendered by datatables
+"""
+
+import requests
+from types import SimpleNamespace
+from utility.helper_functions import get_trait_db_obs
+
+
+def fetch_trait_data(requestform):
+    """fetch trait data"""
+    db_obj = SimpleNamespace()
+    get_trait_db_obs(db_obj,
+                     [trait.strip()
+                      for trait in requestform['trait_list'].split(',')])
+
+    return process_dataset(db_obj.trait_list)
+
+
+def process_dataset(trait_list):
+    """process datasets and strains"""
+
+    input_data = {}
+    traits = []
+    strains = []
+
+    # xtodo unique traits and strains
+
+    for trait in trait_list:
+        traits.append(trait[0].name)
+
+        input_data[trait[0].name] = {}
+        for strain in trait[0].data:
+            strains.append(strain)
+            input_data[trait[0].name][strain] = trait[0].data[strain].value
+        # "sample_names": list(set(strains)),
+        # "trait_names": form_traits,
+        # "trait_sample_data": form_strains,
+
+    return {
+        "input": input_data,
+        "trait_names": traits,
+        "sample_names": strains
+    }
+
+
+def process_wgcna_data(response):
+    """function for processing modeigene genes
+    for create row data for datataba"""
+    mod_eigens = response["output"]["ModEigens"]
+
+    sample_names = response["input"]["sample_names"]
+
+    mod_dataset = [[sample] for sample in sample_names]
+
+    for _, mod_values in mod_eigens.items():
+        for (index, _sample) in enumerate(sample_names):
+            mod_dataset[index].append(round(mod_values[index], 3))
+
+    return {
+        "col_names": ["sample_names", *mod_eigens.keys()],
+        "mod_dataset": mod_dataset
+    }
+
+
+def process_image(response):
+    """function to process image check if byte string is empty"""
+    image_data = response["output"]["image_data"]
+    return ({
+        "image_generated": True,
+        "image_data": image_data
+    } if image_data else {
+        "image_generated": False
+    })
+
+
+def run_wgcna(form_data):
+    """function to run wgcna"""
+
+    GN3_URL = "http://127.0.0.1:8081"
+
+    wgcna_api = f"{GN3_URL}/api/wgcna/run_wgcna"
+
+    # parse form data
+
+    trait_dataset = fetch_trait_data(form_data)
+    form_data["minModuleSize"] = int(form_data["MinModuleSize"])
+
+    response = requests.post(wgcna_api, json={
+        "sample_names": list(set(trait_dataset["sample_names"])),
+        "trait_names": trait_dataset["trait_names"],
+        "trait_sample_data": list(trait_dataset["input"].values()),
+        **form_data
+
+    }
+    ).json()
+
+    return {
+        "results": response,
+        "data": process_wgcna_data(response["data"]),
+        "image": process_image(response["data"])
+    }