aboutsummaryrefslogtreecommitdiff
path: root/wqflask
diff options
context:
space:
mode:
Diffstat (limited to 'wqflask')
-rw-r--r--wqflask/maintenance/quantile_normalize.py18
-rw-r--r--wqflask/utility/elasticsearch_tools.py121
-rw-r--r--wqflask/utility/tools.py5
-rw-r--r--wqflask/wqflask/user_session.py1
4 files changed, 1 insertions, 144 deletions
diff --git a/wqflask/maintenance/quantile_normalize.py b/wqflask/maintenance/quantile_normalize.py
index 0cc963e5..32780ca6 100644
--- a/wqflask/maintenance/quantile_normalize.py
+++ b/wqflask/maintenance/quantile_normalize.py
@@ -5,14 +5,10 @@ import urllib.parse
import numpy as np
import pandas as pd
-from elasticsearch import Elasticsearch, TransportError
-from elasticsearch.helpers import bulk
from flask import Flask, g, request
from wqflask import app
-from utility.elasticsearch_tools import get_elasticsearch_connection
-from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT, SQL_URI
def parse_db_uri():
@@ -106,20 +102,6 @@ if __name__ == '__main__':
Conn = MySQLdb.Connect(**parse_db_uri())
Cursor = Conn.cursor()
- # es = Elasticsearch([{
- # "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT
- # }], timeout=60) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None
-
- es = get_elasticsearch_connection(for_user=False)
-
- #input_filename = "/home/zas1024/cfw_data/" + sys.argv[1] + ".txt"
- #input_df = create_dataframe(input_filename)
- #output_df = quantileNormalize(input_df)
-
- #output_df.to_csv('quant_norm.csv', sep='\t')
-
- #out_filename = sys.argv[1][:-4] + '_quantnorm.txt'
-
success, _ = bulk(es, set_data(sys.argv[1]))
response = es.search(
diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py
deleted file mode 100644
index eae3ba03..00000000
--- a/wqflask/utility/elasticsearch_tools.py
+++ /dev/null
@@ -1,121 +0,0 @@
-# Elasticsearch support
-#
-# Some helpful commands to view the database:
-#
-# You can test the server being up with
-#
-# curl -H 'Content-Type: application/json' http://localhost:9200
-#
-# List all indices
-#
-# curl -H 'Content-Type: application/json' 'localhost:9200/_cat/indices?v'
-#
-# To see the users index 'table'
-#
-# curl http://localhost:9200/users
-#
-# To list all user ids
-#
-# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d '
-# {
-# "query" : {
-# "match_all" : {}
-# },
-# "stored_fields": []
-# }'
-#
-# To view a record
-#
-# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d '
-# {
-# "query" : {
-# "match" : { "email_address": "pjotr2017@thebird.nl"}
-# }
-# }'
-#
-#
-# To delete the users index and data (dangerous!)
-#
-# curl -XDELETE -H 'Content-Type: application/json' 'localhost:9200/users'
-
-
-from elasticsearch import Elasticsearch, TransportError
-import logging
-
-from utility.logger import getLogger
-logger = getLogger(__name__)
-
-from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT
-
-
-def test_elasticsearch_connection():
- es = Elasticsearch(['http://' + ELASTICSEARCH_HOST + \
- ":" + str(ELASTICSEARCH_PORT) + '/'], verify_certs=True)
- if not es.ping():
- logger.warning("Elasticsearch is DOWN")
-
-
-def get_elasticsearch_connection(for_user=True):
- """Return a connection to ES. Returns None on failure"""
- logger.info("get_elasticsearch_connection")
- es = None
- try:
- assert(ELASTICSEARCH_HOST)
- assert(ELASTICSEARCH_PORT)
- logger.info("ES HOST", ELASTICSEARCH_HOST)
-
- es = Elasticsearch([{
- "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT
- }], timeout=30, retry_on_timeout=True) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None
-
- if for_user:
- setup_users_index(es)
-
- es_logger = logging.getLogger("elasticsearch")
- es_logger.setLevel(logging.INFO)
- es_logger.addHandler(logging.NullHandler())
- except Exception as e:
- logger.error("Failed to get elasticsearch connection", e)
- es = None
-
- return es
-
-
-def setup_users_index(es_connection):
- if es_connection:
- index_settings = {
- "properties": {
- "email_address": {
- "type": "keyword"}}}
-
- es_connection.indices.create(index='users', ignore=400)
- es_connection.indices.put_mapping(
- body=index_settings, index="users", doc_type="local")
-
-
-def get_user_by_unique_column(es, column_name, column_value, index="users", doc_type="local"):
- return get_item_by_unique_column(es, column_name, column_value, index=index, doc_type=doc_type)
-
-
-def save_user(es, user, user_id):
- es_save_data(es, "users", "local", user, user_id)
-
-
-def get_item_by_unique_column(es, column_name, column_value, index, doc_type):
- item_details = None
- try:
- response = es.search(
- index=index, doc_type=doc_type, body={
- "query": {"match": {column_name: column_value}}
- })
- if len(response["hits"]["hits"]) > 0:
- item_details = response["hits"]["hits"][0]["_source"]
- except TransportError as te:
- pass
- return item_details
-
-
-def es_save_data(es, index, doc_type, data_item, data_id,):
- from time import sleep
- es.create(index, doc_type, body=data_item, id=data_id)
- sleep(1) # Delay 1 second to allow indexing
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 0efe8ca9..f28961ec 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -287,6 +287,7 @@ JS_GN_PATH = get_setting('JS_GN_PATH')
GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID')
GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET')
+GITHUB_AUTH_URL = ""
if GITHUB_CLIENT_ID != 'UNKNOWN' and GITHUB_CLIENT_SECRET:
GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id=" + \
GITHUB_CLIENT_ID + "&client_secret=" + GITHUB_CLIENT_SECRET
@@ -301,10 +302,6 @@ if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET:
"&redirect_uri=" + GN2_BRANCH_URL + "n/login/orcid_oauth2"
ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL')
-ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST')
-ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT')
-# import utility.elasticsearch_tools as es
-# es.test_elasticsearch_connection()
SMTP_CONNECT = get_setting('SMTP_CONNECT')
SMTP_USERNAME = get_setting('SMTP_USERNAME')
diff --git a/wqflask/wqflask/user_session.py b/wqflask/wqflask/user_session.py
index 67e2e158..d3c4a62f 100644
--- a/wqflask/wqflask/user_session.py
+++ b/wqflask/wqflask/user_session.py
@@ -10,7 +10,6 @@ from flask import (Flask, g, render_template, url_for, request, make_response,
from wqflask import app
from utility import hmac
-#from utility.elasticsearch_tools import get_elasticsearch_connection
from utility.redis_tools import get_redis_conn, get_user_id, get_user_by_unique_column, set_user_attribute, get_user_collections, save_collections
Redis = get_redis_conn()
generated by cgit v1.2.3 (git 2.44.0) at 2024-11-23 02:54:22 +0000