aboutsummaryrefslogtreecommitdiff
path: root/wqflask/utility/elasticsearch_tools.py
blob: 76dcaebf5823a4c3ac0ca666ba44fb98db6b4445 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
# Elasticsearch support
#
# Some helpful commands to view the database:
#
# You can test the server being up with
#
#   curl -H 'Content-Type: application/json' http://localhost:9200
#
# List all indices
#
#   curl -H 'Content-Type: application/json' 'localhost:9200/_cat/indices?v'
#
# To see the users index 'table'
#
#   curl http://localhost:9200/users
#
# To list all user ids
#
# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d '
# {
#     "query" : {
#         "match_all" : {}
#     },
#     "stored_fields": []
# }'
#
# To view a record
#
#   curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d '
#   {
#     "query" : {
#       "match" : { "email_address": "pjotr2017@thebird.nl"}
#     }
#   }'
#
#
# To delete the users index and data (dangerous!)
#
#   curl -XDELETE -H 'Content-Type: application/json' 'localhost:9200/users'


from elasticsearch import Elasticsearch, TransportError
import logging

from utility.logger import getLogger
logger = getLogger(__name__)

from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT

def test_elasticsearch_connection():
    es = Elasticsearch(['http://'+ELASTICSEARCH_HOST+":"+str(ELASTICSEARCH_PORT)+'/'], verify_certs=True)
    if not es.ping():
        logger.warning("Elasticsearch is DOWN")

def get_elasticsearch_connection():
    """Return a connection to ES. Returns None on failure"""
    logger.info("get_elasticsearch_connection")
    es = None
    try:
        assert(ELASTICSEARCH_HOST)
        assert(ELASTICSEARCH_PORT)
        logger.info("ES HOST",ELASTICSEARCH_HOST)

        es = Elasticsearch([{
            "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT
        }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None

        setup_users_index(es)

        es_logger = logging.getLogger("elasticsearch")
        es_logger.setLevel(logging.INFO)
        es_logger.addHandler(logging.NullHandler())
    except Exception as e:
        logger.error("Failed to get elasticsearch connection", e)
        es = None

    return es

def setup_users_index(es_connection):
    if es_connection:
        index_settings = {
            "properties": {
                "email_address": {
                    "type": "keyword"}}}

        es_connection.indices.create(index='users', ignore=400)
        es_connection.indices.close(index="users")
        es_connection.indices.put_mapping(body=index_settings, index="users", doc_type="local")
        es_connection.indices.open(index="users")

def get_user_by_unique_column(es, column_name, column_value, index="users", doc_type="local"):
    return get_item_by_unique_column(es, column_name, column_value, index=index, doc_type=doc_type)

def save_user(es, user, user_id):
    es_save_data(es, "users", "local", user, user_id)

def get_item_by_unique_column(es, column_name, column_value, index, doc_type):
    item_details = None
    try:
        response = es.search(
            index = index, doc_type = doc_type, body = {
                "query": { "match": { column_name: column_value } }
            })
        if len(response["hits"]["hits"]) > 0:
            item_details = response["hits"]["hits"][0]["_source"]
    except TransportError as te:
        pass
    return item_details

def es_save_data(es, index, doc_type, data_item, data_id,):
    from time import sleep
    es.create(index, doc_type, body=data_item, id=data_id)
    sleep(1) # Delay 1 second to allow indexing