Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CHECK-2437 add support for using analyzers by language #258

Merged
merged 5 commits into from
Sep 28, 2022
266 −78
Merged

CHECK-2437 add support for using analyzers by language #258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift click to select a range
bd7f88e
CHECK-2437 add support for using analyzers by language
DGaffney Sep 23, 2022
14f7a9b
CHECK-2437 remove old dependencies from half-implementation of analyzers
DGaffney Sep 23, 2022
75c32b2
CHECK-2437 shift es client
DGaffney Sep 23, 2022
dbfe4d8
CHECK-2437 add tests for new use case
DGaffney Sep 26, 2022
2474023
CHECK-2437 add fix for tests to actually pass
DGaffney Sep 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion app/main/controller/bulk_similarity_controller.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 3,6 @@
from elasticsearch import Elasticsearch
from elasticsearch import helpers
from app.main.lib.fields import JsonObject
from app.main.lib.elasticsearch import language_to_analyzer
from app.main.lib.shared_models.shared_model import SharedModel
from app.main.lib.text_similarity import get_document_body
from app.main.lib import similarity
Expand Down
1 change: 0 additions & 1 deletion app/main/controller/bulk_update_similarity_controller.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 3,6 @@
from elasticsearch import Elasticsearch
from elasticsearch import helpers
from app.main.lib.fields import JsonObject
from app.main.lib.elasticsearch import language_to_analyzer
from app.main.lib.shared_models.shared_model import SharedModel
from app.main.controller.bulk_similarity_controller import BulkSimilarityResource

Expand Down
101 changes: 37 additions & 64 deletions app/main/lib/elasticsearch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 5,8 @@
from elasticsearch.helpers import scan

from flask import request, current_app as app

from app.main.lib.language_analyzers import SUPPORTED_LANGUAGES
def get_all_documents_matching_context(context):
matches, clause_count = generate_matches(context)
es = Elasticsearch(app.config['ELASTICSEARCH_URL'], timeout=30)
Expand Down Expand Up @@ -68,31 70,41 @@ def merge_contexts(body, found_doc):
body["contexts"].append(context)
return body

def store_document(body, doc_id):
es = Elasticsearch(app.config['ELASTICSEARCH_URL'])
if doc_id:
try:
found_doc = es.get(index=app.config['ELASTICSEARCH_SIMILARITY'], id=doc_id)
except elasticsearch.exceptions.NotFoundError:
found_doc = None
if found_doc:
result = es.update(
id=doc_id,
body={"doc": merge_contexts(body, found_doc)},
index=app.config['ELASTICSEARCH_SIMILARITY']
)
else:
result = es.index(
id=doc_id,
body=body,
index=app.config['ELASTICSEARCH_SIMILARITY']
)
else:
result = es.index(
body=body,
index=app.config['ELASTICSEARCH_SIMILARITY']
)
# es.indices.refresh(index=app.config['ELASTICSEARCH_SIMILARITY'])
def update_or_create_document(body, doc_id, index):
es = Elasticsearch(app.config['ELASTICSEARCH_URL'], timeout=30)
result = None
if doc_id:
try:
found_doc = es.get(index=index, id=doc_id)
except elasticsearch.exceptions.NotFoundError:
found_doc = None
if found_doc:
result = es.update(
id=doc_id,
body={"doc": merge_contexts(body, found_doc)},
index=index
)
else:
result = es.index(
id=doc_id,
body=body,
index=index
)
else:
result = es.index(
body=body,
index=index
)
return result

def store_document(body, doc_id, language=None):
indices = [app.config['ELASTICSEARCH_SIMILARITY']]
if language and language in SUPPORTED_LANGUAGES:
indices.append(app.config['ELASTICSEARCH_SIMILARITY'] "_" language)
results = []
for index in indices:
results.append(update_or_create_document(body, doc_id, index))
result = results[0]
success = False
if result['result'] == 'created' or result['result'] == 'updated':
success = True
Expand Down Expand Up @@ -128,42 140,3 @@ def delete_document(doc_id, context, quiet):
}
else:
return False

def language_to_analyzer(lang):
analyzer_dict = {
'ar': 'arabic',
'hy': 'armenian',
'eu': 'basque',
'bn': 'bengali',
'pt-br': 'brazilian', # TODO
'bg': 'bulgarian',
'ca': 'catalan',
'cjk': 'cjk', # TODO
'cs': 'czech',
'da': 'danish',
'nl': 'dutch',
'en': 'english',
'fi': 'finnish',
'fr': 'french',
'gl': 'galician',
'de': 'german',
'gr': 'greek',
'hi': 'hindi',
'hu': 'hungarian',
'id': 'indonesian',
'ga': 'irish',
'it': 'italian',
'lv': 'latvian',
'lt': 'lithuanian',
'no': 'norwegian',
'fa': 'persian',
'pt': 'portuguese',
'ro': 'romanian',
'ru': 'russian',
'ku': 'sorani',
'es': 'spanish',
'sv': 'swedish',
'tr': 'turkish',
'th': 'thai'
}
return analyzer_dict.get(lang, 'standard')
183 changes: 183 additions & 0 deletions app/main/lib/language_analyzers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 1,183 @@
import json
from elasticsearch import Elasticsearch
from flask import request, current_app as app
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unable to import 'flask'

SUPPORTED_LANGUAGES = ["en", "pt", "es", "hi", "bn"]
#via https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-lang-analyzer.html#bengali-analyzer
SETTINGS_BY_LANGUAGE = {
"en": {
"analysis": {
"filter": {
"english_stop": {
"type": "stop",
"stopwords": "_english_"
},
"english_keywords": {
"type": "keyword_marker",
"keywords": ["example"]
},
"english_stemmer": {
"type": "stemmer",
"language": "english"
},
"english_possessive_stemmer": {
"type": "stemmer",
"language": "possessive_english"
}
},
"analyzer": {
"rebuilt_english": {
"tokenizer": "standard",
"filter": [
"english_possessive_stemmer",
"lowercase",
"english_stop",
"english_keywords",
"english_stemmer"
]
}
}
}
},
"es": {
DGaffney marked this conversation as resolved.
Show resolved Hide resolved
"analysis": {
"filter": {
"spanish_stop": {
"type": "stop",
"stopwords": "_spanish_"
},
"spanish_keywords": {
"type": "keyword_marker",
"keywords": ["ejemplo"]
},
"spanish_stemmer": {
"type": "stemmer",
"language": "light_spanish"
}
},
"analyzer": {
"rebuilt_spanish": {
"tokenizer": "standard",
"filter": [
"lowercase",
"spanish_stop",
"spanish_keywords",
"spanish_stemmer"
]
}
}
}
},
"pt": {
DGaffney marked this conversation as resolved.
Show resolved Hide resolved
"analysis": {
"filter": {
"portuguese_stop": {
"type": "stop",
"stopwords": "_portuguese_"
},
"portuguese_keywords": {
"type": "keyword_marker",
"keywords": ["exemplo"]
},
"portuguese_stemmer": {
"type": "stemmer",
"language": "light_portuguese"
}
},
"analyzer": {
"rebuilt_portuguese": {
"tokenizer": "standard",
"filter": [
"lowercase",
"portuguese_stop",
"portuguese_keywords",
"portuguese_stemmer"
]
}
}
}
},
"hi": {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similar blocks of code found in 2 locations. Consider refactoring.

"analysis": {
"filter": {
"hindi_stop": {
"type": "stop",
"stopwords": "_hindi_"
},
"hindi_keywords": {
"type": "keyword_marker",
"keywords": ["उदाहरण"]
},
"hindi_stemmer": {
"type": "stemmer",
"language": "hindi"
}
},
"analyzer": {
"rebuilt_hindi": {
"tokenizer": "standard",
"filter": [
"lowercase",
"decimal_digit",
"hindi_keywords",
"indic_normalization",
"hindi_normalization",
"hindi_stop",
"hindi_stemmer"
]
}
}
}
},
"bn": {
DGaffney marked this conversation as resolved.
Show resolved Hide resolved
"analysis": {
"filter": {
"bengali_stop": {
"type": "stop",
"stopwords": "_bengali_"
},
"bengali_keywords": {
"type": "keyword_marker",
"keywords": ["উদাহরণ"]
},
"bengali_stemmer": {
"type": "stemmer",
"language": "bengali"
}
},
"analyzer": {
"rebuilt_bengali": {
"tokenizer": "standard",
"filter": [
"lowercase",
"decimal_digit",
"bengali_keywords",
"indic_normalization",
"bengali_normalization",
"bengali_stop",
"bengali_stemmer"
]
}
}
}
}
}

def init_indices():
es = Elasticsearch(app.config['ELASTICSEARCH_URL'])
indices = es.cat.indices(h='index', s='index').split()
for lang in SUPPORTED_LANGUAGES:
index_name = app.config['ELASTICSEARCH_SIMILARITY'] "_" lang
if index_name not in indices:
es.indices.create(index=index_name)
es.indices.close(index=index_name)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bad indentation. Found 4 spaces, expected 8

es.indices.put_mapping(
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bad indentation. Found 4 spaces, expected 8

body=json.load(open('./elasticsearch/alegre_similarity_base.json')),
# include_type_name=True,
index=index_name
)
es.indices.put_settings(
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bad indentation. Found 4 spaces, expected 8

body=SETTINGS_BY_LANGUAGE['pt'],
# include_type_name=True,
index=index_name
)
es.indices.open(index=index_name)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bad indentation. Found 4 spaces, expected 8

5 changes: 3 additions & 2 deletions app/main/lib/similarity.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 14,7 @@ def get_body_for_text_document(params):
models = models|set(params['models'])
if not models:
models = ['elasticsearch']
body = {'content': params.get('text'), 'created_at': params.get("created_at", datetime.now()), 'limit': params.get("limit", DEFAULT_SEARCH_LIMIT), 'models': list(models)}
body = {'language': params.get('language'), 'content': params.get('text'), 'created_at': params.get("created_at", datetime.now()), 'limit': params.get("limit", DEFAULT_SEARCH_LIMIT), 'models': list(models)}
for key in ['context', 'threshold', 'fuzzy']:
if key in params:
body[key] = params[key]
Expand Down Expand Up @@ -51,7 51,8 @@ def add_item(item, similarity_type):
response = add_image(item)
elif similarity_type == "text":
doc_id = item.pop("doc_id", None)
response = add_text(item, doc_id)
language = item.pop("language", None)
response = add_text(item, doc_id, language)
app.logger.info(f"[Alegre Similarity] [Item {item}, Similarity type: {similarity_type}] response for delete was {response}")
return response

Expand Down
Loading