Feat/milvus support (#671)

Co-authored-by: StyleZhang <jasonapring2015@outlook.com>
Co-authored-by: JzoNg <jzongcode@gmail.com>

api/events/event_handlers/__init__.py

@@ -7,3 +7,4 @@ from .clean_when_dataset_deleted import handle
 from .update_app_dataset_join_when_app_model_config_updated import handle
 from .generate_conversation_name_when_first_message_created import handle
 from .generate_conversation_summary_when_few_message_created import handle
+from .create_document_index import handle

api/events/event_handlers/create_document_index.py

@@ -0,0 +1,48 @@
+from events.dataset_event import dataset_was_deleted
+from events.event_handlers.document_index_event import document_index_created
+from tasks.clean_dataset_task import clean_dataset_task
+import datetime
+import logging
+import time
+
+import click
+from celery import shared_task
+from werkzeug.exceptions import NotFound
+
+from core.indexing_runner import IndexingRunner, DocumentIsPausedException
+from extensions.ext_database import db
+from models.dataset import Document
+
+
+@document_index_created.connect
+def handle(sender, **kwargs):
+    dataset_id = sender
+    document_ids = kwargs.get('document_ids', None)
+    documents = []
+    start_at = time.perf_counter()
+    for document_id in document_ids:
+        logging.info(click.style('Start process document: {}'.format(document_id), fg='green'))
+
+        document = db.session.query(Document).filter(
+            Document.id == document_id,
+            Document.dataset_id == dataset_id
+        ).first()
+
+        if not document:
+            raise NotFound('Document not found')
+
+        document.indexing_status = 'parsing'
+        document.processing_started_at = datetime.datetime.utcnow()
+        documents.append(document)
+        db.session.add(document)
+    db.session.commit()
+
+    try:
+        indexing_runner = IndexingRunner()
+        indexing_runner.run(documents)
+        end_at = time.perf_counter()
+        logging.info(click.style('Processed dataset: {} latency: {}'.format(dataset_id, end_at - start_at), fg='green'))
+    except DocumentIsPausedException as ex:
+        logging.info(click.style(str(ex), fg='yellow'))
+    except Exception:
+        pass

api/events/event_handlers/document_index_event.py

@@ -0,0 +1,4 @@
+from blinker import signal
+
+# sender: document
+document_index_created = signal('document-index-created')