fix: fix: update plugin debugging host and port environment variables

Co-authored-by: liusurong.lsr <liusurong.lsr@alibaba-inc.com>

.gitattributes

@@ -1,5 +1,5 @@
 # Ensure that .sh scripts use LF as line separator, even if they are checked out
-# to Windows(NTFS) file-system, by a user of Docker for Window. 
+# to Windows(NTFS) file-system, by a user of Docker for Windows. 
 # These .sh scripts will be run from the Container after `docker compose up -d`.
 # If they appear to be CRLF style, Dash from the Container will fail to execute
 # them.

.github/workflows/build-push.yml

@@ -5,7 +5,6 @@ on:
     branches:
       - "main"
       - "deploy/dev"
-      - "dev/plugin-deploy"
   release:
     types: [published]
 

CONTRIBUTING_DE.md

@@ -0,0 +1,155 @@
+# MITWIRKEN
+
+So, du möchtest zu Dify beitragen – das ist großartig, wir können es kaum erwarten, zu sehen, was du beisteuern wirst. Als ein Startup mit begrenzter Mitarbeiterzahl und Finanzierung haben wir große Ambitionen, den intuitivsten Workflow zum Aufbau und zur Verwaltung von LLM-Anwendungen zu entwickeln. Jede Unterstützung aus der Community zählt wirklich.
+
+Dieser Leitfaden, ebenso wie Dify selbst, ist ein ständig in Entwicklung befindliches Projekt. Wir schätzen Ihr Verständnis, falls er zeitweise hinter dem tatsächlichen Projekt zurückbleibt, und freuen uns über jegliches Feedback, das uns hilft, ihn zu verbessern.
+
+Bezüglich der Lizenzierung nehmen Sie sich bitte einen Moment Zeit, um unser kurzes [License and Contributor Agreement](./LICENSE) zu lesen. Die Community hält sich außerdem an den [Code of Conduct](https://github.com/langgenius/.github/blob/main/CODE_OF_CONDUCT.md).
+
+## Bevor Sie loslegen
+
+[Finde](https://github.com/langgenius/dify/issues?q=is:issue+is:open) ein bestehendes Issue, oder [öffne](https://github.com/langgenius/dify/issues/new/choose) ein neues. Wir kategorisieren Issues in zwei Typen:
+
+### Feature-Anfragen
+
+* Wenn Sie eine neue Feature-Anfrage stellen, bitten wir Sie zu erklären, was das vorgeschlagene Feature bewirken soll und so viel Kontext wie möglich bereitzustellen. [@perzeusss](https://github.com/perzeuss) hat einen soliden [Feature Request Copilot](https://udify.app/chat/MK2kVSnw1gakVwMX) entwickelt, der Ihnen dabei hilft, Ihre Anforderungen zu formulieren. Probieren Sie ihn gerne aus.
+
+* Wenn Sie eines der bestehenden Issues übernehmen möchten, hinterlassen Sie einfach einen Kommentar darunter, in dem Sie uns dies mitteilen.
+
+  Ein Teammitglied, das in der entsprechenden Richtung arbeitet, wird hinzugezogen. Wenn alles in Ordnung ist, gibt es das Okay, mit der Codierung zu beginnen. Wir bitten Sie, mit der Umsetzung des Features zu warten, damit keine Ihrer Arbeiten verloren gehen sollte unsererseits Änderungen vorgeschlagen werden.
+
+  Je nachdem, in welchen Bereich das vorgeschlagene Feature fällt, können Sie mit verschiedenen Teammitgliedern sprechen. Hier ist eine Übersicht der Bereiche, an denen unsere Teammitglieder derzeit arbeiten:
+
+  | Member                                                       | Scope                                                |
+  | ------------------------------------------------------------ | ---------------------------------------------------- |
+  | [@yeuoly](https://github.com/Yeuoly)                         | Architecting Agents                                  |
+  | [@jyong](https://github.com/JohnJyong)                       | RAG pipeline design                                  |
+  | [@GarfieldDai](https://github.com/GarfieldDai)               | Building workflow orchestrations                     |
+  | [@iamjoel](https://github.com/iamjoel) & [@zxhlyh](https://github.com/zxhlyh) | Making our frontend a breeze to use                  |
+  | [@guchenhe](https://github.com/guchenhe) & [@crazywoola](https://github.com/crazywoola) | Developer experience, points of contact for anything |
+  | [@takatost](https://github.com/takatost)                     | Overall product direction and architecture           |
+
+  Wie wir Prioritäten setzen:
+
+  | Feature Type                                                 | Priority        |
+  | ------------------------------------------------------------ | --------------- |
+  | Funktionen mit hoher Priorität, wie sie von einem Teammitglied gekennzeichnet wurden     | High Priority   |
+  | Beliebte Funktionsanfragen von unserem [Community-Feedback-Board](https://github.com/langgenius/dify/discussions/categories/feedbacks) | Medium Priority |
+  | Nicht-Kernfunktionen und kleinere Verbesserungen                    | Low Priority    |
+  | Wertvoll, aber nicht unmittelbar                                  | Future-Feature  |
+
+### Sonstiges (e.g. bug report, performance optimization, typo correction)
+
+* Fangen Sie sofort an zu programmieren..
+
+  Wie wir Prioritäten setzen:
+
+  | Issue Type                                                   | Priority        |
+  | ------------------------------------------------------------ | --------------- |
+  | Fehler in Kernfunktionen (Anmeldung nicht möglich, Anwendungen funktionieren nicht, Sicherheitslücken) | Critical        |
+  | Nicht-kritische Fehler, Leistungsverbesserungen                     | Medium Priority |
+  | Kleinere Fehlerkorrekturen (Schreibfehler, verwirrende, aber funktionierende Benutzeroberfläche)                | Low Priority    |
+
+## Installieren
+
+Hier sind die Schritte, um Dify für die Entwicklung einzurichten:
+
+### 1. Fork dieses Repository
+
+### 2. Clone das Repo
+
+ Klonen Sie das geforkte Repository von Ihrem Terminal aus:
+
+```shell
+git clone git@github.com:<github_username>/dify.git
+```
+
+### 3. Abhängigkeiten prüfen
+
+Dify benötigt die folgenden Abhängigkeiten zum Bauen – stellen Sie sicher, dass sie auf Ihrem System installiert sind:
+
+* [Docker](https://www.docker.com/)
+* [Docker Compose](https://docs.docker.com/compose/install/)
+* [Node.js v18.x (LTS)](http://nodejs.org)
+* [pnpm](https://pnpm.io/)
+* [Python](https://www.python.org/) version 3.11.x or 3.12.x
+
+### 4. Installationen
+
+Dify setzt sich aus einem Backend und einem Frontend zusammen. Wechseln Sie in das Backend-Verzeichnis mit `cd api/` und folgen Sie der [Backend README](api/README.md) zur Installation. Öffnen Sie in einem separaten Terminal das Frontend-Verzeichnis mit `cd web/` und folgen Sie der [Frontend README](web/README.md) zur Installation.
+
+Überprüfen Sie die [Installation FAQ](https://docs.dify.ai/learn-more/faq/install-faq) für eine Liste bekannter Probleme und Schritte zur Fehlerbehebung.
+
+### 5. Besuchen Sie dify in Ihrem Browser
+
+Um Ihre Einrichtung zu validieren, öffnen Sie Ihren Browser und navigieren Sie zu [http://localhost:3000](http://localhost:3000) (Standardwert oder Ihre selbst konfigurierte URL und Port). Sie sollten nun Dify im laufenden Betrieb sehen.
+
+## Entwickeln
+
+Wenn Sie einen Modellanbieter hinzufügen, ist [dieser Leitfaden](https://github.com/langgenius/dify/blob/main/api/core/model_runtime/README.md) für Sie.
+
+Wenn Sie einen Tool-Anbieter für Agent oder Workflow hinzufügen möchten, ist [dieser Leitfaden](./api/core/tools/README.md) für Sie.
+
+Um Ihnen eine schnelle Orientierung zu bieten, wo Ihr Beitrag passt, folgt eine kurze, kommentierte Übersicht des Backends und Frontends von Dify:
+
+### Backend
+
+Dify’s Backend ist in Python geschrieben und nutzt [Flask](https://flask.palletsprojects.com/en/3.0.x/) als Web-Framework. Es verwendet [SQLAlchemy](https://www.sqlalchemy.org/) für ORM und [Celery](https://docs.celeryq.dev/en/stable/getting-started/introduction.html) für Task-Queueing. Die Autorisierungslogik erfolgt über Flask-login.
+
+```text
+[api/]
+├── constants             // Konstante Einstellungen, die in der gesamten Codebasis verwendet werden.
+├── controllers           // API-Routendefinitionen und Logik zur Bearbeitung von Anfragen.
+├── core                  // Orchestrierung von Kernanwendungen, Modellintegrationen und Tools.
+├── docker                // Konfigurationen im Zusammenhang mit Docker und Containerisierung.
+├── events                // Ereignisbehandlung und -verarbeitung
+├── extensions            // Erweiterungen mit Frameworks/Plattformen von Drittanbietern.
+├── fields                // Felddefinitionen für die Serialisierung/Marshalling.
+├── libs                  // Wiederverwendbare Bibliotheken und Hilfsprogramme
+├── migrations            // Skripte für die Datenbankmigration.
+├── models                // Datenbankmodelle und Schemadefinitionen.
+├── services              // Gibt die Geschäftslogik an.
+├── storage               // Speicherung privater Schlüssel.   
+├── tasks                 // Handhabung von asynchronen Aufgaben und Hintergrundaufträgen.
+└── tests
+```
+
+### Frontend
+
+Die Website basiert auf einem [Next.js](https://nextjs.org/)-Boilerplate in TypeScript und verwendet [Tailwind CSS](https://tailwindcss.com/) für das Styling. [React-i18next](https://react.i18next.com/) wird für die Internationalisierung genutzt.
+
+```text
+[web/]
+├── app                   // Layouts, Seiten und Komponenten
+│   ├── (commonLayout)    // gemeinsames Layout für die gesamte Anwendung
+│   ├── (shareLayout)     // Layouts, die speziell für tokenspezifische Sitzungen gemeinsam genutzt werden 
+│   ├── activate          // Seite aufrufen
+│   ├── components        // gemeinsam genutzt von Seiten und Layouts
+│   ├── install           // Seite installieren
+│   ├── signin            // Anmeldeseite
+│   └── styles            // global geteilte Stile
+├── assets                // Statische Vermögenswerte
+├── bin                   // Skripte, die beim Build-Schritt ausgeführt werden
+├── config                // einstellbare Einstellungen und Optionen 
+├── context               // gemeinsame Kontexte, die von verschiedenen Teilen der Anwendung verwendet werden
+├── dictionaries          // Sprachspezifische Übersetzungsdateien 
+├── docker                // Container-Konfigurationen
+├── hooks                 // Wiederverwendbare Haken
+├── i18n                  // Konfiguration der Internationalisierung
+├── models                // beschreibt Datenmodelle und Formen von API-Antworten
+├── public                // Meta-Assets wie Favicon
+├── service               // legt Formen von API-Aktionen fest
+├── test                  
+├── types                 // Beschreibungen von Funktionsparametern und Rückgabewerten
+└── utils                 // Gemeinsame Nutzenfunktionen
+```
+
+## Einreichung Ihrer PR
+
+Am Ende ist es Zeit, einen Pull Request (PR) in unserem Repository zu eröffnen. Für wesentliche Features mergen wir diese zunächst in den `deploy/dev`-Branch zum Testen, bevor sie in den `main`-Branch übernommen werden. Falls Sie auf Probleme wie Merge-Konflikte stoßen oder nicht wissen, wie man einen Pull Request erstellt, schauen Sie sich [GitHub's Pull Request Tutorial](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests) an.
+
+Und das war's! Sobald Ihr PR gemerged wurde, werden Sie als Mitwirkender in unserem [README](https://github.com/langgenius/dify/blob/main/README.md) aufgeführt.
+
+## Hilfe bekommen
+
+Wenn Sie beim Beitragen jemals nicht weiter wissen oder eine brennende Frage haben, richten Sie Ihre Anfrage einfach über das entsprechende GitHub-Issue an uns oder besuchen Sie unseren [Discord](https://discord.gg/8Tpq4AcN9c) für ein kurzes Gespräch.

README.md

@@ -49,6 +49,8 @@
   <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
   <a href="./README_TR.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
   <a href="./README_VI.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
+  <a href="./README_DE.md"><img alt="README in Deutsch" src="https://img.shields.io/badge/German-d9d9d9"></a>
+  <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </p>
 
 

README_AR.md

@@ -45,6 +45,7 @@
   <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
   <a href="./README_TR.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
   <a href="./README_VI.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
+  <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </p>
 
 <div style="text-align: right;">
@@ -53,8 +54,7 @@
 
 **1. سير العمل**:  قم ببناء واختبار سير عمل الذكاء الاصطناعي القوي على قماش بصري، مستفيدًا من جميع الميزات التالية وأكثر.
 
-  https://github.com/langgenius/dify/assets/13230914/356df23e-1604-483d-80a6-9517ece318aa
-
+  <https://github.com/langgenius/dify/assets/13230914/356df23e-1604-483d-80a6-9517ece318aa>
 
 **2. الدعم الشامل للنماذج**: تكامل سلس مع مئات من LLMs الخاصة / مفتوحة المصدر من عشرات من موفري التحليل والحلول المستضافة ذاتيًا، مما يغطي GPT و Mistral و Llama3 وأي نماذج متوافقة مع واجهة OpenAI API. يمكن العثور على قائمة كاملة بمزودي النموذج المدعومين [هنا](https://docs.dify.ai/getting-started/readme/model-providers).
 
@@ -69,7 +69,9 @@
 **6. الـ LLMOps**: راقب وتحلل سجلات التطبيق والأداء على مر الزمن. يمكنك تحسين الأوامر والبيانات والنماذج باستمرار استنادًا إلى البيانات الإنتاجية والتعليقات.
 
 **7.الواجهة الخلفية (Backend) كخدمة**: تأتي جميع عروض Dify مع APIs مطابقة، حتى يمكنك دمج Dify بسهولة في منطق أعمالك الخاص.
+
 ## مقارنة الميزات
+
 <table style="width: 100%;">
   <tr>
     <th align="center">الميزة</th>
@@ -136,8 +138,8 @@
   </tr>
 </table>
 
-
 ## استخدام Dify
+
 - **سحابة </br>**
 نحن نستضيف [خدمة Dify Cloud](https://dify.ai) لأي شخص لتجربتها بدون أي إعدادات. توفر كل قدرات النسخة التي تمت استضافتها ذاتيًا، وتتضمن 200 أمر GPT-4 مجانًا في خطة الصندوق الرملي.
 
@@ -147,15 +149,19 @@
 
 - **مشروع Dify للشركات / المؤسسات</br>**
 نحن نوفر ميزات إضافية مركزة على الشركات. [جدول اجتماع معنا](https://cal.com/guchenhe/30min) أو [أرسل لنا بريدًا إلكترونيًا](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry) لمناقشة احتياجات الشركات. </br>
+
 > بالنسبة للشركات الناشئة والشركات الصغيرة التي تستخدم خدمات AWS، تحقق من [Dify Premium على AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6) ونشرها في شبكتك الخاصة على AWS VPC بنقرة واحدة. إنها عرض AMI بأسعار معقولة مع خيار إنشاء تطبيقات بشعار وعلامة تجارية مخصصة.
+>
 ## البقاء قدمًا
 
 قم بإضافة نجمة إلى Dify على GitHub وتلق تنبيهًا فوريًا بالإصدارات الجديدة.
 
 ![نجمنا](https://github.com/langgenius/dify/assets/13230914/b823edc1-6388-4e25-ad45-2f6b187adbb4)
+
 ## البداية السريعة
+>
 > قبل تثبيت Dify، تأكد من أن جهازك يلبي الحد الأدنى من متطلبات النظام التالية:
-> 
+>
 >- معالج >= 2 نواة
 >- ذاكرة وصول عشوائي (RAM) >= 4 جيجابايت
 
@@ -188,24 +194,26 @@ docker compose up -d
 انشر Dify إلى منصة السحابة بنقرة واحدة باستخدام [terraform](https://www.terraform.io/)
 
 ##### Azure Global
+
 - [Azure Terraform بواسطة @nikawang](https://github.com/nikawang/dify-azure-terraform)
 
 ##### Google Cloud
+
 - [Google Cloud Terraform بواسطة @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
 #### استخدام AWS CDK للنشر
 
 انشر Dify على AWS باستخدام [CDK](https://aws.amazon.com/cdk/)
 
-##### AWS 
+##### AWS
+
 - [AWS CDK بواسطة @KevinZhao](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
 
 ## المساهمة
 
-لأولئك الذين يرغبون في المساهمة، انظر إلى [دليل المساهمة](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md) لدينا. 
+لأولئك الذين يرغبون في المساهمة، انظر إلى [دليل المساهمة](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md) لدينا.
 في الوقت نفسه، يرجى النظر في دعم Dify عن طريق مشاركته على وسائل التواصل الاجتماعي وفي الفعاليات والمؤتمرات.
 
-
 > نحن نبحث عن مساهمين لمساعدة في ترجمة Dify إلى لغات أخرى غير اللغة الصينية المندرين أو الإنجليزية. إذا كنت مهتمًا بالمساعدة، يرجى الاطلاع على [README للترجمة](https://github.com/langgenius/dify/blob/main/web/i18n/README.md) لمزيد من المعلومات، واترك لنا تعليقًا في قناة `global-users` على [خادم المجتمع على Discord](https://discord.gg/8Tpq4AcN9c).
 
 **المساهمون**
@@ -215,26 +223,26 @@ docker compose up -d
 </a>
 
 ## المجتمع والاتصال
-* [مناقشة Github](https://github.com/langgenius/dify/discussions). الأفضل لـ: مشاركة التعليقات وطرح الأسئلة.
-* [المشكلات على GitHub](https://github.com/langgenius/dify/issues). الأفضل لـ: الأخطاء التي تواجهها في استخدام Dify.AI، واقتراحات الميزات. انظر [دليل المساهمة](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
-* [Discord](https://discord.gg/FngNHpbcY7). الأفضل لـ: مشاركة تطبيقاتك والترفيه مع المجتمع.
-* [تويتر](https://twitter.com/dify_ai). الأفضل لـ: مشاركة تطبيقاتك والترفيه مع المجتمع.
+- [مناقشة Github](https://github.com/langgenius/dify/discussions). الأفضل لـ: مشاركة التعليقات وطرح الأسئلة.
+- [المشكلات على GitHub](https://github.com/langgenius/dify/issues). الأفضل لـ: الأخطاء التي تواجهها في استخدام Dify.AI، واقتراحات الميزات. انظر [دليل المساهمة](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
+- [Discord](https://discord.gg/FngNHpbcY7). الأفضل لـ: مشاركة تطبيقاتك والترفيه مع المجتمع.
+- [تويتر](https://twitter.com/dify_ai). الأفضل لـ: مشاركة تطبيقاتك والترفيه مع المجتمع.
 
 ## تاريخ النجمة
 
 [![Star History Chart](https://api.star-history.com/svg?repos=langgenius/dify&type=Date)](https://star-history.com/#langgenius/dify&Date)
 
-
 ## الكشف عن الأمان
 
-لحماية خصوصيتك، يرجى تجنب نشر مشكلات الأمان على GitHub. بدلاً من ذلك، أرسل أسئلتك إلى security@dify.ai وسنقدم لك إجابة أكثر تفصيلاً.
+لحماية خصوصيتك، يرجى تجنب نشر مشكلات الأمان على GitHub. بدلاً من ذلك، أرسل أسئلتك إلى <security@dify.ai> وسنقدم لك إجابة أكثر تفصيلاً.
 
 ## الرخصة
 
 هذا المستودع متاح تحت [رخصة البرنامج الحر Dify](LICENSE)، والتي تعتبر بشكل أساسي Apache 2.0 مع بعض القيود الإضافية.
+
 ## الكشف عن الأمان
 
-لحماية خصوصيتك، يرجى تجنب نشر مشكلات الأمان على GitHub. بدلاً من ذلك، أرسل أسئلتك إلى security@dify.ai وسنقدم لك إجابة أكثر تفصيلاً.
+لحماية خصوصيتك، يرجى تجنب نشر مشكلات الأمان على GitHub. بدلاً من ذلك، أرسل أسئلتك إلى <security@dify.ai> وسنقدم لك إجابة أكثر تفصيلاً.
 
 ## الرخصة
 

README_BN.md

@@ -0,0 +1,258 @@
+![cover-v5-optimized](https://github.com/langgenius/dify/assets/13230914/f9e19af5-61ba-4119-b926-d10c4c06ebab)
+
+<p align="center">
+  📌 <a href="https://dify.ai/blog/introducing-dify-workflow-file-upload-a-demo-on-ai-podcast">ডিফাই ওয়ার্কফ্লো ফাইল আপলোড পরিচিতি: গুগল নোটবুক-এলএম পডকাস্ট পুনর্নির্মাণ</a>
+</p>
+
+<p align="center">
+  <a href="https://cloud.dify.ai">ডিফাই ক্লাউড</a> ·
+  <a href="https://docs.dify.ai/getting-started/install-self-hosted">সেল্ফ-হোস্টিং</a> ·
+  <a href="https://docs.dify.ai">ডকুমেন্টেশন</a> ·
+  <a href="https://udify.app/chat/22L1zSxg6yW1cWQg">ব্যাবসায়িক অনুসন্ধান</a>
+</p>
+
+<p align="center">
+    <a href="https://dify.ai" target="_blank">
+        <img alt="Static Badge" src="https://img.shields.io/badge/Product-F04438"></a>
+    <a href="https://dify.ai/pricing" target="_blank">
+        <img alt="Static Badge" src="https://img.shields.io/badge/free-pricing?logo=free&color=%20%23155EEF&label=pricing&labelColor=%20%23528bff"></a>
+    <a href="https://discord.gg/FngNHpbcY7" target="_blank">
+        <img src="https://img.shields.io/discord/1082486657678311454?logo=discord&labelColor=%20%235462eb&logoColor=%20%23f5f5f5&color=%20%235462eb"
+            alt="chat on Discord"></a>
+    <a href="https://reddit.com/r/difyai" target="_blank">  
+        <img src="https://img.shields.io/reddit/subreddit-subscribers/difyai?style=plastic&logo=reddit&label=r%2Fdifyai&labelColor=white" 
+            alt="join Reddit"></a>
+    <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
+        <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
+            alt="follow on X(Twitter)"></a>
+    <a href="https://www.linkedin.com/company/langgenius/" target="_blank">
+        <img src="https://custom-icon-badges.demolab.com/badge/LinkedIn-0A66C2?logo=linkedin-white&logoColor=fff"
+            alt="follow on LinkedIn"></a>
+    <a href="https://hub.docker.com/u/langgenius" target="_blank">
+        <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
+    <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
+        <img alt="Commits last month" src="https://img.shields.io/github/commit-activity/m/langgenius/dify?labelColor=%20%2332b583&color=%20%2312b76a"></a>
+    <a href="https://github.com/langgenius/dify/" target="_blank">
+        <img alt="Issues closed" src="https://img.shields.io/github/issues-search?query=repo%3Alanggenius%2Fdify%20is%3Aclosed&label=issues%20closed&labelColor=%20%237d89b0&color=%20%235d6b98"></a>
+    <a href="https://github.com/langgenius/dify/discussions/" target="_blank">
+        <img alt="Discussion posts" src="https://img.shields.io/github/discussions/langgenius/dify?labelColor=%20%239b8afb&color=%20%237a5af8"></a>
+</p>
+
+<p align="center">
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-d9d9d9"></a>
+  <a href="./README_CN.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-d9d9d9"></a>
+  <a href="./README_JA.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-d9d9d9"></a>
+  <a href="./README_ES.md"><img alt="README en Español" src="https://img.shields.io/badge/Español-d9d9d9"></a>
+  <a href="./README_FR.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-d9d9d9"></a>
+  <a href="./README_KL.md"><img alt="README tlhIngan Hol" src="https://img.shields.io/badge/Klingon-d9d9d9"></a>
+  <a href="./README_KR.md"><img alt="README in Korean" src="https://img.shields.io/badge/한국어-d9d9d9"></a>
+  <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
+  <a href="./README_TR.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
+  <a href="./README_VI.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
+  <a href="./README_DE.md"><img alt="README in Deutsch" src="https://img.shields.io/badge/German-d9d9d9"></a>
+  <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
+</p>
+
+ডিফাই একটি ওপেন-সোর্স LLM অ্যাপ ডেভেলপমেন্ট প্ল্যাটফর্ম। এটি ইন্টুইটিভ ইন্টারফেস, এজেন্টিক AI ওয়ার্কফ্লো, RAG পাইপলাইন, এজেন্ট ক্যাপাবিলিটি, মডেল ম্যানেজমেন্ট, মনিটরিং সুবিধা এবং আরও অনেক কিছু একত্রিত করে, যা দ্রুত প্রোটোটাইপ থেকে প্রোডাকশন পর্যন্ত নিয়ে যেতে সহায়তা করে।
+
+## কুইক স্টার্ট
+>
+>  ডিফাই ইনস্টল করার আগে, নিশ্চিত করুন যে আপনার মেশিন নিম্নলিখিত ন্যূনতম কনফিগারেশনের প্রয়োজনীয়তা পূরন করে :
+>
+>- সিপিউ >= 2 কোর
+>- র‍্যাম >= 4 জিবি
+
+</br>
+
+ডিফাই সার্ভার চালু করার সবচেয়ে সহজ উপায় [docker compose](docker/docker-compose.yaml) মাধ্যমে। নিম্নলিখিত কমান্ডগুলো ব্যবহার করে ডিফাই চালানোর আগে, নিশ্চিত করুন যে আপনার মেশিনে [Docker](https://docs.docker.com/get-docker/) এবং [Docker Compose](https://docs.docker.com/compose/install/) ইনস্টল করা আছে :
+```bash
+cd dify
+cd docker
+cp .env.example .env
+docker compose up -d
+```
+চালানোর পর, আপনি আপনার ব্রাউজারে [http://localhost/install](http://localhost/install)-এ ডিফাই ড্যাশবোর্ডে অ্যাক্সেস করতে পারেন এবং ইনিশিয়ালাইজেশন প্রক্রিয়া শুরু করতে পারেন।
+
+#### সাহায্যের খোঁজে
+
+ডিফাই সেট আপ করতে সমস্যা হলে দয়া করে আমাদের [FAQ](https://docs.dify.ai/getting-started/install-self-hosted/faqs) দেখুন। যদি তবুও সমস্যা থেকে থাকে, তাহলে [কমিউনিটি এবং আমাদের](#community--contact)  সাথে যোগাযোগ করুন।
+
+> যদি আপনি ডিফাইতে অবদান রাখতে বা অতিরিক্ত উন্নয়ন করতে চান, আমাদের [সোর্স কোড থেকে ডিপ্লয়মেন্টের গাইড](https://docs.dify.ai/getting-started/install-self-hosted/local-source-code) দেখুন।
+
+## প্রধান ফিচারসমূহ
+
+**১. ওয়ার্কফ্লো**:
+    ভিজ্যুয়াল ক্যানভাসে  AI ওয়ার্কফ্লো তৈরি এবং পরীক্ষা করুন, নিম্নলিখিত সব ফিচার এবং  তার বাইরেও আরও অনেক কিছু ব্যবহার করে।
+
+  <https://github.com/langgenius/dify/assets/13230914/356df23e-1604-483d-80a6-9517ece318aa>
+
+**২.  মডেল সাপোর্ট**:
+  GPT, Mistral, Llama3, এবং যেকোনো OpenAI API-সামঞ্জস্যপূর্ণ মডেলসহ, কয়েক ডজন ইনফারেন্স প্রদানকারী এবং সেল্ফ-হোস্টেড সমাধান থেকে শুরু করে প্রোপ্রাইটরি/ওপেন-সোর্স LLM-এর সাথে সহজে ইন্টিগ্রেশন। সমর্থিত মডেল প্রদানকারীদের একটি সম্পূর্ণ তালিকা পাওয়া যাবে [এখানে](https://docs.dify.ai/getting-started/readme/model-providers)।
+
+![providers-v5](https://github.com/langgenius/dify/assets/13230914/5a17bdbe-097a-4100-8363-40255b70f6e3)
+
+**3.  প্রম্পট IDE**:
+      প্রম্পট তৈরি, মডেলের পারফরম্যান্স তুলনা এবং চ্যাট-বেজড অ্যাপে টেক্সট-টু-স্পিচের মতো  বৈশিষ্ট্য যুক্ত করার জন্য ইন্টুইটিভ ইন্টারফেস। 
+
+**4. RAG পাইপলাইন**:
+      ডকুমেন্ট ইনজেশন থেকে শুরু করে রিট্রিভ পর্যন্ত সবকিছুই বিস্তৃত RAG ক্যাপাবিলিটির আওতাভুক্ত।  PDF, PPT এবং অন্যান্য সাধারণ ডকুমেন্ট ফর্ম্যাট থেকে টেক্সট এক্সট্রাকশনের জন্য আউট-অফ-বক্স সাপোর্ট। 
+
+**5.  এজেন্ট ক্যাপাবিলিটি**:
+  LLM ফাংশন কলিং বা ReAct উপর ভিত্তি করে এজেন্ট ডিফাইন করতে পারেন এবং এজেন্টের জন্য পূর্ব-নির্মিত বা কাস্টম টুলস যুক্ত করতে পারেন। Dify AI এজেন্টদের জন্য 50+ বিল্ট-ইন টুলস সরবরাহ করে, যেমন Google Search, DALL·E, Stable Diffusion এবং WolframAlpha।
+
+**6.  এলএলএম-অপ্স**:
+      সময়ের সাথে সাথে অ্যাপ্লিকেশন লগ এবং পারফরম্যান্স মনিটর এবং বিশ্লেষণ করুন।  প্রডাকশন ডেটা এবং annotation এর উপর ভিত্তি করে প্রম্পট, ডেটাসেট এবং মডেলগুলিকে ক্রমাগত উন্নত করতে পারেন।
+
+**7. ব্যাকএন্ড-অ্যাজ-এ-সার্ভিস**:
+    ডিফাই-এর সমস্ত অফার সংশ্লিষ্ট API-সহ আছে, যাতে আপনি অনায়াসে ডিফাইকে আপনার নিজস্ব বিজনেস লজিকে ইন্টেগ্রেট করতে পারেন।
+
+## বৈশিষ্ট্য তুলনা
+
+<table style="width: 100%;">
+  <tr>
+    <th align="center">বৈশিষ্ট্য</th>
+    <th align="center">Dify.AI</th>
+    <th align="center">LangChain</th>
+    <th align="center">Flowise</th>
+    <th align="center">OpenAI Assistants API</th>
+  </tr>
+  <tr>
+    <td align="center">প্রোগ্রামিং পদ্ধতি</td>
+    <td align="center">API + App-oriented</td>
+    <td align="center">Python Code</td>
+    <td align="center">App-oriented</td>
+    <td align="center">API-oriented</td>
+  </tr>
+  <tr>
+    <td align="center">সাপোর্টেড LLMs</td>
+    <td align="center">Rich Variety</td>
+    <td align="center">Rich Variety</td>
+    <td align="center">Rich Variety</td>
+    <td align="center">OpenAI-only</td>
+  </tr>
+  <tr>
+    <td align="center">RAG ইঞ্জিন</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+  </tr>
+  <tr>
+    <td align="center">এজেন্ট</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">✅</td>
+  </tr>
+  <tr>
+    <td align="center">ওয়ার্কফ্লো</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+  </tr>
+  <tr>
+    <td align="center">অবজার্ভেবল</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">❌</td>
+  </tr>
+  <tr>
+    <td align="center">এন্টারপ্রাইজ ফিচার (SSO/Access control)</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">❌</td>
+    <td align="center">❌</td>
+  </tr>
+  <tr>
+    <td align="center">লোকাল ডেপ্লয়মেন্ট</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+  </tr>
+</table>
+
+## ডিফাই-এর ব্যবহার 
+
+- **ক্লাউড </br>**
+জিরো সেটাপে ব্যবহার করতে আমাদের [Dify Cloud](https://dify.ai) সার্ভিসটি ব্যবহার করতে পারেন। এখানে সেল্ফহোস্টিং-এর সকল ফিচার ও ক্যাপাবিলিটিসহ স্যান্ডবক্সে ২০০ জিপিটি-৪ কল ফ্রি পাবেন।
+
+- **সেল্ফহোস্টিং ডিফাই কমিউনিটি সংস্করণ</br>**
+সেল্ফহোস্ট করতে এই [স্টার্টার গাইড](#quick-start) ব্যবহার করে দ্রুত আপনার এনভায়রনমেন্টে ডিফাই চালান।
+আরো ইন-ডেপথ রেফারেন্সের জন্য [ডকুমেন্টেশন](https://docs.dify.ai) দেখেন।
+
+- **এন্টারপ্রাইজ / প্রতিষ্ঠানের জন্য Dify</br>**
+আমরা এন্টারপ্রাইজ/প্রতিষ্ঠান-কেন্দ্রিক সেবা প্রদান করে থাকি  । [এই চ্যাটবটের মাধ্যমে আপনার প্রশ্নগুলি আমাদের জন্য লগ করুন।](https://udify.app/chat/22L1zSxg6yW1cWQg) অথবা [আমাদের ইমেল পাঠান](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry) আপনার চাহিদা সম্পর্কে আলোচনা করার জন্য। </br>
+
+  > AWS ব্যবহারকারী স্টার্টআপ এবং ছোট ব্যবসার জন্য, [AWS মার্কেটপ্লেসে Dify Premium](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6) দেখুন এবং এক-ক্লিকের মাধ্যমে এটি আপনার নিজস্ব AWS VPC-তে ডিপ্লয় করুন। এটি একটি সাশ্রয়ী মূল্যের AMI অফার, যাতে কাস্টম লোগো এবং ব্র্যান্ডিং সহ অ্যাপ তৈরির সুবিধা আছে।
+
+## এগিয়ে থাকুন
+
+GitHub-এ ডিফাইকে স্টার দিয়ে রাখুন এবং নতুন রিলিজের খবর তাৎক্ষণিকভাবে পান।
+
+![star-us](https://github.com/langgenius/dify/assets/13230914/b823edc1-6388-4e25-ad45-2f6b187adbb4)
+
+## Advanced Setup
+
+যদি আপনার কনফিগারেশনটি কাস্টমাইজ করার প্রয়োজন হয়, তাহলে অনুগ্রহ করে আমাদের [.env.example](docker/.env.example) ফাইল দেখুন এবং আপনার `.env` ফাইলে সংশ্লিষ্ট মানগুলি আপডেট করুন। এছাড়াও, আপনার নির্দিষ্ট এনভায়রনমেন্ট এবং প্রয়োজনীয়তার উপর ভিত্তি করে আপনাকে `docker-compose.yaml` ফাইলে সমন্বয় করতে হতে পারে, যেমন ইমেজ ভার্সন পরিবর্তন করা, পোর্ট ম্যাপিং করা, অথবা ভলিউম মাউন্ট করা। 
+যেকোনো পরিবর্তন করার পর, অনুগ্রহ করে `docker-compose up -d` পুনরায় চালান। ভেরিয়েবলের সম্পূর্ণ তালিকা [এখানে] (https://docs.dify.ai/getting-started/install-self-hosted/environments) খুঁজে পেতে পারেন।
+
+যদি আপনি একটি হাইলি এভেইলেবল সেটআপ কনফিগার করতে চান, তাহলে কমিউনিটি  [Helm Charts](https://helm.sh/) এবং YAML ফাইল রয়েছে যা Dify কে Kubernetes-এ ডিপ্লয় করার প্রক্রিয়া বর্ণনা করে।
+
+- [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
+- [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
+- [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
+
+#### টেরাফর্ম ব্যবহার করে ডিপ্লয়
+
+[terraform](https://www.terraform.io/) ব্যবহার করে এক ক্লিকেই ক্লাউড প্ল্যাটফর্মে Dify ডিপ্লয় করুন।
+
+##### অ্যাজুর গ্লোবাল
+
+- [Azure Terraform by @nikawang](https://github.com/nikawang/dify-azure-terraform)
+
+##### গুগল ক্লাউড
+
+- [Google Cloud Terraform by @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
+
+#### AWS CDK ব্যবহার করে ডিপ্লয়
+
+[CDK](https://aws.amazon.com/cdk/) দিয়ে AWS-এ Dify ডিপ্লয় করুন
+
+##### AWS
+
+- [AWS CDK by @KevinZhao](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
+
+## Contributing
+
+যারা কোড অবদান রাখতে চান, তাদের জন্য আমাদের [অবদান নির্দেশিকা] দেখুন (https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md)।
+একই সাথে, সোশ্যাল মিডিয়া এবং ইভেন্ট এবং কনফারেন্সে এটি শেয়ার করে Dify কে সমর্থন করুন।
+
+> আমরা ম্যান্ডারিন বা ইংরেজি ছাড়া অন্য ভাষায় Dify অনুবাদ করতে সাহায্য করার জন্য অবদানকারীদের খুঁজছি। আপনি যদি সাহায্য করতে আগ্রহী হন, তাহলে আরও তথ্যের জন্য [i18n README](https://github.com/langgenius/dify/blob/main/web/i18n/README.md) দেখুন এবং আমাদের [ডিসকর্ড কমিউনিটি সার্ভার](https://discord.gg/8Tpq4AcN9c) এর `গ্লোবাল-ইউজারস` চ্যানেলে আমাদের একটি মন্তব্য করুন।
+
+## কমিউনিটি এবং যোগাযোগ
+
+- [Github Discussion](https://github.com/langgenius/dify/discussions) ফিডব্যাক এবং প্রতিক্রিয়া জানানোর মাধ্যম।
+- [GitHub Issues](https://github.com/langgenius/dify/issues). Dify.AI ব্যবহার করে আপনি যেসব বাগের সম্মুখীন হন এবং ফিচার প্রস্তাবনা।  আমাদের [অবদান নির্দেশিকা](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md) দেখুন।
+- [Discord](https://discord.gg/FngNHpbcY7) আপনার এপ্লিকেশন শেয়ার এবং কমিউনিটি আড্ডার মাধ্যম। 
+- [X(Twitter)](https://twitter.com/dify_ai) আপনার এপ্লিকেশন শেয়ার এবং কমিউনিটি আড্ডার মাধ্যম। 
+
+**অবদানকারীদের তালিকা**
+
+<a href="https://github.com/langgenius/dify/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=langgenius/dify" />
+</a>
+
+## স্টার হিস্ট্রি
+
+[![Star History Chart](https://api.star-history.com/svg?repos=langgenius/dify&type=Date)](https://star-history.com/#langgenius/dify&Date)
+
+## নিরাপত্তা  বিষয়ক
+
+আপনার গোপনীয়তা রক্ষা করতে, অনুগ্রহ করে GitHub-এ নিরাপত্তা সংক্রান্ত সমস্যা পোস্ট করা এড়িয়ে চলুন। পরিবর্তে, আপনার প্রশ্নগুলি <security@dify.ai> ঠিকানায় পাঠান এবং আমরা আপনাকে আরও বিস্তারিত উত্তর প্রদান করব।
+
+## লাইসেন্স
+
+এই রিপোজিটরিটি [ডিফাই ওপেন সোর্স লাইসেন্স](LICENSE) এর অধিনে , যা মূলত অ্যাপাচি ২.০, তবে কিছু অতিরিক্ত বিধিনিষেধ রয়েছে।

README_CN.md

@@ -45,6 +45,7 @@
   <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
   <a href="./README_TR.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
   <a href="./README_VI.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
+  <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </div>
 
 

README_DE.md

@@ -0,0 +1,259 @@
+![cover-v5-optimized](https://github.com/langgenius/dify/assets/13230914/f9e19af5-61ba-4119-b926-d10c4c06ebab)
+
+<p align="center">
+  📌 <a href="https://dify.ai/blog/introducing-dify-workflow-file-upload-a-demo-on-ai-podcast">Einführung in Dify Workflow File Upload: Google NotebookLM Podcast nachbilden</a>
+</p>
+
+<p align="center">
+  <a href="https://cloud.dify.ai">Dify Cloud</a> ·
+  <a href="https://docs.dify.ai/getting-started/install-self-hosted">Selbstgehostetes</a> ·
+  <a href="https://docs.dify.ai">Dokumentation</a> ·
+  <a href="https://udify.app/chat/22L1zSxg6yW1cWQg">Anfrage an Unternehmen</a>
+</p>
+
+<p align="center">
+    <a href="https://dify.ai" target="_blank">
+        <img alt="Static Badge" src="https://img.shields.io/badge/Product-F04438"></a>
+    <a href="https://dify.ai/pricing" target="_blank">
+        <img alt="Static Badge" src="https://img.shields.io/badge/free-pricing?logo=free&color=%20%23155EEF&label=pricing&labelColor=%20%23528bff"></a>
+    <a href="https://discord.gg/FngNHpbcY7" target="_blank">
+        <img src="https://img.shields.io/discord/1082486657678311454?logo=discord&labelColor=%20%235462eb&logoColor=%20%23f5f5f5&color=%20%235462eb"
+            alt="chat on Discord"></a>
+    <a href="https://reddit.com/r/difyai" target="_blank">  
+        <img src="https://img.shields.io/reddit/subreddit-subscribers/difyai?style=plastic&logo=reddit&label=r%2Fdifyai&labelColor=white"
+            alt="join Reddit"></a>
+    <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
+        <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
+            alt="follow on X(Twitter)"></a>
+    <a href="https://www.linkedin.com/company/langgenius/" target="_blank">
+        <img src="https://custom-icon-badges.demolab.com/badge/LinkedIn-0A66C2?logo=linkedin-white&logoColor=fff"
+            alt="follow on LinkedIn"></a>
+    <a href="https://hub.docker.com/u/langgenius" target="_blank">
+        <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
+    <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
+        <img alt="Commits last month" src="https://img.shields.io/github/commit-activity/m/langgenius/dify?labelColor=%20%2332b583&color=%20%2312b76a"></a>
+    <a href="https://github.com/langgenius/dify/" target="_blank">
+        <img alt="Issues closed" src="https://img.shields.io/github/issues-search?query=repo%3Alanggenius%2Fdify%20is%3Aclosed&label=issues%20closed&labelColor=%20%237d89b0&color=%20%235d6b98"></a>
+    <a href="https://github.com/langgenius/dify/discussions/" target="_blank">
+        <img alt="Discussion posts" src="https://img.shields.io/github/discussions/langgenius/dify?labelColor=%20%239b8afb&color=%20%237a5af8"></a>
+</p>
+
+<p align="center">
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-d9d9d9"></a>
+  <a href="./README_CN.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-d9d9d9"></a>
+  <a href="./README_JA.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-d9d9d9"></a>
+  <a href="./README_ES.md"><img alt="README en Español" src="https://img.shields.io/badge/Español-d9d9d9"></a>
+  <a href="./README_FR.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-d9d9d9"></a>
+  <a href="./README_KL.md"><img alt="README tlhIngan Hol" src="https://img.shields.io/badge/Klingon-d9d9d9"></a>
+  <a href="./README_KR.md"><img alt="README in Korean" src="https://img.shields.io/badge/한국어-d9d9d9"></a>
+  <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
+  <a href="./README_TR.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
+  <a href="./README_VI.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
+  <a href="./README_DE.md"><img alt="README in Deutsch" src="https://img.shields.io/badge/German-d9d9d9"></a>
+  <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
+</p>
+
+Dify ist eine Open-Source-Plattform zur Entwicklung von LLM-Anwendungen. Ihre intuitive Benutzeroberfläche vereint agentenbasierte KI-Workflows, RAG-Pipelines, Agentenfunktionen, Modellverwaltung, Überwachungsfunktionen und mehr, sodass Sie schnell von einem Prototyp in die Produktion übergehen können.
+
+## Schnellstart
+> Bevor Sie Dify installieren, stellen Sie sicher, dass Ihr System die folgenden Mindestanforderungen erfüllt:
+> 
+>- CPU >= 2 Core
+>- RAM >= 4 GiB
+
+</br>
+
+Der einfachste Weg, den Dify-Server zu starten, ist über [docker compose](docker/docker-compose.yaml). Stellen Sie vor dem Ausführen von Dify mit den folgenden Befehlen sicher, dass [Docker](https://docs.docker.com/get-docker/) und [Docker Compose](https://docs.docker.com/compose/install/) auf Ihrem System installiert sind:
+
+```bash
+cd dify
+cd docker
+cp .env.example .env
+docker compose up -d
+```
+
+Nachdem Sie den Server gestartet haben, können Sie über Ihren Browser auf das Dify Dashboard unter [http://localhost/install](http://localhost/install) zugreifen und den Initialisierungsprozess starten.
+
+#### Hilfe suchen
+Bitte beachten Sie unsere [FAQ](https://docs.dify.ai/getting-started/install-self-hosted/faqs), wenn Sie Probleme bei der Einrichtung von Dify haben. Wenden Sie sich an [die Community und uns](#community--contact), falls weiterhin Schwierigkeiten auftreten.
+
+> Wenn Sie zu Dify beitragen oder zusätzliche Entwicklungen durchführen möchten, lesen Sie bitte unseren [Leitfaden zur Bereitstellung aus dem Quellcode](https://docs.dify.ai/getting-started/install-self-hosted/local-source-code).
+
+## Wesentliche Merkmale
+**1. Workflow**: 
+  Erstellen und testen Sie leistungsstarke KI-Workflows auf einer visuellen Oberfläche, wobei Sie alle der folgenden Funktionen und darüber hinaus nutzen können.
+
+
+  https://github.com/langgenius/dify/assets/13230914/356df23e-1604-483d-80a6-9517ece318aa
+
+
+
+**2. Umfassende Modellunterstützung**: 
+  Nahtlose Integration mit Hunderten von proprietären und Open-Source-LLMs von Dutzenden Inferenzanbietern und selbstgehosteten Lösungen, die GPT, Mistral, Llama3 und alle mit der OpenAI API kompatiblen Modelle abdecken. Eine vollständige Liste der unterstützten Modellanbieter finden Sie [hier](https://docs.dify.ai/getting-started/readme/model-providers).
+
+
+![providers-v5](https://github.com/langgenius/dify/assets/13230914/5a17bdbe-097a-4100-8363-40255b70f6e3)
+
+
+**3. Prompt IDE**: 
+  Intuitive Benutzeroberfläche zum Erstellen von Prompts, zum Vergleichen der Modellleistung und zum Hinzufügen zusätzlicher Funktionen wie Text-to-Speech in einer chatbasierten Anwendung.
+
+**4. RAG Pipeline**: 
+  Umfassende RAG-Funktionalitäten, die alles von der Dokumenteneinlesung bis zur -abfrage abdecken, mit sofort einsatzbereiter Unterstützung für die Textextraktion aus PDFs, PPTs und anderen gängigen Dokumentformaten.
+
+**5. Fähigkeiten des Agenten**: 
+  Sie können Agenten basierend auf LLM Function Calling oder ReAct definieren und vorgefertigte oder benutzerdefinierte Tools für den Agenten hinzufügen. Dify stellt über 50 integrierte Tools für KI-Agenten bereit, wie zum Beispiel Google Search, DALL·E, Stable Diffusion und WolframAlpha.
+
+**6. LLMOps**: 
+  Überwachen und analysieren Sie Anwendungsprotokolle und die Leistung im Laufe der Zeit. Sie können kontinuierlich Prompts, Datensätze und Modelle basierend auf Produktionsdaten und Annotationen verbessern.
+
+**7. Backend-as-a-Service**: 
+  Alle Dify-Angebote kommen mit entsprechenden APIs, sodass Sie Dify mühelos in Ihre eigene Geschäftslogik integrieren können.
+
+## Vergleich der Merkmale
+<table style="width: 100%;">
+  <tr>
+    <th align="center">Feature</th>
+    <th align="center">Dify.AI</th>
+    <th align="center">LangChain</th>
+    <th align="center">Flowise</th>
+    <th align="center">OpenAI Assistants API</th>
+  </tr>
+  <tr>
+    <td align="center">Programming Approach</td>
+    <td align="center">API + App-oriented</td>
+    <td align="center">Python Code</td>
+    <td align="center">App-oriented</td>
+    <td align="center">API-oriented</td>
+  </tr>
+  <tr>
+    <td align="center">Supported LLMs</td>
+    <td align="center">Rich Variety</td>
+    <td align="center">Rich Variety</td>
+    <td align="center">Rich Variety</td>
+    <td align="center">OpenAI-only</td>
+  </tr>
+  <tr>
+    <td align="center">RAG Engine</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+  </tr>
+  <tr>
+    <td align="center">Agent</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">✅</td>
+  </tr>
+  <tr>
+    <td align="center">Workflow</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+  </tr>
+  <tr>
+    <td align="center">Observability</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">❌</td>
+  </tr>
+  <tr>
+    <td align="center">Enterprise Feature (SSO/Access control)</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">❌</td>
+    <td align="center">❌</td>
+  </tr>
+  <tr>
+    <td align="center">Local Deployment</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+  </tr>
+</table>
+
+## Dify verwenden
+
+- **Cloud </br>**
+Wir hosten einen [Dify Cloud](https://dify.ai)-Service, den jeder ohne Einrichtung ausprobieren kann. Er bietet alle Funktionen der selbstgehosteten Version und beinhaltet 200 kostenlose GPT-4-Aufrufe im Sandbox-Plan.
+
+- **Selbstgehostete Dify Community Edition</br>**
+Starten Sie Dify schnell in Ihrer Umgebung mit diesem [Schnellstart-Leitfaden](#quick-start). Nutzen Sie unsere [Dokumentation](https://docs.dify.ai) für weiterführende Informationen und detaillierte Anweisungen.
+
+- **Dify für Unternehmen / Organisationen</br>**
+Wir bieten zusätzliche, unternehmensspezifische Funktionen. [Über diesen Chatbot können Sie uns Ihre Fragen mitteilen](https://udify.app/chat/22L1zSxg6yW1cWQg) oder [senden Sie uns eine E-Mail](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry), um Ihre unternehmerischen Bedürfnisse zu besprechen. </br>
+  > Für Startups und kleine Unternehmen, die AWS nutzen, schauen Sie sich [Dify Premium on AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6) an und stellen Sie es mit nur einem Klick in Ihrer eigenen AWS VPC bereit. Es handelt sich um ein erschwingliches AMI-Angebot mit der Option, Apps mit individuellem Logo und Branding zu erstellen.
+
+
+## Immer einen Schritt voraus
+
+Star Dify auf GitHub und lassen Sie sich sofort über neue Releases benachrichtigen.
+
+![star-us](https://github.com/langgenius/dify/assets/13230914/b823edc1-6388-4e25-ad45-2f6b187adbb4)
+
+
+## Erweiterte Einstellungen
+
+Falls Sie die Konfiguration anpassen müssen, lesen Sie bitte die Kommentare in unserer [.env.example](docker/.env.example)-Datei und aktualisieren Sie die entsprechenden Werte in Ihrer `.env`-Datei. Zusätzlich müssen Sie eventuell Anpassungen an der `docker-compose.yaml`-Datei vornehmen, wie zum Beispiel das Ändern von Image-Versionen, Portzuordnungen oder Volumen-Mounts, je nach Ihrer spezifischen Einsatzumgebung und Ihren Anforderungen. Nachdem Sie Änderungen vorgenommen haben, starten Sie `docker-compose up -d` erneut. Eine vollständige Liste der verfügbaren Umgebungsvariablen finden Sie [hier](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+
+Falls Sie eine hochverfügbare Konfiguration einrichten möchten, gibt es von der Community bereitgestellte [Helm Charts](https://helm.sh/) und YAML-Dateien, die es ermöglichen, Dify auf Kubernetes bereitzustellen.
+
+- [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
+- [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
+- [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
+
+#### Terraform für die Bereitstellung verwenden
+
+Stellen Sie Dify mit nur einem Klick mithilfe von [terraform](https://www.terraform.io/) auf einer Cloud-Plattform bereit.
+
+##### Azure Global
+- [Azure Terraform by @nikawang](https://github.com/nikawang/dify-azure-terraform)
+
+##### Google Cloud
+- [Google Cloud Terraform by @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
+
+#### Verwendung von AWS CDK für die Bereitstellung
+
+Bereitstellung von Dify auf AWS mit [CDK](https://aws.amazon.com/cdk/)
+
+##### AWS 
+- [AWS CDK by @KevinZhao](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
+
+## Contributing
+
+Falls Sie Code beitragen möchten, lesen Sie bitte unseren [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). Gleichzeitig bitten wir Sie, Dify zu unterstützen, indem Sie es in den sozialen Medien teilen und auf Veranstaltungen und Konferenzen präsentieren.
+
+
+> Wir suchen Mitwirkende, die dabei helfen, Dify in weitere Sprachen zu übersetzen – außer Mandarin oder Englisch. Wenn Sie Interesse an einer Mitarbeit haben, lesen Sie bitte die [i18n README](https://github.com/langgenius/dify/blob/main/web/i18n/README.md) für weitere Informationen und hinterlassen Sie einen Kommentar im `global-users`-Kanal unseres [Discord Community Servers](https://discord.gg/8Tpq4AcN9c).
+
+## Gemeinschaft & Kontakt
+
+* [Github Discussion](https://github.com/langgenius/dify/discussions). Am besten geeignet für: den Austausch von Feedback und das Stellen von Fragen.
+* [GitHub Issues](https://github.com/langgenius/dify/issues). Am besten für: Fehler, auf die Sie bei der Verwendung von Dify.AI stoßen, und Funktionsvorschläge. Siehe unseren [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
+* [Discord](https://discord.gg/FngNHpbcY7).  Am besten geeignet für: den Austausch von Bewerbungen und den Austausch mit der Community.
+* [X(Twitter)](https://twitter.com/dify_ai). Am besten geeignet für: den Austausch von Bewerbungen und den Austausch mit der Community.
+
+**Mitwirkende**
+
+<a href="https://github.com/langgenius/dify/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=langgenius/dify" />
+</a>
+
+## Star-Geschichte
+
+[![Star History Chart](https://api.star-history.com/svg?repos=langgenius/dify&type=Date)](https://star-history.com/#langgenius/dify&Date)
+
+
+## Offenlegung der Sicherheit
+
+Um Ihre Privatsphäre zu schützen, vermeiden Sie es bitte, Sicherheitsprobleme auf GitHub zu posten. Schicken Sie Ihre Fragen stattdessen an security@dify.ai und wir werden Ihnen eine ausführlichere Antwort geben.
+
+## Lizenz
+
+Dieses Repository steht unter der [Dify Open Source License](LICENSE), die im Wesentlichen Apache 2.0 mit einigen zusätzlichen Einschränkungen ist.
+

README_ES.md

@@ -45,6 +45,7 @@
   <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
   <a href="./README_TR.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
   <a href="./README_VI.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
+  <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </p>
 
 #

README_FR.md

@@ -45,6 +45,7 @@
   <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
   <a href="./README_TR.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
   <a href="./README_VI.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
+  <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </p>
 
 #

README_JA.md

@@ -15,7 +15,7 @@
     <a href="https://discord.gg/FngNHpbcY7" target="_blank">
         <img src="https://img.shields.io/discord/1082486657678311454?logo=discord&labelColor=%20%235462eb&logoColor=%20%23f5f5f5&color=%20%235462eb"
             alt="Discordでチャット"></a>
-    <a href="https://reddit.com/r/difyai" target="_blank">  
+    <a href="https://reddit.com/r/difyai" target="_blank">
         <img src="https://img.shields.io/reddit/subreddit-subscribers/difyai?style=plastic&logo=reddit&label=r%2Fdifyai&labelColor=white"
             alt="Reddit"></a>
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
@@ -45,6 +45,7 @@
   <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
   <a href="./README_TR.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
   <a href="./README_VI.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
+  <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </p>
 
 #
@@ -56,7 +57,7 @@
 DifyはオープンソースのLLMアプリケーション開発プラットフォームです。直感的なインターフェイスには、AIワークフロー、RAGパイプライン、エージェント機能、モデル管理、観測機能などが組み合わさっており、プロトタイプから生産まで迅速に進めることができます。以下の機能が含まれます：
 </br> </br>
 
-**1. ワークフロー**: 
+**1. ワークフロー**:
   強力なAIワークフローをビジュアルキャンバス上で構築し、テストできます。すべての機能、および以下の機能を使用できます。
 
 
@@ -64,25 +65,25 @@ DifyはオープンソースのLLMアプリケーション開発プラットフ
 
 
 
-**2. 総合的なモデルサポート**: 
+**2. 総合的なモデルサポート**:
   数百ものプロプライエタリ/オープンソースのLLMと、数十もの推論プロバイダーおよびセルフホスティングソリューションとのシームレスな統合を提供します。GPT、Mistral、Llama3、OpenAI APIと互換性のあるすべてのモデルを統合されています。サポートされているモデルプロバイダーの完全なリストは[こちら](https://docs.dify.ai/getting-started/readme/model-providers)をご覧ください。
 
 ![providers-v5](https://github.com/langgenius/dify/assets/13230914/5a17bdbe-097a-4100-8363-40255b70f6e3)
 
 
-**3. プロンプトIDE**: 
+**3. プロンプトIDE**:
   プロンプトの作成、モデルパフォーマンスの比較が行え、チャットベースのアプリに音声合成などの機能も追加できます。
 
-**4. RAGパイプライン**: 
+**4. RAGパイプライン**:
   ドキュメントの取り込みから検索までをカバーする広範なRAG機能ができます。ほかにもPDF、PPT、その他の一般的なドキュメントフォーマットからのテキスト抽出のサポートも提供します。
 
-**5. エージェント機能**: 
+**5. エージェント機能**:
   LLM Function CallingやReActに基づくエージェントの定義が可能で、AIエージェント用のプリビルトまたはカスタムツールを追加できます。Difyには、Google検索、DALL·E、Stable Diffusion、WolframAlphaなどのAIエージェント用の50以上の組み込みツールが提供します。
 
-**6. LLMOps**: 
+**6. LLMOps**:
   アプリケーションのログやパフォーマンスを監視と分析し、生産のデータと注釈に基づいて、プロンプト、データセット、モデルを継続的に改善できます。
 
-**7. Backend-as-a-Service**: 
+**7. Backend-as-a-Service**:
   すべての機能はAPIを提供されており、Difyを自分のビジネスロジックに簡単に統合できます。
 
 
@@ -164,7 +165,7 @@ DifyはオープンソースのLLMアプリケーション開発プラットフ
 
 - **企業/組織向けのDify</br>**
 企業中心の機能を提供しています。[メールを送信](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry)して企業のニーズについて相談してください。 </br>
-  > AWSを使用しているスタートアップ企業や中小企業の場合は、[AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t23mebxzwjhu6)のDify Premiumをチェックして、ワンクリックで自分のAWS VPCにデプロイできます。さらに、手頃な価格のAMIオファリングとして、ロゴやブランディングをカスタマイズしてアプリケーションを作成するオプションがあります。
+  > AWSを使用しているスタートアップ企業や中小企業の場合は、[AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6)のDify Premiumをチェックして、ワンクリックで自分のAWS VPCにデプロイできます。さらに、手頃な価格のAMIオファリングとして、ロゴやブランディングをカスタマイズしてアプリケーションを作成するオプションがあります。
 
 
 ## 最新の情報を入手
@@ -177,7 +178,7 @@ GitHub上でDifyにスターを付けることで、Difyに関する新しいニ
 
 ## クイックスタート
 > Difyをインストールする前に、お使いのマシンが以下の最小システム要件を満たしていることを確認してください：
-> 
+>
 >- CPU >= 2コア
 >- RAM >= 4GB
 
@@ -219,7 +220,7 @@ docker compose up -d
 
 [CDK](https://aws.amazon.com/cdk/) を使用して、DifyをAWSにデプロイします
 
-##### AWS 
+##### AWS
 - [@KevinZhaoによるAWS CDK](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
 
 ## 貢献

README_KL.md

@@ -45,6 +45,7 @@
   <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
   <a href="./README_TR.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
   <a href="./README_VI.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
+  <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </p>
 
 #

README_KR.md

@@ -45,6 +45,7 @@
   <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
   <a href="./README_TR.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
   <a href="./README_VI.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
+    <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </p>
 
 

README_PT.md

@@ -50,6 +50,7 @@
   <a href="./README_TR.md"><img alt="README em Turco" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
   <a href="./README_VI.md"><img alt="README em Vietnamita" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
   <a href="./README_PT.md"><img alt="README em Português - BR" src="https://img.shields.io/badge/Portugu%C3%AAs-BR?style=flat&label=BR&color=d9d9d9"></a>
+  <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </p>
 
 Dify é uma plataforma de desenvolvimento de aplicativos LLM de código aberto. Sua interface intuitiva combina workflow de IA, pipeline RAG, capacidades de agente, gerenciamento de modelos, recursos de observabilidade e muito mais, permitindo que você vá rapidamente do protótipo à produção. Aqui está uma lista das principais funcionalidades:

README_SI.md

@@ -47,6 +47,7 @@
   <a href="./README_TR.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
   <a href="./README_VI.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
   <a href="./README_SI.md"><img alt="README Slovenščina" src="https://img.shields.io/badge/Sloven%C5%A1%C4%8Dina-d9d9d9"></a>
+  <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </p>
 
 

README_TR.md

@@ -45,6 +45,7 @@
   <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
   <a href="./README_TR.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
   <a href="./README_VI.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
+  <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </p>
 
 

README_VI.md

@@ -45,6 +45,7 @@
   <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
   <a href="./README_TR.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
   <a href="./README_VI.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
+  <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </p>
 
 

api/.env.example

@@ -427,7 +427,6 @@ PLUGIN_DAEMON_URL=http://127.0.0.1:5002
 PLUGIN_REMOTE_INSTALL_PORT=5003
 PLUGIN_REMOTE_INSTALL_HOST=localhost
 PLUGIN_MAX_PACKAGE_SIZE=15728640
-INNER_API_KEY=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1
 INNER_API_KEY_FOR_PLUGIN=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1
 
 # Marketplace configuration

api/.ruff.toml

@@ -7,7 +7,7 @@ line-length = 120
 quote-style = "double"
 
 [lint]
-preview = true
+preview = false
 select = [
     "B", # flake8-bugbear rules
     "C4", # flake8-comprehensions
@@ -18,7 +18,6 @@ select = [
     "N", # pep8-naming
     "PT", # flake8-pytest-style rules
     "PLC0208", # iteration-over-set
-    "PLC2801", # unnecessary-dunder-call
     "PLC0414", # useless-import-alias
     "PLE0604", # invalid-all-object
     "PLE0605", # invalid-all-format
@@ -46,7 +45,6 @@ ignore = [
     "E712", # true-false-comparison
     "E721", # type-comparison
     "E722", # bare-except
-    "E731", # lambda-assignment
     "F821", # undefined-name
     "F841", # unused-variable
     "FURB113", # repeated-append

api/configs/feature/__init__.py

@@ -389,11 +389,6 @@ class InnerAPIConfig(BaseSettings):
         default=False,
     )
 
-    INNER_API_KEY: Optional[str] = Field(
-        description="API key for accessing the internal API",
-        default=None,
-    )
-
 
 class LoggingConfig(BaseSettings):
     """

api/configs/middleware/vdb/oracle_config.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import Field, PositiveInt
+from pydantic import Field
 from pydantic_settings import BaseSettings
 
 
@@ -9,16 +9,6 @@ class OracleConfig(BaseSettings):
     Configuration settings for Oracle database
     """
 
-    ORACLE_HOST: Optional[str] = Field(
-        description="Hostname or IP address of the Oracle database server (e.g., 'localhost' or 'oracle.example.com')",
-        default=None,
-    )
-
-    ORACLE_PORT: PositiveInt = Field(
-        description="Port number on which the Oracle database server is listening (default is 1521)",
-        default=1521,
-    )
-
     ORACLE_USER: Optional[str] = Field(
         description="Username for authenticating with the Oracle database",
         default=None,
@@ -29,7 +19,28 @@ class OracleConfig(BaseSettings):
         default=None,
     )
 
-    ORACLE_DATABASE: Optional[str] = Field(
-        description="Name of the Oracle database or service to connect to (e.g., 'ORCL' or 'pdborcl')",
+    ORACLE_DSN: Optional[str] = Field(
+        description="Oracle database connection string. For traditional database, use format 'host:port/service_name'. "
+        "For autonomous database, use the service name from tnsnames.ora in the wallet",
         default=None,
     )
+
+    ORACLE_CONFIG_DIR: Optional[str] = Field(
+        description="Directory containing the tnsnames.ora configuration file. Only used in thin mode connection",
+        default=None,
+    )
+
+    ORACLE_WALLET_LOCATION: Optional[str] = Field(
+        description="Oracle wallet directory path containing the wallet files for secure connection",
+        default=None,
+    )
+
+    ORACLE_WALLET_PASSWORD: Optional[str] = Field(
+        description="Password to decrypt the Oracle wallet, if it is encrypted",
+        default=None,
+    )
+
+    ORACLE_IS_AUTONOMOUS: bool = Field(
+        description="Flag indicating whether connecting to Oracle Autonomous Database",
+        default=False,
+    )

api/contexts/__init__.py

@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING
 from contexts.wrapper import RecyclableContextVar
 
 if TYPE_CHECKING:
+    from core.model_runtime.entities.model_entities import AIModelEntity
     from core.plugin.entities.plugin_daemon import PluginModelProviderEntity
     from core.tools.plugin_tool.provider import PluginToolProviderController
     from core.workflow.entities.variable_pool import VariablePool
@@ -20,11 +21,19 @@ To avoid race-conditions caused by gunicorn thread recycling, using RecyclableCo
 plugin_tool_providers: RecyclableContextVar[dict[str, "PluginToolProviderController"]] = RecyclableContextVar(
     ContextVar("plugin_tool_providers")
 )
+
 plugin_tool_providers_lock: RecyclableContextVar[Lock] = RecyclableContextVar(ContextVar("plugin_tool_providers_lock"))
 
 plugin_model_providers: RecyclableContextVar[list["PluginModelProviderEntity"] | None] = RecyclableContextVar(
     ContextVar("plugin_model_providers")
 )
+
 plugin_model_providers_lock: RecyclableContextVar[Lock] = RecyclableContextVar(
     ContextVar("plugin_model_providers_lock")
 )
+
+plugin_model_schema_lock: RecyclableContextVar[Lock] = RecyclableContextVar(ContextVar("plugin_model_schema_lock"))
+
+plugin_model_schemas: RecyclableContextVar[dict[str, "AIModelEntity"]] = RecyclableContextVar(
+    ContextVar("plugin_model_schemas")
+)

api/controllers/console/app/site.py

@@ -2,7 +2,6 @@ from datetime import UTC, datetime
 
 from flask_login import current_user  # type: ignore
 from flask_restful import Resource, marshal_with, reqparse  # type: ignore
-from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden, NotFound
 
 from constants.languages import supported_language
@@ -51,37 +50,35 @@ class AppSite(Resource):
         if not current_user.is_editor:
             raise Forbidden()
 
-        with Session(db.engine) as session:
-            site = session.query(Site).filter(Site.app_id == app_model.id).first()
+        site = db.session.query(Site).filter(Site.app_id == app_model.id).first()
+        if not site:
+            raise NotFound
 
-            if not site:
-                raise NotFound
+        for attr_name in [
+            "title",
+            "icon_type",
+            "icon",
+            "icon_background",
+            "description",
+            "default_language",
+            "chat_color_theme",
+            "chat_color_theme_inverted",
+            "customize_domain",
+            "copyright",
+            "privacy_policy",
+            "custom_disclaimer",
+            "customize_token_strategy",
+            "prompt_public",
+            "show_workflow_steps",
+            "use_icon_as_answer_icon",
+        ]:
+            value = args.get(attr_name)
+            if value is not None:
+                setattr(site, attr_name, value)
 
-            for attr_name in [
-                "title",
-                "icon_type",
-                "icon",
-                "icon_background",
-                "description",
-                "default_language",
-                "chat_color_theme",
-                "chat_color_theme_inverted",
-                "customize_domain",
-                "copyright",
-                "privacy_policy",
-                "custom_disclaimer",
-                "customize_token_strategy",
-                "prompt_public",
-                "show_workflow_steps",
-                "use_icon_as_answer_icon",
-            ]:
-                value = args.get(attr_name)
-                if value is not None:
-                    setattr(site, attr_name, value)
-
-            site.updated_by = current_user.id
-            site.updated_at = datetime.now(UTC).replace(tzinfo=None)
-            session.commit()
+        site.updated_by = current_user.id
+        site.updated_at = datetime.now(UTC).replace(tzinfo=None)
+        db.session.commit()
 
         return site
 

api/controllers/console/datasets/datasets_document.py

@@ -617,7 +617,7 @@ class DocumentDetailApi(DocumentResource):
             raise InvalidMetadataError(f"Invalid metadata value: {metadata}")
 
         if metadata == "only":
-            response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details}
+            response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata}
         elif metadata == "without":
             dataset_process_rules = DatasetService.get_process_rules(dataset_id)
             document_process_rules = document.dataset_process_rule.to_dict()
@@ -678,7 +678,7 @@ class DocumentDetailApi(DocumentResource):
                 "disabled_by": document.disabled_by,
                 "archived": document.archived,
                 "doc_type": document.doc_type,
-                "doc_metadata": document.doc_metadata_details,
+                "doc_metadata": document.doc_metadata,
                 "segment_count": document.segment_count,
                 "average_segment_length": document.average_segment_length,
                 "hit_count": document.hit_count,

api/controllers/console/datasets/metadata.py

@@ -1,143 +0,0 @@
-from flask_login import current_user  # type: ignore  # type: ignore
-from flask_restful import Resource, marshal_with, reqparse  # type: ignore
-from werkzeug.exceptions import NotFound
-
-from controllers.console import api
-from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required
-from fields.dataset_fields import dataset_metadata_fields
-from libs.login import login_required
-from services.dataset_service import DatasetService
-from services.entities.knowledge_entities.knowledge_entities import (
-    MetadataArgs,
-    MetadataOperationData,
-)
-from services.metadata_service import MetadataService
-
-
-def _validate_name(name):
-    if not name or len(name) < 1 or len(name) > 40:
-        raise ValueError("Name must be between 1 to 40 characters.")
-    return name
-
-
-def _validate_description_length(description):
-    if len(description) > 400:
-        raise ValueError("Description cannot exceed 400 characters.")
-    return description
-
-
-class DatasetListApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @enterprise_license_required
-    @marshal_with(dataset_metadata_fields)
-    def post(self, dataset_id):
-        parser = reqparse.RequestParser()
-        parser.add_argument("type", type=str, required=True, nullable=True, location="json")
-        parser.add_argument("name", type=str, required=True, nullable=True, location="json")
-        args = parser.parse_args()
-        metadata_args = MetadataArgs(**args)
-
-        dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
-        if dataset is None:
-            raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
-
-        metadata = MetadataService.create_metadata(dataset_id_str, metadata_args)
-        return metadata, 201
-
-
-class DatasetMetadataApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @enterprise_license_required
-    def patch(self, dataset_id, metadata_id):
-        parser = reqparse.RequestParser()
-        parser.add_argument("name", type=str, required=True, nullable=True, location="json")
-        args = parser.parse_args()
-
-        dataset_id_str = str(dataset_id)
-        metadata_id_str = str(metadata_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
-        if dataset is None:
-            raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
-
-        metadata = MetadataService.update_metadata_name(dataset_id_str, metadata_id_str, args.get("name"))
-        return metadata, 200
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @enterprise_license_required
-    def delete(self, dataset_id, metadata_id):
-        dataset_id_str = str(dataset_id)
-        metadata_id_str = str(metadata_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
-        if dataset is None:
-            raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
-
-        MetadataService.delete_metadata(dataset_id_str, metadata_id_str)
-        return 200
-
-
-class DatasetMetadataBuiltInFieldApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @enterprise_license_required
-    def get(self):
-        built_in_fields = MetadataService.get_built_in_fields()
-        return built_in_fields, 200
-
-
-class DatasetMetadataBuiltInFieldActionApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @enterprise_license_required
-    def post(self, dataset_id, action):
-        dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
-        if dataset is None:
-            raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
-
-        if action == "enable":
-            MetadataService.enable_built_in_field(dataset)
-        elif action == "disable":
-            MetadataService.disable_built_in_field(dataset)
-        return 200
-
-
-class DocumentMetadataApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @enterprise_license_required
-    def post(self, dataset_id):
-        dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
-        if dataset is None:
-            raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
-
-        parser = reqparse.RequestParser()
-        parser.add_argument("operation_data", type=list, required=True, nullable=True, location="json")
-        args = parser.parse_args()
-        metadata_args = MetadataOperationData(**args)
-
-        MetadataService.update_documents_metadata(dataset, metadata_args)
-
-        return 200
-
-
-api.add_resource(DatasetListApi, "/datasets/<uuid:dataset_id>/metadata")
-api.add_resource(DatasetMetadataApi, "/datasets/<uuid:dataset_id>/metadata/<uuid:metadata_id>")
-api.add_resource(DatasetMetadataBuiltInFieldApi, "/datasets/metadata/built-in")
-api.add_resource(DatasetMetadataBuiltInFieldActionApi, "/datasets/metadata/built-in/<string:action>")
-api.add_resource(DocumentMetadataApi, "/datasets/<uuid:dataset_id>/documents/metadata")

api/core/app/apps/advanced_chat/generate_task_pipeline.py

@@ -582,6 +582,15 @@ class AdvancedChatAppGenerateTaskPipeline:
                         session.commit()
 
                     yield workflow_finish_resp
+                elif event.stopped_by in (
+                    QueueStopEvent.StopBy.INPUT_MODERATION,
+                    QueueStopEvent.StopBy.ANNOTATION_REPLY,
+                ):
+                    # When hitting input-moderation or annotation-reply, the workflow will not start
+                    with Session(db.engine, expire_on_commit=False) as session:
+                        # Save message
+                        self._save_message(session=session)
+                        session.commit()
 
                 yield self._message_end_to_stream_response()
                 break

api/core/model_runtime/model_providers/__base/ai_model.py

@@ -1,8 +1,11 @@
 import decimal
+import hashlib
+from threading import Lock
 from typing import Optional
 
 from pydantic import BaseModel, ConfigDict, Field
 
+import contexts
 from core.model_runtime.entities.common_entities import I18nObject
 from core.model_runtime.entities.defaults import PARAMETER_RULE_TEMPLATE
 from core.model_runtime.entities.model_entities import (
@@ -139,15 +142,35 @@ class AIModel(BaseModel):
         :return: model schema
         """
         plugin_model_manager = PluginModelManager()
-        return plugin_model_manager.get_model_schema(
-            tenant_id=self.tenant_id,
-            user_id="unknown",
-            plugin_id=self.plugin_id,
-            provider=self.provider_name,
-            model_type=self.model_type.value,
-            model=model,
-            credentials=credentials or {},
-        )
+        cache_key = f"{self.tenant_id}:{self.plugin_id}:{self.provider_name}:{self.model_type.value}:{model}"
+        # sort credentials
+        sorted_credentials = sorted(credentials.items()) if credentials else []
+        cache_key += ":".join([hashlib.md5(f"{k}:{v}".encode()).hexdigest() for k, v in sorted_credentials])
+
+        try:
+            contexts.plugin_model_schemas.get()
+        except LookupError:
+            contexts.plugin_model_schemas.set({})
+            contexts.plugin_model_schema_lock.set(Lock())
+
+        with contexts.plugin_model_schema_lock.get():
+            if cache_key in contexts.plugin_model_schemas.get():
+                return contexts.plugin_model_schemas.get()[cache_key]
+
+            schema = plugin_model_manager.get_model_schema(
+                tenant_id=self.tenant_id,
+                user_id="unknown",
+                plugin_id=self.plugin_id,
+                provider=self.provider_name,
+                model_type=self.model_type.value,
+                model=model,
+                credentials=credentials or {},
+            )
+
+            if schema:
+                contexts.plugin_model_schemas.get()[cache_key] = schema
+
+            return schema
 
     def get_customizable_model_schema_from_credentials(self, model: str, credentials: dict) -> Optional[AIModelEntity]:
         """
@@ -157,14 +180,9 @@ class AIModel(BaseModel):
         :param credentials: model credentials
         :return: model schema
         """
-        return self._get_customizable_model_schema(model, credentials)
 
-    def _get_customizable_model_schema(self, model: str, credentials: dict) -> Optional[AIModelEntity]:
-        """
-        Get customizable model schema and fill in the template
-        """
+        # get customizable model schema
         schema = self.get_customizable_model_schema(model, credentials)
-
         if not schema:
             return None
 

api/core/model_runtime/model_providers/model_provider_factory.py

@@ -1,3 +1,4 @@
+import hashlib
 import logging
 import os
 from collections.abc import Sequence
@@ -206,17 +207,35 @@ class ModelProviderFactory:
         Get model schema
         """
         plugin_id, provider_name = self.get_plugin_id_and_provider_name_from_provider(provider)
-        model_schema = self.plugin_model_manager.get_model_schema(
-            tenant_id=self.tenant_id,
-            user_id="unknown",
-            plugin_id=plugin_id,
-            provider=provider_name,
-            model_type=model_type.value,
-            model=model,
-            credentials=credentials,
-        )
+        cache_key = f"{self.tenant_id}:{plugin_id}:{provider_name}:{model_type.value}:{model}"
+        # sort credentials
+        sorted_credentials = sorted(credentials.items()) if credentials else []
+        cache_key += ":".join([hashlib.md5(f"{k}:{v}".encode()).hexdigest() for k, v in sorted_credentials])
 
-        return model_schema
+        try:
+            contexts.plugin_model_schemas.get()
+        except LookupError:
+            contexts.plugin_model_schemas.set({})
+            contexts.plugin_model_schema_lock.set(Lock())
+
+        with contexts.plugin_model_schema_lock.get():
+            if cache_key in contexts.plugin_model_schemas.get():
+                return contexts.plugin_model_schemas.get()[cache_key]
+
+            schema = self.plugin_model_manager.get_model_schema(
+                tenant_id=self.tenant_id,
+                user_id="unknown",
+                plugin_id=plugin_id,
+                provider=provider_name,
+                model_type=model_type.value,
+                model=model,
+                credentials=credentials or {},
+            )
+
+            if schema:
+                contexts.plugin_model_schemas.get()[cache_key] = schema
+
+            return schema
 
     def get_models(
         self,

api/core/plugin/entities/plugin.py

@@ -180,7 +180,7 @@ class ToolProviderID(GenericProviderID):
     def __init__(self, value: str, is_hardcoded: bool = False) -> None:
         super().__init__(value, is_hardcoded)
         if self.organization == "langgenius":
-            if self.provider_name in ["jina", "siliconflow", "stepfun"]:
+            if self.provider_name in ["jina", "siliconflow", "stepfun", "gitee_ai"]:
                 self.plugin_name = f"{self.provider_name}_tool"
 
 

api/core/provider_manager.py

@@ -111,6 +111,12 @@ class ProviderManager:
 
         # Get all provider model records of the workspace
         provider_name_to_provider_model_records_dict = self._get_all_provider_models(tenant_id)
+        for provider_name in list(provider_name_to_provider_model_records_dict.keys()):
+            provider_id = ModelProviderID(provider_name)
+            if str(provider_id) not in provider_name_to_provider_model_records_dict:
+                provider_name_to_provider_model_records_dict[str(provider_id)] = (
+                    provider_name_to_provider_model_records_dict[provider_name]
+                )
 
         # Get all provider entities
         model_provider_factory = ModelProviderFactory(tenant_id)

api/core/rag/datasource/keyword/jieba/jieba.py

@@ -88,17 +88,16 @@ class Jieba(BaseKeyword):
         keyword_table = self._get_dataset_keyword_table()
 
         k = kwargs.get("top_k", 4)
-        document_ids_filter = kwargs.get("document_ids_filter")
+
         sorted_chunk_indices = self._retrieve_ids_by_query(keyword_table or {}, query, k)
 
         documents = []
         for chunk_index in sorted_chunk_indices:
-            segment_query = db.session.query(DocumentSegment).filter(
-                DocumentSegment.dataset_id == self.dataset.id, DocumentSegment.index_node_id == chunk_index
+            segment = (
+                db.session.query(DocumentSegment)
+                .filter(DocumentSegment.dataset_id == self.dataset.id, DocumentSegment.index_node_id == chunk_index)
+                .first()
             )
-            if document_ids_filter:
-                segment_query = segment_query.filter(DocumentSegment.document_id.in_(document_ids_filter))
-            segment = segment_query.first()
 
             if segment:
                 documents.append(

api/core/rag/datasource/retrieval_service.py

@@ -42,7 +42,6 @@ class RetrievalService:
         reranking_model: Optional[dict] = None,
         reranking_mode: str = "reranking_model",
         weights: Optional[dict] = None,
-        document_ids_filter: Optional[list[str]] = None,
     ):
         if not query:
             return []
@@ -66,7 +65,6 @@ class RetrievalService:
                         top_k=top_k,
                         all_documents=all_documents,
                         exceptions=exceptions,
-                        document_ids_filter=document_ids_filter,
                     )
                 )
             if RetrievalMethod.is_support_semantic_search(retrieval_method):
@@ -82,7 +80,6 @@ class RetrievalService:
                         all_documents=all_documents,
                         retrieval_method=retrieval_method,
                         exceptions=exceptions,
-                        document_ids_filter=document_ids_filter,
                     )
                 )
             if RetrievalMethod.is_support_fulltext_search(retrieval_method):
@@ -134,14 +131,7 @@ class RetrievalService:
 
     @classmethod
     def keyword_search(
-        cls,
-        flask_app: Flask,
-        dataset_id: str,
-        query: str,
-        top_k: int,
-        all_documents: list,
-        exceptions: list,
-        document_ids_filter: Optional[list[str]] = None,
+        cls, flask_app: Flask, dataset_id: str, query: str, top_k: int, all_documents: list, exceptions: list
     ):
         with flask_app.app_context():
             try:
@@ -150,10 +140,7 @@ class RetrievalService:
                     raise ValueError("dataset not found")
 
                 keyword = Keyword(dataset=dataset)
-
-                documents = keyword.search(
-                    cls.escape_query_for_search(query), top_k=top_k, document_ids_filter=document_ids_filter
-                )
+                documents = keyword.search(cls.escape_query_for_search(query), top_k=top_k)
                 all_documents.extend(documents)
             except Exception as e:
                 exceptions.append(str(e))
@@ -170,7 +157,6 @@ class RetrievalService:
         all_documents: list,
         retrieval_method: str,
         exceptions: list,
-        document_ids_filter: Optional[list[str]] = None,
     ):
         with flask_app.app_context():
             try:
@@ -185,7 +171,6 @@ class RetrievalService:
                     top_k=top_k,
                     score_threshold=score_threshold,
                     filter={"group_id": [dataset.id]},
-                    document_ids_filter=document_ids_filter,
                 )
 
                 if documents:

api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py

@@ -53,7 +53,7 @@ class AnalyticdbVector(BaseVector):
         self.analyticdb_vector.delete_by_metadata_field(key, value)
 
     def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
-        return self.analyticdb_vector.search_by_vector(query_vector, **kwargs)
+        return self.analyticdb_vector.search_by_vector(query_vector)
 
     def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]:
         return self.analyticdb_vector.search_by_full_text(query, **kwargs)

api/core/rag/datasource/vdb/analyticdb/analyticdb_vector_sql.py

@@ -194,11 +194,6 @@ class AnalyticdbVectorBySql:
 
     def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
         top_k = kwargs.get("top_k", 4)
-        document_ids_filter = kwargs.get("document_ids_filter")
-        where_clause = "WHERE 1=1"
-        if document_ids_filter:
-            document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
-            where_clause += f"AND metadata_->>'document_id' IN ({document_ids})"
         score_threshold = float(kwargs.get("score_threshold") or 0.0)
         with self._get_cursor() as cur:
             query_vector_str = json.dumps(query_vector)
@@ -207,7 +202,7 @@ class AnalyticdbVectorBySql:
                 f"SELECT t.id AS id, t.vector AS vector, (1.0 - t.score) AS score, "
                 f"t.page_content as page_content, t.metadata_ AS metadata_ "
                 f"FROM (SELECT id, vector, page_content, metadata_, vector <=> %s AS score "
-                f"FROM {self.table_name} {where_clause} ORDER BY score LIMIT {top_k} ) t",
+                f"FROM {self.table_name} ORDER BY score LIMIT {top_k} ) t",
                 (query_vector_str,),
             )
             documents = []
@@ -225,17 +220,12 @@ class AnalyticdbVectorBySql:
 
     def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]:
         top_k = kwargs.get("top_k", 4)
-        document_ids_filter = kwargs.get("document_ids_filter")
-        where_clause = ""
-        if document_ids_filter:
-            document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
-            where_clause += f"AND metadata_->>'document_id' IN ({document_ids})"
         with self._get_cursor() as cur:
             cur.execute(
                 f"""SELECT id, vector, page_content, metadata_, 
                 ts_rank(to_tsvector, to_tsquery_from_text(%s, 'zh_cn'), 32) AS score
                 FROM {self.table_name}
-                WHERE to_tsvector@@to_tsquery_from_text(%s, 'zh_cn') {where_clause}
+                WHERE to_tsvector@@to_tsquery_from_text(%s, 'zh_cn')
                 ORDER BY score DESC
                 LIMIT {top_k}""",
                 (f"'{query}'", f"'{query}'"),

api/core/rag/datasource/vdb/baidu/baidu_vector.py

@@ -123,21 +123,11 @@ class BaiduVector(BaseVector):
 
     def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
         query_vector = [float(val) if isinstance(val, np.float64) else val for val in query_vector]
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
-            anns = AnnSearch(
-                vector_field=self.field_vector,
-                vector_floats=query_vector,
-                params=HNSWSearchParams(ef=kwargs.get("ef", 10), limit=kwargs.get("top_k", 4)),
-                filter=f"document_id IN ({document_ids})",
-            )
-        else:
-            anns = AnnSearch(
-                vector_field=self.field_vector,
-                vector_floats=query_vector,
-                params=HNSWSearchParams(ef=kwargs.get("ef", 10), limit=kwargs.get("top_k", 4)),
-            )
+        anns = AnnSearch(
+            vector_field=self.field_vector,
+            vector_floats=query_vector,
+            params=HNSWSearchParams(ef=kwargs.get("ef", 10), limit=kwargs.get("top_k", 4)),
+        )
         res = self._db.table(self._collection_name).search(
             anns=anns,
             projections=[self.field_id, self.field_text, self.field_metadata],

api/core/rag/datasource/vdb/chroma/chroma_vector.py

@@ -95,15 +95,7 @@ class ChromaVector(BaseVector):
 
     def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
         collection = self._client.get_or_create_collection(self._collection_name)
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            results: QueryResult = collection.query(
-                query_embeddings=query_vector,
-                n_results=kwargs.get("top_k", 4),
-                where={"document_id": {"$in": document_ids_filter}},
-            )
-        else:
-            results: QueryResult = collection.query(query_embeddings=query_vector, n_results=kwargs.get("top_k", 4))
+        results: QueryResult = collection.query(query_embeddings=query_vector, n_results=kwargs.get("top_k", 4))
         score_threshold = float(kwargs.get("score_threshold") or 0.0)
 
         # Check if results contain data

api/core/rag/datasource/vdb/elasticsearch/elasticsearch_vector.py

@@ -117,9 +117,6 @@ class ElasticSearchVector(BaseVector):
         top_k = kwargs.get("top_k", 4)
         num_candidates = math.ceil(top_k * 1.5)
         knn = {"field": Field.VECTOR.value, "query_vector": query_vector, "k": top_k, "num_candidates": num_candidates}
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            knn["filter"] = {"terms": {"metadata.document_id": document_ids_filter}}
 
         results = self._client.search(index=self._collection_name, knn=knn, size=top_k)
 
@@ -148,9 +145,6 @@ class ElasticSearchVector(BaseVector):
 
     def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]:
         query_str = {"match": {Field.CONTENT_KEY.value: query}}
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            query_str["filter"] = {"terms": {"metadata.document_id": document_ids_filter}}
         results = self._client.search(index=self._collection_name, query=query_str, size=kwargs.get("top_k", 4))
         docs = []
         for hit in results["hits"]["hits"]:

api/core/rag/datasource/vdb/lindorm/lindorm_vector.py

@@ -168,12 +168,7 @@ class LindormVectorStore(BaseVector):
             raise ValueError("All elements in query_vector should be floats")
 
         top_k = kwargs.get("top_k", 10)
-        document_ids_filter = kwargs.get("document_ids_filter")
-        filters = []
-        if document_ids_filter:
-            filters.append({"terms": {"metadata.document_id": document_ids_filter}})
-        query = default_vector_search_query(query_vector=query_vector, k=top_k, filters=filters, **kwargs)
-
+        query = default_vector_search_query(query_vector=query_vector, k=top_k, **kwargs)
         try:
             params = {}
             if self._using_ugc:
@@ -211,10 +206,7 @@ class LindormVectorStore(BaseVector):
         should = kwargs.get("should")
         minimum_should_match = kwargs.get("minimum_should_match", 0)
         top_k = kwargs.get("top_k", 10)
-        filters = kwargs.get("filter", [])
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            filters.append({"terms": {"metadata.document_id": document_ids_filter}})
+        filters = kwargs.get("filter")
         routing = self._routing
         full_text_query = default_text_search_query(
             query_text=query,

api/core/rag/datasource/vdb/milvus/milvus_vector.py

@@ -72,8 +72,18 @@ class MilvusVector(BaseVector):
         self._client = self._init_client(config)
         self._consistency_level = "Session"  # Consistency level for Milvus operations
         self._fields: list[str] = []  # List of fields in the collection
+        if self._client.has_collection(collection_name):
+            self._load_collection_fields()
         self._hybrid_search_enabled = self._check_hybrid_search_support()  # Check if hybrid search is supported
 
+    def _load_collection_fields(self, fields: Optional[list[str]] = None) -> None:
+        if fields is None:
+            # Load collection fields from remote server
+            collection_info = self._client.describe_collection(self._collection_name)
+            fields = [field["name"] for field in collection_info["fields"]]
+        # Since primary field is auto-id, no need to track it
+        self._fields = [f for f in fields if f != Field.PRIMARY_KEY.value]
+
     def _check_hybrid_search_support(self) -> bool:
         """
         Check if the current Milvus version supports hybrid search.
@@ -218,18 +228,12 @@ class MilvusVector(BaseVector):
         """
         Search for documents by vector similarity.
         """
-        document_ids_filter = kwargs.get("document_ids_filter")
-        filter = ""
-        if document_ids_filter:
-            document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
-            filter = f'metadata["document_id"] in ({document_ids})'
         results = self._client.search(
             collection_name=self._collection_name,
             data=[query_vector],
             anns_field=Field.VECTOR.value,
             limit=kwargs.get("top_k", 4),
             output_fields=[Field.CONTENT_KEY.value, Field.METADATA_KEY.value],
-            filter=filter,
         )
 
         return self._process_search_results(
@@ -245,11 +249,6 @@ class MilvusVector(BaseVector):
         if not self._hybrid_search_enabled or not self.field_exists(Field.SPARSE_VECTOR.value):
             logger.warning("Full-text search is not supported in current Milvus version (requires >= 2.5.0)")
             return []
-        document_ids_filter = kwargs.get("document_ids_filter")
-        filter = ""
-        if document_ids_filter:
-            document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
-            filter = f'metadata["document_id"] in ({document_ids})'
 
         results = self._client.search(
             collection_name=self._collection_name,
@@ -257,7 +256,6 @@ class MilvusVector(BaseVector):
             anns_field=Field.SPARSE_VECTOR.value,
             limit=kwargs.get("top_k", 4),
             output_fields=[Field.CONTENT_KEY.value, Field.METADATA_KEY.value],
-            filter=filter,
         )
 
         return self._process_search_results(
@@ -318,10 +316,7 @@ class MilvusVector(BaseVector):
                     )
                     schema.add_function(bm25_function)
 
-                for x in schema.fields:
-                    self._fields.append(x.name)
-                # Since primary field is auto-id, no need to track it
-                self._fields.remove(Field.PRIMARY_KEY.value)
+                self._load_collection_fields([f.name for f in schema.fields])
 
                 # Create Index params for the collection
                 index_params_obj = IndexParams()

api/core/rag/datasource/vdb/myscale/myscale_vector.py

@@ -131,10 +131,6 @@ class MyScaleVector(BaseVector):
             if self._metric.upper() == "COSINE" and order == SortOrder.ASC and score_threshold > 0.0
             else ""
         )
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
-            where_str = f"{where_str} AND metadata['document_id'] in ({document_ids})"
         sql = f"""
             SELECT text, vector, metadata, {dist} as dist FROM {self._config.database}.{self._collection_name}
             {where_str} ORDER BY dist {order.value} LIMIT {top_k}

api/core/rag/datasource/vdb/oceanbase/oceanbase_vector.py

@@ -154,11 +154,6 @@ class OceanBaseVector(BaseVector):
         return []
 
     def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
-        document_ids_filter = kwargs.get("document_ids_filter")
-        where_clause = None
-        if document_ids_filter:
-            document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
-            where_clause = f"metadata->>'$.document_id' in ({document_ids})"
         ef_search = kwargs.get("ef_search", self._hnsw_ef_search)
         if ef_search != self._hnsw_ef_search:
             self._client.set_ob_hnsw_ef_search(ef_search)
@@ -172,7 +167,6 @@ class OceanBaseVector(BaseVector):
             distance_func=func.l2_distance,
             output_column_names=["text", "metadata"],
             with_dist=True,
-            where_clause=where_clause,
         )
         docs = []
         for text, metadata, distance in cur:

api/core/rag/datasource/vdb/opensearch/opensearch_vector.py

@@ -154,9 +154,6 @@ class OpenSearchVector(BaseVector):
             "size": kwargs.get("top_k", 4),
             "query": {"knn": {Field.VECTOR.value: {Field.VECTOR.value: query_vector, "k": kwargs.get("top_k", 4)}}},
         }
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            query["query"] = {"terms": {"metadata.document_id": document_ids_filter}}
 
         try:
             response = self._client.search(index=self._collection_name.lower(), body=query)
@@ -182,9 +179,6 @@ class OpenSearchVector(BaseVector):
 
     def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]:
         full_text_query = {"query": {"match": {Field.CONTENT_KEY.value: query}}}
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            full_text_query["query"]["terms"] = {"metadata.document_id": document_ids_filter}
 
         response = self._client.search(index=self._collection_name.lower(), body=full_text_query)
 

api/core/rag/datasource/vdb/oracle/oraclevector.py

@@ -23,25 +23,30 @@ oracledb.defaults.fetch_lobs = False
 
 
 class OracleVectorConfig(BaseModel):
-    host: str
-    port: int
     user: str
     password: str
-    database: str
+    dsn: str
+    config_dir: str | None = None
+    wallet_location: str | None = None
+    wallet_password: str | None = None
+    is_autonomous: bool = False
 
     @model_validator(mode="before")
     @classmethod
     def validate_config(cls, values: dict) -> dict:
-        if not values["host"]:
-            raise ValueError("config ORACLE_HOST is required")
-        if not values["port"]:
-            raise ValueError("config ORACLE_PORT is required")
         if not values["user"]:
             raise ValueError("config ORACLE_USER is required")
         if not values["password"]:
             raise ValueError("config ORACLE_PASSWORD is required")
-        if not values["database"]:
-            raise ValueError("config ORACLE_DB is required")
+        if not values["dsn"]:
+            raise ValueError("config ORACLE_DSN is required")
+        if values.get("is_autonomous", False):
+            if not values.get("config_dir"):
+                raise ValueError("config_dir is required for autonomous database")
+            if not values.get("wallet_location"):
+                raise ValueError("wallet_location is required for autonomous database")
+            if not values.get("wallet_password"):
+                raise ValueError("wallet_password is required for autonomous database")
         return values
 
 
@@ -56,7 +61,7 @@ CREATE TABLE IF NOT EXISTS {table_name} (
 SQL_CREATE_INDEX = """
 CREATE INDEX IF NOT EXISTS idx_docs_{table_name} ON {table_name}(text) 
 INDEXTYPE IS CTXSYS.CONTEXT PARAMETERS 
-('FILTER CTXSYS.NULL_FILTER SECTION GROUP CTXSYS.HTML_SECTION_GROUP LEXER sys.my_chinese_vgram_lexer')
+('FILTER CTXSYS.NULL_FILTER SECTION GROUP CTXSYS.HTML_SECTION_GROUP LEXER multilingual_lexer')
 """
 
 
@@ -103,14 +108,25 @@ class OracleVector(BaseVector):
             )
 
     def _create_connection_pool(self, config: OracleVectorConfig):
-        return oracledb.create_pool(
-            user=config.user,
-            password=config.password,
-            dsn="{}:{}/{}".format(config.host, config.port, config.database),
-            min=1,
-            max=50,
-            increment=1,
-        )
+        pool_params = {
+            "user": config.user,
+            "password": config.password,
+            "dsn": config.dsn,
+            "min": 1,
+            "max": 50,
+            "increment": 1,
+        }
+
+        if config.is_autonomous:
+            pool_params.update(
+                {
+                    "config_dir": config.config_dir,
+                    "wallet_location": config.wallet_location,
+                    "wallet_password": config.wallet_password,
+                }
+            )
+
+        return oracledb.create_pool(**pool_params)
 
     @contextmanager
     def _get_cursor(self):
@@ -185,15 +201,10 @@ class OracleVector(BaseVector):
         :return: List of Documents that are nearest to the query vector.
         """
         top_k = kwargs.get("top_k", 4)
-        document_ids_filter = kwargs.get("document_ids_filter")
-        where_clause = ""
-        if document_ids_filter:
-            document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
-            where_clause = f"WHERE metadata->>'document_id' in ({document_ids})"
         with self._get_cursor() as cur:
             cur.execute(
                 f"SELECT meta, text, vector_distance(embedding,:1) AS distance FROM {self.table_name}"
-                f" {where_clause} ORDER BY distance fetch first {top_k} rows only",
+                f" ORDER BY distance fetch first {top_k} rows only",
                 [numpy.array(query_vector)],
             )
             docs = []
@@ -246,15 +257,9 @@ class OracleVector(BaseVector):
                     if token not in stop_words:
                         entities.append(token)
             with self._get_cursor() as cur:
-                document_ids_filter = kwargs.get("document_ids_filter")
-                where_clause = ""
-                if document_ids_filter:
-                    document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
-                    where_clause = f" AND metadata->>'document_id' in ({document_ids}) "
                 cur.execute(
                     f"select meta, text, embedding FROM {self.table_name}"
-                    f"WHERE CONTAINS(text, :1, 1) > 0 {where_clause} "
-                    f"order by score(1) desc fetch first {top_k} rows only",
+                    f" WHERE CONTAINS(text, :1, 1) > 0 order by score(1) desc fetch first {top_k} rows only",
                     [" ACCUM ".join(entities)],
                 )
                 docs = []
@@ -298,10 +303,12 @@ class OracleVectorFactory(AbstractVectorFactory):
         return OracleVector(
             collection_name=collection_name,
             config=OracleVectorConfig(
-                host=dify_config.ORACLE_HOST or "localhost",
-                port=dify_config.ORACLE_PORT,
                 user=dify_config.ORACLE_USER or "system",
                 password=dify_config.ORACLE_PASSWORD or "oracle",
-                database=dify_config.ORACLE_DATABASE or "orcl",
+                dsn=dify_config.ORACLE_DSN or "oracle:1521/freepdb1",
+                config_dir=dify_config.ORACLE_CONFIG_DIR,
+                wallet_location=dify_config.ORACLE_WALLET_LOCATION,
+                wallet_password=dify_config.ORACLE_WALLET_PASSWORD,
+                is_autonomous=dify_config.ORACLE_IS_AUTONOMOUS,
             ),
         )

api/core/rag/datasource/vdb/pgvecto_rs/pgvecto_rs.py

@@ -189,9 +189,6 @@ class PGVectoRS(BaseVector):
                 .limit(kwargs.get("top_k", 4))
                 .order_by("distance")
             )
-            document_ids_filter = kwargs.get("document_ids_filter")
-            if document_ids_filter:
-                stmt = stmt.where(self._table.meta["document_id"].in_(document_ids_filter))
             res = session.execute(stmt)
             results = [(row[0], row[1]) for row in res]
 

api/core/rag/datasource/vdb/pgvector/pgvector.py

@@ -155,16 +155,10 @@ class PGVector(BaseVector):
         :return: List of Documents that are nearest to the query vector.
         """
         top_k = kwargs.get("top_k", 4)
-        document_ids_filter = kwargs.get("document_ids_filter")
-        where_clause = ""
-        if document_ids_filter:
-            document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
-            where_clause = f" WHERE metadata->>'document_id' in ({document_ids}) "
 
         with self._get_cursor() as cur:
             cur.execute(
                 f"SELECT meta, text, embedding <=> %s AS distance FROM {self.table_name}"
-                f" {where_clause}"
                 f" ORDER BY distance LIMIT {top_k}",
                 (json.dumps(query_vector),),
             )
@@ -182,16 +176,10 @@ class PGVector(BaseVector):
         top_k = kwargs.get("top_k", 5)
 
         with self._get_cursor() as cur:
-            document_ids_filter = kwargs.get("document_ids_filter")
-            where_clause = ""
-            if document_ids_filter:
-                document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
-                where_clause = f" AND metadata->>'document_id' in ({document_ids}) "
             cur.execute(
                 f"""SELECT meta, text, ts_rank(to_tsvector(coalesce(text, '')), plainto_tsquery(%s)) AS score
                 FROM {self.table_name}
                 WHERE to_tsvector(text) @@ plainto_tsquery(%s)
-                {where_clause}
                 ORDER BY score DESC
                 LIMIT {top_k}""",
                 # f"'{query}'" is required in order to account for whitespace in query

api/core/rag/datasource/vdb/qdrant/qdrant_vector.py

@@ -286,26 +286,27 @@ class QdrantVector(BaseVector):
         from qdrant_client.http import models
         from qdrant_client.http.exceptions import UnexpectedResponse
 
-        try:
-            filter = models.Filter(
-                must=[
-                    models.FieldCondition(
-                        key="metadata.doc_id",
-                        match=models.MatchAny(any=ids),
-                    ),
-                ],
-            )
-            self._client.delete(
-                collection_name=self._collection_name,
-                points_selector=FilterSelector(filter=filter),
-            )
-        except UnexpectedResponse as e:
-            # Collection does not exist, so return
-            if e.status_code == 404:
-                return
-            # Some other error occurred, so re-raise the exception
-            else:
-                raise e
+        for node_id in ids:
+            try:
+                filter = models.Filter(
+                    must=[
+                        models.FieldCondition(
+                            key="metadata.doc_id",
+                            match=models.MatchValue(value=node_id),
+                        ),
+                    ],
+                )
+                self._client.delete(
+                    collection_name=self._collection_name,
+                    points_selector=FilterSelector(filter=filter),
+                )
+            except UnexpectedResponse as e:
+                # Collection does not exist, so return
+                if e.status_code == 404:
+                    return
+                # Some other error occurred, so re-raise the exception
+                else:
+                    raise e
 
     def text_exists(self, id: str) -> bool:
         all_collection_name = []
@@ -330,14 +331,6 @@ class QdrantVector(BaseVector):
                 ),
             ],
         )
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            filter.must.append(
-                models.FieldCondition(
-                    key="metadata.document_id",
-                    match=models.MatchAny(any=document_ids_filter),
-                )
-            )
         results = self._client.search(
             collection_name=self._collection_name,
             query_vector=query_vector,
@@ -384,14 +377,6 @@ class QdrantVector(BaseVector):
                 ),
             ]
         )
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            scroll_filter.must.append(
-                models.FieldCondition(
-                    key="metadata.document_id",
-                    match=models.MatchAny(any=document_ids_filter),
-                )
-            )
         response = self._client.scroll(
             collection_name=self._collection_name,
             scroll_filter=scroll_filter,

api/core/rag/datasource/vdb/relyt/relyt_vector.py

@@ -223,12 +223,8 @@ class RelytVector(BaseVector):
         return len(result) > 0
 
     def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
-        document_ids_filter = kwargs.get("document_ids_filter")
-        filter = kwargs.get("filter", {})
-        if document_ids_filter:
-            filter["document_id"] = document_ids_filter
         results = self.similarity_search_with_score_by_vector(
-            k=int(kwargs.get("top_k", 4)), embedding=query_vector, filter=filter
+            k=int(kwargs.get("top_k", 4)), embedding=query_vector, filter=kwargs.get("filter")
         )
 
         # Organize results.
@@ -250,9 +246,9 @@ class RelytVector(BaseVector):
         filter_condition = ""
         if filter is not None:
             conditions = [
-                f"metadata->>'{key!r}' in ({', '.join(map(repr, value))})"
+                f"metadata->>{key!r} in ({', '.join(map(repr, value))})"
                 if len(value) > 1
-                else f"metadata->>'{key!r}' = {value[0]!r}"
+                else f"metadata->>{key!r} = {value[0]!r}"
                 for key, value in filter.items()
             ]
             filter_condition = f"WHERE {' AND '.join(conditions)}"

api/core/rag/datasource/vdb/tencent/tencent_vector.py

@@ -145,16 +145,11 @@ class TencentVector(BaseVector):
         self._db.collection(self._collection_name).delete(document_ids=ids)
 
     def delete_by_metadata_field(self, key: str, value: str) -> None:
-        self._db.collection(self._collection_name).delete(filter=Filter(Filter.In(f"metadata.{key}", [value])))
+        self._db.collection(self._collection_name).delete(filter=Filter(Filter.In(key, [value])))
 
     def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
-        document_ids_filter = kwargs.get("document_ids_filter")
-        filter = None
-        if document_ids_filter:
-            filter = Filter(Filter.In("metadata.document_id", document_ids_filter))
         res = self._db.collection(self._collection_name).search(
             vectors=[query_vector],
-            filter=filter,
             params=document.HNSWSearchParams(ef=kwargs.get("ef", 10)),
             retrieve_vector=False,
             limit=kwargs.get("top_k", 4),

api/core/rag/datasource/vdb/tidb_on_qdrant/tidb_on_qdrant_vector.py

@@ -326,14 +326,6 @@ class TidbOnQdrantVector(BaseVector):
                 ),
             ],
         )
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            filter.must.append(
-                models.FieldCondition(
-                    key="metadata.document_id",
-                    match=models.MatchAny(any=document_ids_filter),
-                )
-            )
         results = self._client.search(
             collection_name=self._collection_name,
             query_vector=query_vector,
@@ -376,14 +368,6 @@ class TidbOnQdrantVector(BaseVector):
                 )
             ]
         )
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            scroll_filter.must.append(
-                models.FieldCondition(
-                    key="metadata.document_id",
-                    match=models.MatchAny(any=document_ids_filter),
-                )
-            )
         response = self._client.scroll(
             collection_name=self._collection_name,
             scroll_filter=scroll_filter,

api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py

@@ -196,11 +196,6 @@ class TiDBVector(BaseVector):
 
         docs = []
         tidb_dist_func = self._get_distance_func()
-        document_ids_filter = kwargs.get("document_ids_filter")
-        where_clause = ""
-        if document_ids_filter:
-            document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
-            where_clause = f" WHERE meta->>'$.document_id' in ({document_ids}) "
 
         with Session(self._engine) as session:
             select_statement = sql_text(f"""
@@ -211,7 +206,6 @@ class TiDBVector(BaseVector):
                     text,
                     {tidb_dist_func}(vector, :query_vector_str) AS distance
                   FROM {self._collection_name}
-                  {where_clause}
                   ORDER BY distance ASC
                   LIMIT :top_k
                 ) t

api/core/rag/datasource/vdb/upstash/upstash_vector.py

@@ -88,20 +88,7 @@ class UpstashVector(BaseVector):
 
     def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
         top_k = kwargs.get("top_k", 4)
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            document_ids = ", ".join(f"'{id}'" for id in document_ids_filter)
-            filter = f"document_id in ({document_ids})"
-        else:
-            filter = ""
-        result = self.index.query(
-            vector=query_vector,
-            top_k=top_k,
-            include_metadata=True,
-            include_data=True,
-            include_vectors=False,
-            filter=filter,
-        )
+        result = self.index.query(vector=query_vector, top_k=top_k, include_metadata=True, include_data=True)
         docs = []
         score_threshold = float(kwargs.get("score_threshold") or 0.0)
         for record in result:

api/core/rag/datasource/vdb/vector_base.py

@@ -49,10 +49,6 @@ class BaseVector(ABC):
     def delete(self) -> None:
         raise NotImplementedError
 
-    @abstractmethod
-    def update_metadata(self, document_id: str, metadata: dict) -> None:
-        raise NotImplementedError
-
     def _filter_duplicate_texts(self, texts: list[Document]) -> list[Document]:
         for text in texts.copy():
             if text.metadata and "doc_id" in text.metadata:

api/core/rag/datasource/vdb/vikingdb/vikingdb_vector.py

@@ -177,11 +177,7 @@ class VikingDBVector(BaseVector):
             query_vector, limit=kwargs.get("top_k", 4)
         )
         score_threshold = float(kwargs.get("score_threshold") or 0.0)
-        docs = self._get_search_res(results, score_threshold)
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            docs = [doc for doc in docs if doc.metadata.get("document_id") in document_ids_filter]
-        return docs
+        return self._get_search_res(results, score_threshold)
 
     def _get_search_res(self, results, score_threshold) -> list[Document]:
         if len(results) == 0:

api/core/rag/datasource/vdb/weaviate/weaviate_vector.py

@@ -168,16 +168,16 @@ class WeaviateVector(BaseVector):
         # check whether the index already exists
         schema = self._default_schema(self._collection_name)
         if self._client.schema.contains(schema):
-            try:
-                self._client.batch.delete_objects(
-                    class_name=self._collection_name,
-                    where={"operator": "ContainsAny", "path": ["id"], "valueTextArray": ids},
-                    output="minimal",
-                )
-            except weaviate.UnexpectedStatusCodeException as e:
-                # tolerate not found error
-                if e.status_code != 404:
-                    raise e
+            for uuid in ids:
+                try:
+                    self._client.data_object.delete(
+                        class_name=self._collection_name,
+                        uuid=uuid,
+                    )
+                except weaviate.UnexpectedStatusCodeException as e:
+                    # tolerate not found error
+                    if e.status_code != 404:
+                        raise e
 
     def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
         """Look up similar documents by embedding vector in Weaviate."""
@@ -187,10 +187,8 @@ class WeaviateVector(BaseVector):
         query_obj = self._client.query.get(collection_name, properties)
 
         vector = {"vector": query_vector}
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            where_filter = {"operator": "ContainsAny", "path": ["document_id"], "valueTextArray": document_ids_filter}
-            query_obj = query_obj.with_where(where_filter)
+        if kwargs.get("where_filter"):
+            query_obj = query_obj.with_where(kwargs.get("where_filter"))
         result = (
             query_obj.with_near_vector(vector)
             .with_limit(kwargs.get("top_k", 4))
@@ -235,10 +233,8 @@ class WeaviateVector(BaseVector):
         if kwargs.get("search_distance"):
             content["certainty"] = kwargs.get("search_distance")
         query_obj = self._client.query.get(collection_name, properties)
-        document_ids_filter = kwargs.get("document_ids_filter")
-        if document_ids_filter:
-            where_filter = {"operator": "ContainsAny", "path": ["document_id"], "valueTextArray": document_ids_filter}
-            query_obj = query_obj.with_where(where_filter)
+        if kwargs.get("where_filter"):
+            query_obj = query_obj.with_where(kwargs.get("where_filter"))
         query_obj = query_obj.with_additional(["vector"])
         properties = ["text"]
         result = query_obj.with_bm25(query=query, properties=properties).with_limit(kwargs.get("top_k", 4)).do()

api/core/rag/index_processor/constant/built_in_field.py

@@ -1,9 +0,0 @@
-from enum import Enum
-
-
-class BuiltInField(str, Enum):
-    document_name = "document_name"
-    uploader = "uploader"
-    upload_date = "upload_date"
-    last_update_date = "last_update_date"
-    source = "source"

api/core/rag/retrieval/dataset_retrieval.py

@@ -203,6 +203,7 @@ class DatasetRetrieval:
                                 "segment_id": segment.id,
                                 "retriever_from": invoke_from.to_source(),
                                 "score": record.score or 0.0,
+                                "doc_metadata": document.doc_metadata,
                             }
 
                             if invoke_from.to_source() == "dev":
@@ -237,7 +238,6 @@ class DatasetRetrieval:
         model_config: ModelConfigWithCredentialsEntity,
         planning_strategy: PlanningStrategy,
         message_id: Optional[str] = None,
-        metadata_filter_document_ids: Optional[dict[str, list[str]]] = None,
     ):
         tools = []
         for dataset in available_datasets:
@@ -292,11 +292,6 @@ class DatasetRetrieval:
                             document.metadata["dataset_name"] = dataset.name
                         results.append(document)
                 else:
-                    document_ids_filter = None
-                    if metadata_filter_document_ids:
-                        document_ids = metadata_filter_document_ids.get(dataset.id, [])
-                        if document_ids:
-                            document_ids_filter = document_ids
                     retrieval_model_config = dataset.retrieval_model or default_retrieval_model
 
                     # get top k
@@ -328,7 +323,6 @@ class DatasetRetrieval:
                             reranking_model=reranking_model,
                             reranking_mode=retrieval_model_config.get("reranking_mode", "reranking_model"),
                             weights=retrieval_model_config.get("weights", None),
-                            document_ids_filter=document_ids_filter,
                         )
                 self._on_query(query, [dataset_id], app_id, user_from, user_id)
 

api/core/tools/entities/tool_entities.py

@@ -185,7 +185,7 @@ class ToolInvokeMessage(BaseModel):
     """
         plain text, image url or link url
     """
-    message: JsonMessage | TextMessage | BlobMessage | VariableMessage | FileMessage | LogMessage | None
+    message: JsonMessage | TextMessage | BlobMessage | LogMessage | FileMessage | None | VariableMessage
     meta: dict[str, Any] | None = None
 
     @field_validator("message", mode="before")

api/core/tools/tool_manager.py

@@ -194,7 +194,7 @@ class ToolManager:
                     db.session.query(BuiltinToolProvider)
                     .filter(
                         BuiltinToolProvider.tenant_id == tenant_id,
-                        (BuiltinToolProvider.provider == provider_id)
+                        (BuiltinToolProvider.provider == str(provider_id_entity))
                         | (BuiltinToolProvider.provider == provider_id_entity.provider_name),
                     )
                     .first()

api/core/tools/utils/dataset_retriever/dataset_multi_retriever_tool.py

@@ -123,6 +123,7 @@ class DatasetMultiRetrieverTool(DatasetRetrieverBaseTool):
                             "segment_id": segment.id,
                             "retriever_from": self.retriever_from,
                             "score": document_score_list.get(segment.index_node_id, None),
+                            "doc_metadata": document.doc_metadata,
                         }
 
                         if self.retriever_from == "dev":

api/core/tools/utils/dataset_retriever/dataset_retriever_tool.py

@@ -172,6 +172,7 @@ class DatasetRetrieverTool(DatasetRetrieverBaseTool):
                                     "segment_id": segment.id,
                                     "retriever_from": self.retriever_from,
                                     "score": record.score or 0.0,
+                                    "doc_metadata": document.doc_metadata,  # type: ignore
                                 }
 
                                 if self.retriever_from == "dev":

api/core/workflow/graph_engine/entities/graph.py

@@ -590,8 +590,6 @@ class Graph(BaseModel):
                             start_node_id=node_id,
                             routes_node_ids=routes_node_ids,
                         )
-                        # Exclude conditional branch nodes
-                        and all(edge.run_condition is None for edge in reverse_edge_mapping.get(node_id, []))
                     ):
                         if node_id not in merge_branch_node_ids:
                             merge_branch_node_ids[node_id] = []

api/core/workflow/nodes/agent/agent_node.py

@@ -8,12 +8,12 @@ from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from core.plugin.manager.exc import PluginDaemonClientSideError
 from core.plugin.manager.plugin import PluginInstallationManager
-from core.tools.entities.tool_entities import ToolProviderType
+from core.tools.entities.tool_entities import ToolParameter, ToolProviderType
 from core.tools.tool_manager import ToolManager
 from core.workflow.entities.node_entities import NodeRunResult
 from core.workflow.entities.variable_pool import VariablePool
 from core.workflow.enums import SystemVariableKey
-from core.workflow.nodes.agent.entities import AgentNodeData
+from core.workflow.nodes.agent.entities import AgentNodeData, ParamsAutoGenerated
 from core.workflow.nodes.base.entities import BaseNodeData
 from core.workflow.nodes.enums import NodeType
 from core.workflow.nodes.event.event import RunCompletedEvent
@@ -156,16 +156,38 @@ class AgentNode(ToolNode):
                 value = cast(list[dict[str, Any]], value)
                 value = [tool for tool in value if tool.get("enabled", False)]
 
+                for tool in value:
+                    if "schemas" in tool:
+                        tool.pop("schemas")
+                    parameters = tool.get("parameters", {})
+                    if all(isinstance(v, dict) for _, v in parameters.items()):
+                        params = {}
+                        for key, param in parameters.items():
+                            if param.get("auto", ParamsAutoGenerated.OPEN.value) == ParamsAutoGenerated.CLOSE.value:
+                                value_param = param.get("value", {})
+                                params[key] = value_param.get("value", "") if value_param is not None else None
+                            else:
+                                params[key] = None
+                        parameters = params
+                    tool["settings"] = {k: v.get("value", None) for k, v in tool.get("settings", {}).items()}
+                    tool["parameters"] = parameters
+
             if not for_log:
                 if parameter.type == "array[tools]":
                     value = cast(list[dict[str, Any]], value)
                     tool_value = []
                     for tool in value:
+                        provider_type = ToolProviderType(tool.get("type", ToolProviderType.BUILT_IN.value))
+                        setting_params = tool.get("settings", {})
+                        parameters = tool.get("parameters", {})
+                        manual_input_params = [key for key, value in parameters.items() if value is not None]
+
+                        parameters = {**parameters, **setting_params}
                         entity = AgentToolEntity(
                             provider_id=tool.get("provider_name", ""),
-                            provider_type=ToolProviderType.BUILT_IN,
+                            provider_type=provider_type,
                             tool_name=tool.get("tool_name", ""),
-                            tool_parameters=tool.get("parameters", {}),
+                            tool_parameters=parameters,
                             plugin_unique_identifier=tool.get("plugin_unique_identifier", None),
                         )
 
@@ -178,11 +200,26 @@ class AgentNode(ToolNode):
                             tool_runtime.entity.description.llm = (
                                 extra.get("descrption", "") or tool_runtime.entity.description.llm
                             )
-
+                        for tool_runtime_params in tool_runtime.entity.parameters:
+                            tool_runtime_params.form = (
+                                ToolParameter.ToolParameterForm.FORM
+                                if tool_runtime_params.name in manual_input_params
+                                else tool_runtime_params.form
+                            )
+                        manual_input_value = {}
+                        if tool_runtime.entity.parameters:
+                            manual_input_value = {
+                                key: value for key, value in parameters.items() if key in manual_input_params
+                            }
+                        runtime_parameters = {
+                            **tool_runtime.runtime.runtime_parameters,
+                            **manual_input_value,
+                        }
                         tool_value.append(
                             {
                                 **tool_runtime.entity.model_dump(mode="json"),
-                                "runtime_parameters": tool_runtime.runtime.runtime_parameters,
+                                "runtime_parameters": runtime_parameters,
+                                "provider_type": provider_type.value,
                             }
                         )
                     value = tool_value

api/core/workflow/nodes/agent/entities.py

@@ -1,3 +1,4 @@
+from enum import Enum
 from typing import Any, Literal, Union
 
 from pydantic import BaseModel
@@ -16,3 +17,8 @@ class AgentNodeData(BaseNodeData):
         type: Literal["mixed", "variable", "constant"]
 
     agent_parameters: dict[str, AgentInput]
+
+
+class ParamsAutoGenerated(Enum):
+    CLOSE = 0
+    OPEN = 1

api/core/workflow/nodes/knowledge_retrieval/entities.py

@@ -1,10 +1,8 @@
-from collections.abc import Sequence
 from typing import Any, Literal, Optional
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 
 from core.workflow.nodes.base import BaseNodeData
-from core.workflow.nodes.llm.entities import VisionConfig
 
 
 class RerankingModelConfig(BaseModel):
@@ -75,48 +73,6 @@ class SingleRetrievalConfig(BaseModel):
     model: ModelConfig
 
 
-SupportedComparisonOperator = Literal[
-    # for string or array
-    "contains",
-    "not contains",
-    "starts with",
-    "ends with",
-    "is",
-    "is not",
-    "empty",
-    "is not empty",
-    # for number
-    "=",
-    "≠",
-    ">",
-    "<",
-    "≥",
-    "≤",
-    # for time
-    "before",
-    "after",
-]
-
-
-class Condition(BaseModel):
-    """
-    Conditon detail
-    """
-
-    metadata_name: str
-    comparison_operator: SupportedComparisonOperator
-    value: str | Sequence[str] | None = None
-
-
-class MetadataFilteringCondition(BaseModel):
-    """
-    Metadata Filtering Condition.
-    """
-
-    logical_operator: Optional[Literal["and", "or"]] = "and"
-    conditions: Optional[list[Condition]] = Field(default=None, deprecated=True)
-
-
 class KnowledgeRetrievalNodeData(BaseNodeData):
     """
     Knowledge retrieval Node Data.
@@ -128,7 +84,3 @@ class KnowledgeRetrievalNodeData(BaseNodeData):
     retrieval_mode: Literal["single", "multiple"]
     multiple_retrieval_config: Optional[MultipleRetrievalConfig] = None
     single_retrieval_config: Optional[SingleRetrievalConfig] = None
-    metadata_filtering_mode: Optional[Literal["disabled", "automatic", "manual"]] = "disabled"
-    metadata_model_config: Optional[ModelConfig] = None
-    metadata_filtering_conditions: Optional[MetadataFilteringCondition] = None
-    vision: VisionConfig = Field(default_factory=VisionConfig)

api/core/workflow/nodes/knowledge_retrieval/exc.py

@@ -16,7 +16,3 @@ class ModelNotSupportedError(KnowledgeRetrievalNodeError):
 
 class ModelQuotaExceededError(KnowledgeRetrievalNodeError):
     """Raised when the model provider quota is exceeded."""
-
-
-class InvalidModelTypeError(KnowledgeRetrievalNodeError):
-    """Raised when the model is not a Large Language Model."""

api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py

@@ -1,8 +1,6 @@
-import json
 import logging
-from collections import defaultdict
 from collections.abc import Mapping, Sequence
-from typing import Any, Optional, cast
+from typing import Any, cast
 
 from sqlalchemy import func
 
@@ -11,38 +9,21 @@ from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEnti
 from core.entities.agent_entities import PlanningStrategy
 from core.entities.model_entities import ModelStatus
 from core.model_manager import ModelInstance, ModelManager
-from core.model_runtime.entities.message_entities import PromptMessageRole
-from core.model_runtime.entities.model_entities import ModelFeature, ModelPropertyKey, ModelType
+from core.model_runtime.entities.model_entities import ModelFeature, ModelType
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
-from core.prompt.advanced_prompt_transform import AdvancedPromptTransform
-from core.prompt.simple_prompt_transform import ModelMode
 from core.rag.datasource.retrieval_service import RetrievalService
 from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
 from core.rag.retrieval.retrieval_methods import RetrievalMethod
 from core.variables import StringSegment
 from core.workflow.entities.node_entities import NodeRunResult
+from core.workflow.nodes.base import BaseNode
 from core.workflow.nodes.enums import NodeType
-from core.workflow.nodes.event.event import ModelInvokeCompletedEvent
-from core.workflow.nodes.knowledge_retrieval.template_prompts import (
-    METADATA_FILTER_ASSISTANT_PROMPT_1,
-    METADATA_FILTER_ASSISTANT_PROMPT_2,
-    METADATA_FILTER_COMPLETION_PROMPT,
-    METADATA_FILTER_SYSTEM_PROMPT,
-    METADATA_FILTER_USER_PROMPT_1,
-    METADATA_FILTER_USER_PROMPT_3,
-)
-from core.workflow.nodes.list_operator.exc import InvalidConditionError
-from core.workflow.nodes.llm.entities import LLMNodeChatModelMessage, LLMNodeCompletionModelPromptTemplate
-from core.workflow.nodes.llm.node import LLMNode
-from core.workflow.nodes.question_classifier.template_prompts import QUESTION_CLASSIFIER_USER_PROMPT_2
 from extensions.ext_database import db
-from libs.json_in_md_parser import parse_and_check_json_markdown
-from models.dataset import Dataset, DatasetMetadata, Document
+from models.dataset import Dataset, Document
 from models.workflow import WorkflowNodeExecutionStatus
 
 from .entities import KnowledgeRetrievalNodeData
 from .exc import (
-    InvalidModelTypeError,
     KnowledgeRetrievalNodeError,
     ModelCredentialsNotInitializedError,
     ModelNotExistError,
@@ -61,14 +42,13 @@ default_retrieval_model = {
 }
 
 
-class KnowledgeRetrievalNode(LLMNode):
+class KnowledgeRetrievalNode(BaseNode[KnowledgeRetrievalNodeData]):
     _node_data_cls = KnowledgeRetrievalNodeData
     _node_type = NodeType.KNOWLEDGE_RETRIEVAL
 
     def _run(self) -> NodeRunResult:
-        node_data = cast(KnowledgeRetrievalNodeData, self.node_data)
         # extract variables
-        variable = self.graph_runtime_state.variable_pool.get(node_data.query_variable_selector)
+        variable = self.graph_runtime_state.variable_pool.get(self.node_data.query_variable_selector)
         if not isinstance(variable, StringSegment):
             return NodeRunResult(
                 status=WorkflowNodeExecutionStatus.FAILED,
@@ -83,7 +63,7 @@ class KnowledgeRetrievalNode(LLMNode):
             )
         # retrieve knowledge
         try:
-            results = self._fetch_dataset_retriever(node_data=node_data, query=query)
+            results = self._fetch_dataset_retriever(node_data=self.node_data, query=query)
             outputs = {"result": results}
             return NodeRunResult(
                 status=WorkflowNodeExecutionStatus.SUCCEEDED, inputs=variables, process_data=None, outputs=outputs
@@ -137,9 +117,6 @@ class KnowledgeRetrievalNode(LLMNode):
             if not dataset:
                 continue
             available_datasets.append(dataset)
-        metadata_filter_document_ids = self._get_metadata_filter_condition(
-            [dataset.id for dataset in available_datasets], query, node_data
-        )
         all_documents = []
         dataset_retrieval = DatasetRetrieval()
         if node_data.retrieval_mode == DatasetRetrieveConfigEntity.RetrieveStrategy.SINGLE.value:
@@ -169,7 +146,6 @@ class KnowledgeRetrievalNode(LLMNode):
                     model_config=model_config,
                     model_instance=model_instance,
                     planning_strategy=planning_strategy,
-                    metadata_filter_document_ids=metadata_filter_document_ids,
                 )
         elif node_data.retrieval_mode == DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE.value:
             if node_data.multiple_retrieval_config is None:
@@ -264,6 +240,7 @@ class KnowledgeRetrievalNode(LLMNode):
                                 "segment_word_count": segment.word_count,
                                 "segment_position": segment.position,
                                 "segment_index_node_hash": segment.index_node_hash,
+                                "doc_metadata": document.doc_metadata,
                             },
                             "title": document.name,
                         }
@@ -282,134 +259,6 @@ class KnowledgeRetrievalNode(LLMNode):
                 item["metadata"]["position"] = position
         return retrieval_resource_list
 
-    def _get_metadata_filter_condition(
-        self, dataset_ids: list, query: str, node_data: KnowledgeRetrievalNodeData
-    ) -> dict[str, list[str]]:
-        document_query = db.session.query(Document.id).filter(
-            Document.dataset_id.in_(dataset_ids),
-            Document.indexing_status == "completed",
-            Document.enabled == True,
-            Document.archived == False,
-        )
-        if node_data.metadata_filtering_mode == "disabled":
-            return None
-        elif node_data.metadata_filtering_mode == "automatic":
-            automatic_metadata_filters = self._automatic_metadata_filter_func(dataset_ids, query, node_data)
-            if automatic_metadata_filters:
-                for filter in automatic_metadata_filters:
-                    self._process_metadata_filter_func(
-                        filter.get("condition"), filter.get("metadata_name"), filter.get("value"), document_query
-                    )
-        elif node_data.metadata_filtering_mode == "manual":
-            for condition in node_data.metadata_filtering_conditions.conditions:
-                metadata_name = condition.metadata_name
-                expected_value = condition.value
-                if isinstance(expected_value, str):
-                    expected_value = self.graph_runtime_state.variable_pool.convert_template(expected_value).text
-                self._process_metadata_filter_func(
-                    condition.comparison_operator, metadata_name, expected_value, document_query
-                )
-        else:
-            raise ValueError("Invalid metadata filtering mode")
-        documnents = document_query.all()
-        # group by dataset_id
-        metadata_filter_document_ids = defaultdict(list)
-        for document in documnents:
-            metadata_filter_document_ids[document.dataset_id].append(document.id)
-        return metadata_filter_document_ids
-
-    def _automatic_metadata_filter_func(
-        self, dataset_ids: list, query: str, node_data: KnowledgeRetrievalNodeData
-    ) -> list[dict[str, Any]]:
-        # get all metadata field
-        metadata_fields = db.session.query(DatasetMetadata).filter(DatasetMetadata.dataset_id.in_(dataset_ids)).all()
-        all_metadata_fields = [metadata_field.field_name for metadata_field in metadata_fields]
-        # get metadata model config
-        metadata_model_config = node_data.metadata_model_config
-        if metadata_model_config is None:
-            raise ValueError("metadata_model_config is required")
-        # get metadata model instance
-        # fetch model config
-        model_instance, model_config = self._fetch_model_config(node_data.metadata_model_config)
-        # fetch prompt messages
-        prompt_template = self._get_prompt_template(
-            node_data=node_data,
-            query=query or "",
-            metadata_fields=all_metadata_fields,
-        )
-        prompt_messages, stop = self._fetch_prompt_messages(
-            prompt_template=prompt_template,
-            sys_query=query,
-            memory=None,
-            model_config=model_config,
-            sys_files=[],
-            vision_enabled=node_data.vision.enabled,
-            vision_detail=node_data.vision.configs.detail,
-            variable_pool=self.graph_runtime_state.variable_pool,
-            jinja2_variables=[],
-        )
-
-        result_text = ""
-        try:
-            # handle invoke result
-            generator = self._invoke_llm(
-                node_data_model=node_data.model,
-                model_instance=model_instance,
-                prompt_messages=prompt_messages,
-                stop=stop,
-            )
-
-            for event in generator:
-                if isinstance(event, ModelInvokeCompletedEvent):
-                    result_text = event.text
-                    break
-
-            result_text_json = parse_and_check_json_markdown(result_text, [])
-            automatic_metadata_filters = []
-            if "metadata_map" in result_text_json:
-                metadata_map = result_text_json["metadata_map"]
-                for item in metadata_map:
-                    if item.get("metadata_field_name") in all_metadata_fields:
-                        automatic_metadata_filters.append(
-                            {
-                                "metadata_name": item.get("metadata_field_name"),
-                                "value": item.get("metadata_field_value"),
-                                "condition": item.get("comparison_operator"),
-                            }
-                        )
-        except Exception as e:
-            return None
-        return automatic_metadata_filters
-
-    def _process_metadata_filter_func(*, condition: str, metadata_name: str, value: str, query):
-        match condition:
-            case "contains":
-                query = query.filter(Document.doc_metadata[metadata_name].like(f"%{value}%"))
-            case "not contains":
-                query = query.filter(Document.doc_metadata[metadata_name].notlike(f"%{value}%"))
-            case "start with":
-                query = query.filter(Document.doc_metadata[metadata_name].like(f"{value}%"))
-            case "end with":
-                query = query.filter(Document.doc_metadata[metadata_name].like(f"%{value}"))
-            case "is", "=":
-                query = query.filter(Document.doc_metadata[metadata_name] == value)
-            case "is not", "≠":
-                query = query.filter(Document.doc_metadata[metadata_name] != value)
-            case "is empty":
-                query = query.filter(Document.doc_metadata[metadata_name].is_(None))
-            case "is not empty":
-                query = query.filter(Document.doc_metadata[metadata_name].isnot(None))
-            case "before", "<":
-                query = query.filter(Document.doc_metadata[metadata_name] < value)
-            case "after", ">":
-                query = query.filter(Document.doc_metadata[metadata_name] > value)
-            case "≤", ">=":
-                query = query.filter(Document.doc_metadata[metadata_name] <= value)
-            case "≥", ">=":
-                query = query.filter(Document.doc_metadata[metadata_name] >= value)
-            case _:
-                raise InvalidConditionError(f"Invalid condition: {condition}")
-
     @classmethod
     def _extract_variable_selector_to_variable_mapping(
         cls,
@@ -495,94 +344,3 @@ class KnowledgeRetrievalNode(LLMNode):
             parameters=completion_params,
             stop=stop,
         )
-
-    def _calculate_rest_token(
-        self,
-        node_data: KnowledgeRetrievalNodeData,
-        query: str,
-        model_config: ModelConfigWithCredentialsEntity,
-        context: Optional[str],
-    ) -> int:
-        prompt_transform = AdvancedPromptTransform(with_variable_tmpl=True)
-        prompt_template = self._get_prompt_template(node_data, query, None, 2000)
-        prompt_messages = prompt_transform.get_prompt(
-            prompt_template=prompt_template,
-            inputs={},
-            query="",
-            files=[],
-            context=context,
-            memory_config=node_data.memory,
-            memory=None,
-            model_config=model_config,
-        )
-        rest_tokens = 2000
-
-        model_context_tokens = model_config.model_schema.model_properties.get(ModelPropertyKey.CONTEXT_SIZE)
-        if model_context_tokens:
-            model_instance = ModelInstance(
-                provider_model_bundle=model_config.provider_model_bundle, model=model_config.model
-            )
-
-            curr_message_tokens = model_instance.get_llm_num_tokens(prompt_messages)
-
-            max_tokens = 0
-            for parameter_rule in model_config.model_schema.parameter_rules:
-                if parameter_rule.name == "max_tokens" or (
-                    parameter_rule.use_template and parameter_rule.use_template == "max_tokens"
-                ):
-                    max_tokens = (
-                        model_config.parameters.get(parameter_rule.name)
-                        or model_config.parameters.get(parameter_rule.use_template or "")
-                    ) or 0
-
-            rest_tokens = model_context_tokens - max_tokens - curr_message_tokens
-            rest_tokens = max(rest_tokens, 0)
-
-        return rest_tokens
-
-    def _get_prompt_template(self, node_data: KnowledgeRetrievalNodeData, metadata_fields: list, query: str):
-        model_mode = ModelMode.value_of(node_data.metadata_model_config.mode)
-        input_text = query
-        memory_str = ""
-
-        prompt_messages: list[LLMNodeChatModelMessage] = []
-        if model_mode == ModelMode.CHAT:
-            system_prompt_messages = LLMNodeChatModelMessage(
-                role=PromptMessageRole.SYSTEM, text=METADATA_FILTER_SYSTEM_PROMPT
-            )
-            prompt_messages.append(system_prompt_messages)
-            user_prompt_message_1 = LLMNodeChatModelMessage(
-                role=PromptMessageRole.USER, text=METADATA_FILTER_USER_PROMPT_1
-            )
-            prompt_messages.append(user_prompt_message_1)
-            assistant_prompt_message_1 = LLMNodeChatModelMessage(
-                role=PromptMessageRole.ASSISTANT, text=METADATA_FILTER_ASSISTANT_PROMPT_1
-            )
-            prompt_messages.append(assistant_prompt_message_1)
-            user_prompt_message_2 = LLMNodeChatModelMessage(
-                role=PromptMessageRole.USER, text=QUESTION_CLASSIFIER_USER_PROMPT_2
-            )
-            prompt_messages.append(user_prompt_message_2)
-            assistant_prompt_message_2 = LLMNodeChatModelMessage(
-                role=PromptMessageRole.ASSISTANT, text=METADATA_FILTER_ASSISTANT_PROMPT_2
-            )
-            prompt_messages.append(assistant_prompt_message_2)
-            user_prompt_message_3 = LLMNodeChatModelMessage(
-                role=PromptMessageRole.USER,
-                text=METADATA_FILTER_USER_PROMPT_3.format(
-                    input_text=input_text,
-                    metadata_fields=json.dumps(metadata_fields, ensure_ascii=False),
-                ),
-            )
-            prompt_messages.append(user_prompt_message_3)
-            return prompt_messages
-        elif model_mode == ModelMode.COMPLETION:
-            return LLMNodeCompletionModelPromptTemplate(
-                text=METADATA_FILTER_COMPLETION_PROMPT.format(
-                    input_text=input_text,
-                    metadata_fields=json.dumps(metadata_fields, ensure_ascii=False),
-                )
-            )
-
-        else:
-            raise InvalidModelTypeError(f"Model mode {model_mode} not support.")

api/core/workflow/nodes/llm/node.py

@@ -191,6 +191,22 @@ class LLMNode(BaseNode[LLMNodeData]):
                     # deduct quota
                     self.deduct_llm_quota(tenant_id=self.tenant_id, model_instance=model_instance, usage=usage)
                     break
+            outputs = {"text": result_text, "usage": jsonable_encoder(usage), "finish_reason": finish_reason}
+
+            yield RunCompletedEvent(
+                run_result=NodeRunResult(
+                    status=WorkflowNodeExecutionStatus.SUCCEEDED,
+                    inputs=node_inputs,
+                    process_data=process_data,
+                    outputs=outputs,
+                    metadata={
+                        NodeRunMetadataKey.TOTAL_TOKENS: usage.total_tokens,
+                        NodeRunMetadataKey.TOTAL_PRICE: usage.total_price,
+                        NodeRunMetadataKey.CURRENCY: usage.currency,
+                    },
+                    llm_usage=usage,
+                )
+            )
         except LLMNodeError as e:
             yield RunCompletedEvent(
                 run_result=NodeRunResult(
@@ -211,23 +227,6 @@ class LLMNode(BaseNode[LLMNodeData]):
                 )
             )
 
-        outputs = {"text": result_text, "usage": jsonable_encoder(usage), "finish_reason": finish_reason}
-
-        yield RunCompletedEvent(
-            run_result=NodeRunResult(
-                status=WorkflowNodeExecutionStatus.SUCCEEDED,
-                inputs=node_inputs,
-                process_data=process_data,
-                outputs=outputs,
-                metadata={
-                    NodeRunMetadataKey.TOTAL_TOKENS: usage.total_tokens,
-                    NodeRunMetadataKey.TOTAL_PRICE: usage.total_price,
-                    NodeRunMetadataKey.CURRENCY: usage.currency,
-                },
-                llm_usage=usage,
-            )
-        )
-
     def _invoke_llm(
         self,
         node_data_model: ModelConfig,
@@ -459,6 +458,7 @@ class LLMNode(BaseNode[LLMNodeData]):
                 "index_node_hash": metadata.get("segment_index_node_hash"),
                 "content": context_dict.get("content"),
                 "page": metadata.get("page"),
+                "doc_metadata": metadata.get("doc_metadata"),
             }
 
             return source

api/fields/dataset_fields.py

@@ -53,8 +53,6 @@ external_knowledge_info_fields = {
     "external_knowledge_api_endpoint": fields.String,
 }
 
-doc_metadata_fields = {"id": fields.String, "name": fields.String, "type": fields.String}
-
 dataset_detail_fields = {
     "id": fields.String,
     "name": fields.String,
@@ -78,8 +76,6 @@ dataset_detail_fields = {
     "doc_form": fields.String,
     "external_knowledge_info": fields.Nested(external_knowledge_info_fields),
     "external_retrieval_model": fields.Nested(external_retrieval_model_fields, allow_null=True),
-    "doc_metadata": fields.List(fields.Nested(doc_metadata_fields)),
-    "built_in_field_enabled": fields.Boolean,
 }
 
 dataset_query_detail_fields = {
@@ -91,9 +87,3 @@ dataset_query_detail_fields = {
     "created_by": fields.String,
     "created_at": TimestampField,
 }
-
-dataset_metadata_fields = {
-    "id": fields.String,
-    "type": fields.String,
-    "name": fields.String,
-}

api/fields/document_fields.py

@@ -3,13 +3,6 @@ from flask_restful import fields  # type: ignore
 from fields.dataset_fields import dataset_fields
 from libs.helper import TimestampField
 
-document_metadata_fields = {
-    "id": fields.String,
-    "name": fields.String,
-    "type": fields.String,
-    "value": fields.String,
-}
-
 document_fields = {
     "id": fields.String,
     "position": fields.Integer,
@@ -32,7 +25,6 @@ document_fields = {
     "word_count": fields.Integer,
     "hit_count": fields.Integer,
     "doc_form": fields.String,
-    "doc_metadata": fields.List(fields.Nested(document_metadata_fields), attribute="doc_metadata_details"),
 }
 
 document_with_segments_fields = {
@@ -59,7 +51,6 @@ document_with_segments_fields = {
     "hit_count": fields.Integer,
     "completed_segments": fields.Integer,
     "total_segments": fields.Integer,
-    "doc_metadata": fields.List(fields.Nested(document_metadata_fields), attribute="doc_metadata_details"),
 }
 
 dataset_and_document_fields = {

api/fields/hit_testing_fields.py

@@ -7,6 +7,7 @@ document_fields = {
     "data_source_type": fields.String,
     "name": fields.String,
     "doc_type": fields.String,
+    "doc_metadata": fields.Raw,
 }
 
 segment_fields = {

api/migrations/README

@@ -1 +1,2 @@
 Single-database configuration for Flask.
+

api/migrations/versions/2024_12_20_0628-e1944c35e15e_add_retry_index_field_to_node_execution_.py

@@ -1,4 +1,5 @@
 """add retry_index field to node-execution model
+
 Revision ID: e1944c35e15e
 Revises: 11b07f66c737
 Create Date: 2024-12-20 06:28:30.287197

api/migrations/versions/2025_02_27_0917-d20049ed0af6_add_metadata_function.py

@@ -1,90 +0,0 @@
-"""add_metadata_function
-
-Revision ID: d20049ed0af6
-Revises: 08ec4f75af5e
-Create Date: 2025-02-27 09:17:48.903213
-
-"""
-from alembic import op
-import models as models
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision = 'd20049ed0af6'
-down_revision = '08ec4f75af5e'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('dataset_metadata_bindings',
-    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
-    sa.Column('tenant_id', models.types.StringUUID(), nullable=False),
-    sa.Column('dataset_id', models.types.StringUUID(), nullable=False),
-    sa.Column('metadata_id', models.types.StringUUID(), nullable=False),
-    sa.Column('document_id', models.types.StringUUID(), nullable=False),
-    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
-    sa.Column('created_by', models.types.StringUUID(), nullable=False),
-    sa.PrimaryKeyConstraint('id', name='dataset_metadata_binding_pkey')
-    )
-    with op.batch_alter_table('dataset_metadata_bindings', schema=None) as batch_op:
-        batch_op.create_index('dataset_metadata_binding_dataset_idx', ['dataset_id'], unique=False)
-        batch_op.create_index('dataset_metadata_binding_document_idx', ['document_id'], unique=False)
-        batch_op.create_index('dataset_metadata_binding_metadata_idx', ['metadata_id'], unique=False)
-        batch_op.create_index('dataset_metadata_binding_tenant_idx', ['tenant_id'], unique=False)
-
-    op.create_table('dataset_metadatas',
-    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
-    sa.Column('tenant_id', models.types.StringUUID(), nullable=False),
-    sa.Column('dataset_id', models.types.StringUUID(), nullable=False),
-    sa.Column('type', sa.String(length=255), nullable=False),
-    sa.Column('name', sa.String(length=255), nullable=False),
-    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False),
-    sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False),
-    sa.Column('created_by', models.types.StringUUID(), nullable=False),
-    sa.Column('updated_by', models.types.StringUUID(), nullable=True),
-    sa.PrimaryKeyConstraint('id', name='dataset_metadata_pkey')
-    )
-    with op.batch_alter_table('dataset_metadatas', schema=None) as batch_op:
-        batch_op.create_index('dataset_metadata_dataset_idx', ['dataset_id'], unique=False)
-        batch_op.create_index('dataset_metadata_tenant_idx', ['tenant_id'], unique=False)
-
-    with op.batch_alter_table('datasets', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('built_in_field_enabled', sa.Boolean(), server_default=sa.text('false'), nullable=False))
-
-    with op.batch_alter_table('documents', schema=None) as batch_op:
-        batch_op.alter_column('doc_metadata',
-               existing_type=postgresql.JSON(astext_type=sa.Text()),
-               type_=postgresql.JSONB(astext_type=sa.Text()),
-               existing_nullable=True)
-        batch_op.create_index('document_metadata_idx', ['doc_metadata'], unique=False, postgresql_using='gin')
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('documents', schema=None) as batch_op:
-        batch_op.drop_index('document_metadata_idx', postgresql_using='gin')
-        batch_op.alter_column('doc_metadata',
-               existing_type=postgresql.JSONB(astext_type=sa.Text()),
-               type_=postgresql.JSON(astext_type=sa.Text()),
-               existing_nullable=True)
-
-    with op.batch_alter_table('datasets', schema=None) as batch_op:
-        batch_op.drop_column('built_in_field_enabled')
-
-    with op.batch_alter_table('dataset_metadatas', schema=None) as batch_op:
-        batch_op.drop_index('dataset_metadata_tenant_idx')
-        batch_op.drop_index('dataset_metadata_dataset_idx')
-
-    op.drop_table('dataset_metadatas')
-    with op.batch_alter_table('dataset_metadata_bindings', schema=None) as batch_op:
-        batch_op.drop_index('dataset_metadata_binding_tenant_idx')
-        batch_op.drop_index('dataset_metadata_binding_metadata_idx')
-        batch_op.drop_index('dataset_metadata_binding_document_idx')
-        batch_op.drop_index('dataset_metadata_binding_dataset_idx')
-
-    op.drop_table('dataset_metadata_bindings')
-    # ### end Alembic commands ###

api/migrations/versions/2025_03_03_0304-4413929e1ec2_extend_provider_name_column.py

@@ -0,0 +1,39 @@
+"""extend_provider_name_column
+
+Revision ID: 4413929e1ec2
+Revises: 08ec4f75af5e
+Create Date: 2025-03-03 03:04:58.181493
+
+"""
+from alembic import op
+import models as models
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '4413929e1ec2'
+down_revision = '08ec4f75af5e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('dataset_collection_bindings', schema=None) as batch_op:
+        batch_op.alter_column('provider_name',
+               existing_type=sa.VARCHAR(length=40),
+               type_=sa.String(length=255),
+               existing_nullable=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('dataset_collection_bindings', schema=None) as batch_op:
+        batch_op.alter_column('provider_name',
+               existing_type=sa.String(length=255),
+               type_=sa.VARCHAR(length=40),
+               existing_nullable=False)
+
+    # ### end Alembic commands ###

api/models/dataset.py

@@ -16,7 +16,6 @@ from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import Mapped
 
 from configs import dify_config
-from core.rag.index_processor.constant.built_in_field import BuiltInField
 from core.rag.retrieval.retrieval_methods import RetrievalMethod
 from extensions.ext_storage import storage
 from services.entities.knowledge_entities.knowledge_entities import ParentMode, Rule
@@ -61,7 +60,6 @@ class Dataset(db.Model):  # type: ignore[name-defined]
     embedding_model_provider = db.Column(db.String(255), nullable=True)
     collection_binding_id = db.Column(StringUUID, nullable=True)
     retrieval_model = db.Column(JSONB, nullable=True)
-    built_in_field_enabled = db.Column(db.Boolean, nullable=False, server_default=db.text("false"))
 
     @property
     def dataset_keyword_table(self):
@@ -199,19 +197,6 @@ class Dataset(db.Model):  # type: ignore[name-defined]
             "external_knowledge_api_endpoint": json.loads(external_knowledge_api.settings).get("endpoint", ""),
         }
 
-    @property
-    def doc_metadata(self):
-        dataset_metadatas = db.session.query(DatasetMetadata).filter(DatasetMetadata.dataset_id == self.id).all()
-
-        return [
-            {
-                "id": dataset_metadata.id,
-                "name": dataset_metadata.name,
-                "type": dataset_metadata.type,
-            }
-            for dataset_metadata in dataset_metadatas
-        ]
-
     @staticmethod
     def gen_collection_name_by_id(dataset_id: str) -> str:
         normalized_dataset_id = dataset_id.replace("-", "_")
@@ -265,7 +250,6 @@ class Document(db.Model):  # type: ignore[name-defined]
         db.Index("document_dataset_id_idx", "dataset_id"),
         db.Index("document_is_paused_idx", "is_paused"),
         db.Index("document_tenant_idx", "tenant_id"),
-        db.Index("document_metadata_idx", "doc_metadata", postgresql_using="gin"),
     )
 
     # initial fields
@@ -322,7 +306,7 @@ class Document(db.Model):  # type: ignore[name-defined]
     archived_at = db.Column(db.DateTime, nullable=True)
     updated_at = db.Column(db.DateTime, nullable=False, server_default=func.current_timestamp())
     doc_type = db.Column(db.String(40), nullable=True)
-    doc_metadata = db.Column(JSONB, nullable=True)
+    doc_metadata = db.Column(db.JSON, nullable=True)
     doc_form = db.Column(db.String(255), nullable=False, server_default=db.text("'text_model'::character varying"))
     doc_language = db.Column(db.String(255), nullable=True)
 
@@ -413,78 +397,6 @@ class Document(db.Model):  # type: ignore[name-defined]
         )
 
     @property
-    def uploader(self):
-        user = db.session.query(Account).filter(Account.id == self.created_by).first()
-        return user.name if user else None
-
-    @property
-    def upload_date(self):
-        return self.created_at
-
-    @property
-    def last_update_date(self):
-        return self.updated_at
-
-    @property
-    def doc_metadata_details(self):
-        if self.doc_metadata:
-            document_metadatas = (
-                db.session.query(DatasetMetadata)
-                .join(DatasetMetadataBinding, DatasetMetadataBinding.metadata_id == DatasetMetadata.id)
-                .filter(
-                    DatasetMetadataBinding.dataset_id == self.dataset_id, DatasetMetadataBinding.document_id == self.id
-                )
-                .all()
-            )
-            metadata_list = []
-            for metadata in document_metadatas:
-                metadata_dict = {
-                    "id": metadata.id,
-                    "name": metadata.name,
-                    "type": metadata.type,
-                    "value": self.doc_metadata.get(metadata.type),
-                }
-                metadata_list.append(metadata_dict)
-            # deal built-in fields
-            metadata_list.extend(self.get_built_in_fields())
-
-            return metadata_list
-        return None
-    
-    def get_built_in_fields(self):
-        built_in_fields = []
-        built_in_fields.append({
-                    "id": "built-in",
-                    "name": BuiltInField.document_name,
-                    "type": "string",
-                    "value": self.name,
-                })
-        built_in_fields.append({
-                    "id": "built-in",
-                    "name": BuiltInField.uploader,
-                    "type": "string",
-                    "value": self.uploader,
-                })
-        built_in_fields.append({
-                    "id": "built-in",
-                    "name": BuiltInField.upload_date,
-                    "type": "date",
-                    "value": self.created_at,
-                })
-        built_in_fields.append({
-                    "id": "built-in",
-                    "name": BuiltInField.last_update_date,
-                    "type": "date",
-                    "value": self.updated_at,
-                })
-        built_in_fields.append({
-                    "id": "built-in",
-                    "name": BuiltInField.source,
-                    "type": "string",
-                    "value": self.data_source_info,
-                })
-        return built_in_fields
-
     def process_rule_dict(self):
         if self.dataset_process_rule_id:
             return self.dataset_process_rule.to_dict()
@@ -873,7 +785,7 @@ class DatasetCollectionBinding(db.Model):  # type: ignore[name-defined]
     )
 
     id = db.Column(StringUUID, primary_key=True, server_default=db.text("uuid_generate_v4()"))
-    provider_name = db.Column(db.String(40), nullable=False)
+    provider_name = db.Column(db.String(255), nullable=False)
     model_name = db.Column(db.String(255), nullable=False)
     type = db.Column(db.String(40), server_default=db.text("'dataset'::character varying"), nullable=False)
     collection_name = db.Column(db.String(64), nullable=False)
@@ -1018,41 +930,3 @@ class DatasetAutoDisableLog(db.Model):  # type: ignore[name-defined]
     document_id = db.Column(StringUUID, nullable=False)
     notified = db.Column(db.Boolean, nullable=False, server_default=db.text("false"))
     created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))
-
-
-class DatasetMetadata(db.Model):  # type: ignore[name-defined]
-    __tablename__ = "dataset_metadatas"
-    __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="dataset_metadata_pkey"),
-        db.Index("dataset_metadata_tenant_idx", "tenant_id"),
-        db.Index("dataset_metadata_dataset_idx", "dataset_id"),
-    )
-
-    id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()"))
-    tenant_id = db.Column(StringUUID, nullable=False)
-    dataset_id = db.Column(StringUUID, nullable=False)
-    type = db.Column(db.String(255), nullable=False)
-    name = db.Column(db.String(255), nullable=False)
-    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))
-    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))
-    created_by = db.Column(StringUUID, nullable=False)
-    updated_by = db.Column(StringUUID, nullable=True)
-
-
-class DatasetMetadataBinding(db.Model):  # type: ignore[name-defined]
-    __tablename__ = "dataset_metadata_bindings"
-    __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="dataset_metadata_binding_pkey"),
-        db.Index("dataset_metadata_binding_tenant_idx", "tenant_id"),
-        db.Index("dataset_metadata_binding_dataset_idx", "dataset_id"),
-        db.Index("dataset_metadata_binding_metadata_idx", "metadata_id"),
-        db.Index("dataset_metadata_binding_document_idx", "document_id"),
-    )
-
-    id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()"))
-    tenant_id = db.Column(StringUUID, nullable=False)
-    dataset_id = db.Column(StringUUID, nullable=False)
-    metadata_id = db.Column(StringUUID, nullable=False)
-    document_id = db.Column(StringUUID, nullable=False)
-    created_at = db.Column(db.DateTime, nullable=False, server_default=func.current_timestamp())
-    created_by = db.Column(StringUUID, nullable=False)

api/models/workflow.py

@@ -180,7 +180,7 @@ class Workflow(Base):
             features["file_upload"]["enabled"] = image_enabled
             features["file_upload"]["number_limits"] = image_number_limits
             features["file_upload"]["allowed_file_upload_methods"] = image_transfer_methods
-            features["file_upload"]["allowed_file_types"] = ["image"]
+            features["file_upload"]["allowed_file_types"] = features["file_upload"].get("allowed_file_types", ["image"])
             features["file_upload"]["allowed_file_extensions"] = []
             del features["file_upload"]["image"]
             self._features = json.dumps(features)
@@ -256,11 +256,10 @@ class Workflow(Base):
 
         # decrypt secret variables value
         def decrypt_func(var):
-            return (
-                var.model_copy(update={"value": encrypter.decrypt_token(tenant_id=tenant_id, token=var.value)})
-                if isinstance(var, SecretVariable)
-                else var
-            )
+            if isinstance(var, SecretVariable):
+                return var.model_copy(update={"value": encrypter.decrypt_token(tenant_id=tenant_id, token=var.value)})
+            else:
+                return var
 
         results = list(map(decrypt_func, results))
         return results
@@ -286,11 +285,10 @@ class Workflow(Base):
 
         # encrypt secret variables value
         def encrypt_func(var):
-            return (
-                var.model_copy(update={"value": encrypter.encrypt_token(tenant_id=tenant_id, token=var.value)})
-                if isinstance(var, SecretVariable)
-                else var
-            )
+            if isinstance(var, SecretVariable):
+                return var.model_copy(update={"value": encrypter.encrypt_token(tenant_id=tenant_id, token=var.value)})
+            else:
+                return var
 
         encrypted_vars = list(map(encrypt_func, value))
         environment_variables_json = json.dumps(

api/pyproject.toml

@@ -50,7 +50,7 @@ oci = "~2.135.1"
 openai = "~1.61.0"
 openpyxl = "~3.1.5"
 opik = "~1.3.4"
-pandas = { version = "~2.2.2", extras = ["performance", "excel"] }
+pandas = { version = "~2.2.2", extras = ["performance", "excel", "output-formatting"] }
 pandas-stubs = "~2.2.3.241009"
 psycogreen = "~1.0.2"
 psycopg2-binary = "~2.9.6"
@@ -174,4 +174,4 @@ types-tqdm = "~4.67.0.20241221"
 optional = true
 [tool.poetry.group.lint.dependencies]
 dotenv-linter = "~0.5.0"
-ruff = "~0.9.2"
+ruff = "~0.9.9"

api/services/dataset_service.py

@@ -16,7 +16,6 @@ from configs import dify_config
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
-from core.rag.index_processor.constant.built_in_field import BuiltInField
 from core.plugin.entities.plugin import ModelProviderID
 from core.rag.index_processor.constant.index_type import IndexType
 from core.rag.retrieval.retrieval_methods import RetrievalMethod
@@ -600,45 +599,9 @@ class DocumentService:
 
         return document
 
-    @staticmethod
-    def get_document_by_ids(document_ids: list[str]) -> list[Document]:
-        documents = (
-            db.session.query(Document)
-            .filter(
-                Document.id.in_(document_ids),
-                Document.enabled == True,
-                Document.indexing_status == "completed",
-                Document.archived == False,
-            )
-            .all()
-        )
-        return documents
-
     @staticmethod
     def get_document_by_dataset_id(dataset_id: str) -> list[Document]:
-        documents = (
-            db.session.query(Document)
-            .filter(
-                Document.dataset_id == dataset_id,
-                Document.enabled == True,
-            )
-            .all()
-        )
-
-        return documents
-
-    @staticmethod
-    def get_working_documents_by_dataset_id(dataset_id: str) -> list[Document]:
-        documents = (
-            db.session.query(Document)
-            .filter(
-                Document.dataset_id == dataset_id,
-                Document.enabled == True,
-                Document.indexing_status == "completed",
-                Document.archived == False,
-            )
-            .all()
-        )
+        documents = db.session.query(Document).filter(Document.dataset_id == dataset_id, Document.enabled == True).all()
 
         return documents
 
@@ -721,11 +684,7 @@ class DocumentService:
         if document.tenant_id != current_user.current_tenant_id:
             raise ValueError("No permission.")
 
-        if dataset.built_in_field_enabled:
-            if document.doc_metadata:
-                document.doc_metadata[BuiltInField.document_name] = name
-        else:
-            document.name = name
+        document.name = name
 
         db.session.add(document)
         db.session.commit()
@@ -1125,22 +1084,9 @@ class DocumentService:
             doc_form=document_form,
             doc_language=document_language,
         )
-        doc_metadata = {}
-        if dataset.built_in_field_enabled:
-            doc_metadata = {
-                BuiltInField.document_name: name,
-                BuiltInField.uploader: account.name,
-                BuiltInField.upload_date: datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S"),
-                BuiltInField.last_update_date: datetime.datetime.now(datetime.timezone.utc).strftime(
-                    "%Y-%m-%d %H:%M:%S"
-                ),
-                BuiltInField.source: data_source_type,
-            }
         if metadata is not None:
-            doc_metadata.update(metadata.doc_metadata)
+            document.doc_metadata = metadata.doc_metadata
             document.doc_type = metadata.doc_type
-        if doc_metadata:
-            document.doc_metadata = doc_metadata
         return document
 
     @staticmethod

api/services/entities/knowledge_entities/knowledge_entities.py

@@ -124,36 +124,3 @@ class SegmentUpdateArgs(BaseModel):
 class ChildChunkUpdateArgs(BaseModel):
     id: Optional[str] = None
     content: str
-
-
-class MetadataArgs(BaseModel):
-    type: Literal["string", "number", "time"]
-    name: str
-
-
-class MetadataUpdateArgs(BaseModel):
-    name: str
-    value: str
-
-
-class MetadataValueUpdateArgs(BaseModel):
-    fields: list[MetadataUpdateArgs]
-
-
-class MetadataDetail(BaseModel):
-    id: str
-    name: str
-    value: str
-
-
-class DocumentMetadataOperation(BaseModel):
-    document_id: str
-    metadata_list: list[MetadataDetail]
-
-
-class MetadataOperationData(BaseModel):
-    """
-    Metadata operation data
-    """
-
-    operation_data: list[DocumentMetadataOperation]

api/services/metadata_service.py

@@ -1,182 +0,0 @@
-import datetime
-from typing import Optional
-
-from flask_login import current_user  # type: ignore
-
-from core.rag.index_processor.constant.built_in_field import BuiltInField
-from extensions.ext_database import db
-from extensions.ext_redis import redis_client
-from models.dataset import Dataset, DatasetMetadata, DatasetMetadataBinding
-from services.dataset_service import DocumentService
-from services.entities.knowledge_entities.knowledge_entities import (
-    MetadataArgs,
-    MetadataOperationData,
-)
-from tasks.update_documents_metadata_task import update_documents_metadata_task
-
-
-class MetadataService:
-    @staticmethod
-    def create_metadata(dataset_id: str, metadata_args: MetadataArgs) -> DatasetMetadata:
-        metadata = DatasetMetadata(
-            dataset_id=dataset_id,
-            type=metadata_args.type,
-            name=metadata_args.name,
-            created_by=current_user.id,
-        )
-        db.session.add(metadata)
-        db.session.commit()
-        return metadata
-
-    @staticmethod
-    def update_metadata_name(dataset_id: str, metadata_id: str, name: str) -> DatasetMetadata:
-        lock_key = f"dataset_metadata_lock_{dataset_id}"
-        MetadataService.knowledge_base_metadata_lock_check(dataset_id, None)
-        metadata = DatasetMetadata.query.filter_by(id=metadata_id).first()
-        if metadata is None:
-            raise ValueError("Metadata not found.")
-        old_name = metadata.name
-        metadata.name = name
-        metadata.updated_by = current_user.id
-        metadata.updated_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
-
-        # update related documents
-        documents = []
-        dataset_metadata_bindings = DatasetMetadataBinding.query.filter_by(metadata_id=metadata_id).all()
-        if dataset_metadata_bindings:
-            document_ids = [binding.document_id for binding in dataset_metadata_bindings]
-            documents = DocumentService.get_document_by_ids(document_ids)
-            for document in documents:
-                document.doc_metadata[name] = document.doc_metadata.pop(old_name)
-                db.session.add(document)
-        db.session.commit()
-        if document_ids:
-            update_documents_metadata_task.delay(dataset_id, document_ids, lock_key)
-        return metadata
-
-    @staticmethod
-    def delete_metadata(dataset_id: str, metadata_id: str):
-        lock_key = f"dataset_metadata_lock_{dataset_id}"
-        MetadataService.knowledge_base_metadata_lock_check(dataset_id, None)
-        metadata = DatasetMetadata.query.filter_by(id=metadata_id).first()
-        if metadata is None:
-            raise ValueError("Metadata not found.")
-        db.session.delete(metadata)
-
-        # delete related documents
-        dataset_metadata_bindings = DatasetMetadataBinding.query.filter_by(metadata_id=metadata_id).all()
-        if dataset_metadata_bindings:
-            document_ids = [binding.document_id for binding in dataset_metadata_bindings]
-            documents = DocumentService.get_document_by_ids(document_ids)
-            for document in documents:
-                document.doc_metadata.pop(metadata.name)
-                db.session.add(document)
-        db.session.commit()
-        if document_ids:
-            update_documents_metadata_task.delay(dataset_id, document_ids, lock_key)
-
-    @staticmethod
-    def get_built_in_fields():
-        return [
-            {"name": BuiltInField.document_name, "type": "string"},
-            {"name": BuiltInField.uploader, "type": "string"},
-            {"name": BuiltInField.upload_date, "type": "date"},
-            {"name": BuiltInField.last_update_date, "type": "date"},
-            {"name": BuiltInField.source, "type": "string"},
-        ]
-
-    @staticmethod
-    def enable_built_in_field(dataset: Dataset):
-        if dataset.built_in_fields:
-            return
-        lock_key = f"dataset_metadata_lock_{dataset.id}"
-        MetadataService.knowledge_base_metadata_lock_check(dataset.id, None)
-        dataset.built_in_fields = True
-        db.session.add(dataset)
-        documents = DocumentService.get_working_documents_by_dataset_id(dataset.id)
-        document_ids = []
-        if documents:
-            for document in documents:
-                document.doc_metadata[BuiltInField.document_name] = document.name
-                document.doc_metadata[BuiltInField.uploader] = document.uploader
-                document.doc_metadata[BuiltInField.upload_date] = document.upload_date.strftime("%Y-%m-%d %H:%M:%S")
-                document.doc_metadata[BuiltInField.last_update_date] = document.last_update_date.strftime(
-                    "%Y-%m-%d %H:%M:%S"
-                )
-                document.doc_metadata[BuiltInField.source] = document.data_source_type
-                db.session.add(document)
-                document_ids.append(document.id)
-        db.session.commit()
-        if document_ids:
-            update_documents_metadata_task.delay(dataset.id, document_ids, lock_key)
-
-    @staticmethod
-    def disable_built_in_field(dataset: Dataset):
-        if not dataset.built_in_fields:
-            return
-        lock_key = f"dataset_metadata_lock_{dataset.id}"
-        MetadataService.knowledge_base_metadata_lock_check(dataset.id, None)
-        dataset.built_in_fields = False
-        db.session.add(dataset)
-        documents = DocumentService.get_working_documents_by_dataset_id(dataset.id)
-        document_ids = []
-        if documents:
-            for document in documents:
-                document.doc_metadata.pop(BuiltInField.document_name)
-                document.doc_metadata.pop(BuiltInField.uploader)
-                document.doc_metadata.pop(BuiltInField.upload_date)
-                document.doc_metadata.pop(BuiltInField.last_update_date)
-                document.doc_metadata.pop(BuiltInField.source)
-                db.session.add(document)
-                document_ids.append(document.id)
-        db.session.commit()
-        if document_ids:
-            update_documents_metadata_task.delay(dataset.id, document_ids, lock_key)
-
-    @staticmethod
-    def update_documents_metadata(dataset: Dataset, metadata_args: MetadataOperationData):
-        for operation in metadata_args.operation_data:
-            lock_key = f"document_metadata_lock_{operation.document_id}"
-            MetadataService.knowledge_base_metadata_lock_check(None, operation.document_id)
-            document = DocumentService.get_document(operation.document_id)
-            if document is None:
-                raise ValueError("Document not found.")
-            document.doc_metadata = {}
-            for metadata_value in metadata_args.fields:
-                document.doc_metadata[metadata_value.name] = metadata_value.value
-            if dataset.built_in_fields:
-                document.doc_metadata[BuiltInField.document_name] = document.name
-                document.doc_metadata[BuiltInField.uploader] = document.uploader
-                document.doc_metadata[BuiltInField.upload_date] = document.upload_date.strftime("%Y-%m-%d %H:%M:%S")
-                document.doc_metadata[BuiltInField.last_update_date] = document.last_update_date.strftime(
-                    "%Y-%m-%d %H:%M:%S"
-                )
-                document.doc_metadata[BuiltInField.source] = document.data_source_type
-            # deal metadata bindding
-            DatasetMetadataBinding.query.filter_by(document_id=operation.document_id).delete()
-            for metadata_value in operation.metadata_list:
-                dataset_metadata_binding = DatasetMetadataBinding(
-                    tenant_id=current_user.tenant_id,
-                    dataset_id=dataset.id,
-                    document_id=operation.document_id,
-                    metadata_id=metadata_value.id,
-                    created_by=current_user.id,
-                )
-                db.session.add(dataset_metadata_binding)
-            db.session.add(document)
-            db.session.commit()
-
-        update_documents_metadata_task.delay(dataset.id, [document.id], lock_key)
-
-    @staticmethod
-    def knowledge_base_metadata_lock_check(dataset_id: Optional[str], document_id: Optional[str]):
-        if dataset_id:
-            lock_key = f"dataset_metadata_lock_{dataset_id}"
-            if redis_client.get(lock_key):
-                raise ValueError("Another knowledge base metadata operation is running, please wait a moment.")
-            redis_client.set(lock_key, 1, ex=3600)
-        if document_id:
-            lock_key = f"document_metadata_lock_{document_id}"
-            if redis_client.get(lock_key):
-                raise ValueError("Another document metadata operation is running, please wait a moment.")
-            redis_client.set(lock_key, 1, ex=3600)

api/services/plugin/plugin_service.py

@@ -1,6 +1,9 @@
 import logging
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from mimetypes import guess_type
+from typing import Optional
+
+from pydantic import BaseModel
 
 from configs import dify_config
 from core.helper import marketplace
@@ -18,11 +21,71 @@ from core.plugin.entities.plugin_daemon import PluginInstallTask, PluginUploadRe
 from core.plugin.manager.asset import PluginAssetManager
 from core.plugin.manager.debugging import PluginDebuggingManager
 from core.plugin.manager.plugin import PluginInstallationManager
+from extensions.ext_redis import redis_client
 
 logger = logging.getLogger(__name__)
 
 
 class PluginService:
+    class LatestPluginCache(BaseModel):
+        plugin_id: str
+        version: str
+        unique_identifier: str
+
+    REDIS_KEY_PREFIX = "plugin_service:latest_plugin:"
+    REDIS_TTL = 60 * 5  # 5 minutes
+
+    @staticmethod
+    def fetch_latest_plugin_version(plugin_ids: Sequence[str]) -> Mapping[str, Optional[LatestPluginCache]]:
+        """
+        Fetch the latest plugin version
+        """
+        result: dict[str, Optional[PluginService.LatestPluginCache]] = {}
+
+        try:
+            cache_not_exists = []
+
+            # Try to get from Redis first
+            for plugin_id in plugin_ids:
+                cached_data = redis_client.get(f"{PluginService.REDIS_KEY_PREFIX}{plugin_id}")
+                if cached_data:
+                    result[plugin_id] = PluginService.LatestPluginCache.model_validate_json(cached_data)
+                else:
+                    cache_not_exists.append(plugin_id)
+
+            if cache_not_exists:
+                manifests = {
+                    manifest.plugin_id: manifest
+                    for manifest in marketplace.batch_fetch_plugin_manifests(cache_not_exists)
+                }
+
+                for plugin_id, manifest in manifests.items():
+                    latest_plugin = PluginService.LatestPluginCache(
+                        plugin_id=plugin_id,
+                        version=manifest.latest_version,
+                        unique_identifier=manifest.latest_package_identifier,
+                    )
+
+                    # Store in Redis
+                    redis_client.setex(
+                        f"{PluginService.REDIS_KEY_PREFIX}{plugin_id}",
+                        PluginService.REDIS_TTL,
+                        latest_plugin.model_dump_json(),
+                    )
+
+                    result[plugin_id] = latest_plugin
+
+                    # pop plugin_id from cache_not_exists
+                    cache_not_exists.remove(plugin_id)
+
+                for plugin_id in cache_not_exists:
+                    result[plugin_id] = None
+
+            return result
+        except Exception:
+            logger.exception("failed to fetch latest plugin version")
+            return result
+
     @staticmethod
     def get_debugging_key(tenant_id: str) -> str:
         """
@@ -40,9 +103,7 @@ class PluginService:
         plugins = manager.list_plugins(tenant_id)
         plugin_ids = [plugin.plugin_id for plugin in plugins if plugin.source == PluginInstallationSource.Marketplace]
         try:
-            manifests = {
-                manifest.plugin_id: manifest for manifest in marketplace.batch_fetch_plugin_manifests(plugin_ids)
-            }
+            manifests = PluginService.fetch_latest_plugin_version(plugin_ids)
         except Exception:
             manifests = {}
             logger.exception("failed to fetch plugin manifests")
@@ -50,9 +111,11 @@ class PluginService:
         for plugin in plugins:
             if plugin.source == PluginInstallationSource.Marketplace:
                 if plugin.plugin_id in manifests:
-                    # set latest_version
-                    plugin.latest_version = manifests[plugin.plugin_id].latest_version
-                    plugin.latest_unique_identifier = manifests[plugin.plugin_id].latest_package_identifier
+                    latest_plugin_cache = manifests[plugin.plugin_id]
+                    if latest_plugin_cache:
+                        # set latest_version
+                        plugin.latest_version = latest_plugin_cache.version
+                        plugin.latest_unique_identifier = latest_plugin_cache.unique_identifier
 
         return plugins
 

api/tasks/update_documents_metadata_task.py

@@ -1,121 +0,0 @@
-import logging
-import time
-from typing import Optional
-
-import click
-from celery import shared_task  # type: ignoreq
-
-from core.rag.index_processor.constant.built_in_field import BuiltInField
-from core.rag.index_processor.constant.index_type import IndexType
-from core.rag.index_processor.index_processor_factory import IndexProcessorFactory
-from core.rag.models.document import ChildDocument, Document
-from extensions.ext_database import db
-from extensions.ext_redis import redis_client
-from models.dataset import (
-    Document as DatasetDocument,
-)
-from models.dataset import (
-    DocumentSegment,
-)
-from services.dataset_service import DatasetService
-
-
-@shared_task(queue="dataset")
-def update_documents_metadata_task(
-    dataset_id: str,
-    document_ids: list[str],
-    lock_key: Optional[str] = None,
-):
-    """
-    Update documents metadata.
-    :param dataset_id: dataset id
-    :param document_ids: document ids
-
-    Usage: update_documents_metadata_task.delay(dataset_id, document_ids)
-    """
-    logging.info(click.style("Start update documents metadata: {}".format(dataset_id), fg="green"))
-    start_at = time.perf_counter()
-
-    try:
-        dataset = DatasetService.get_dataset(dataset_id)
-        if dataset is None:
-            raise ValueError("Dataset not found.")
-        documents = (
-            db.session.query(DatasetDocument)
-            .filter(
-                DatasetDocument.dataset_id == dataset_id,
-                DatasetDocument.id.in_(document_ids),
-                DatasetDocument.enabled == True,
-                DatasetDocument.indexing_status == "completed",
-                DatasetDocument.archived == False,
-            )
-            .all()
-        )
-        if not documents:
-            raise ValueError("Documents not found.")
-        for dataset_document in documents:
-            index_processor = IndexProcessorFactory(dataset_document.doc_form).init_index_processor()
-
-            segments = (
-                db.session.query(DocumentSegment)
-                .filter(
-                    DocumentSegment.dataset_id == dataset_id,
-                    DocumentSegment.document_id == dataset_document.id,
-                    DocumentSegment.enabled == True,
-                )
-                .all()
-            )
-            if not segments:
-                continue
-            # delete all documents in vector index
-            index_node_ids = [segment.index_node_id for segment in segments]
-            index_processor.clean(dataset, index_node_ids, with_keywords=False, delete_child_chunks=True)
-            # update documents metadata
-            documents = []
-            for segment in segments:
-                document = Document(
-                    page_content=segment.content,
-                    metadata={
-                        "doc_id": segment.index_node_id,
-                        "doc_hash": segment.index_node_hash,
-                        "document_id": dataset_document.id,
-                        "dataset_id": dataset_id,
-                    },
-                )
-
-                if dataset_document.doc_form == IndexType.PARENT_CHILD_INDEX:
-                    child_chunks = segment.child_chunks
-                    if child_chunks:
-                        child_documents = []
-                        for child_chunk in child_chunks:
-                            child_document = ChildDocument(
-                                page_content=child_chunk.content,
-                                metadata={
-                                    "doc_id": child_chunk.index_node_id,
-                                    "doc_hash": child_chunk.index_node_hash,
-                                    "document_id": dataset_document.id,
-                                    "dataset_id": dataset_id,
-                                },
-                            )
-                            if dataset.built_in_field_enabled:
-                                child_document.metadata[BuiltInField.uploader] = dataset_document.created_by
-                                child_document.metadata[BuiltInField.upload_date] = dataset_document.created_at
-                                child_document.metadata[BuiltInField.last_update_date] = dataset_document.updated_at
-                                child_document.metadata[BuiltInField.source] = dataset_document.data_source_type
-                                child_document.metadata[BuiltInField.original_filename] = dataset_document.name
-                            if dataset_document.doc_metadata:
-                                child_document.metadata.update(dataset_document.doc_metadata)
-                            child_documents.append(child_document)
-                        document.children = child_documents
-                documents.append(document)  # noqa: B909
-            # save vector index
-            index_processor.load(dataset, documents)
-        end_at = time.perf_counter()
-        logging.info(
-            click.style("Updated documents metadata: {} latency: {}".format(dataset_id, end_at - start_at), fg="green")
-        )
-    except Exception:
-        logging.exception("Updated documents metadata failed")
-    finally:
-        if lock_key:
-            redis_client.delete(lock_key)

api/tests/integration_tests/.env.example

@@ -87,7 +87,6 @@ ZHINAO_API_KEY=
 # Plugin configuration
 PLUGIN_DAEMON_KEY=
 PLUGIN_DAEMON_URL=
-INNER_API_KEY=
 
 # Marketplace configuration
 MARKETPLACE_API_URL=

api/tests/integration_tests/vdb/oracle/test_oraclevector.py

@@ -13,11 +13,9 @@ class OracleVectorTest(AbstractVectorTest):
         self.vector = OracleVector(
             collection_name=self.collection_name,
             config=OracleVectorConfig(
-                host="localhost",
-                port=1521,
                 user="dify",
                 password="dify",
-                database="FREEPDB1",
+                dsn="localhost:1521/FREEPDB1",
             ),
         )
 

docker/.env.example

@@ -483,11 +483,13 @@ CHROMA_AUTH_PROVIDER=chromadb.auth.token_authn.TokenAuthClientProvider
 CHROMA_AUTH_CREDENTIALS=
 
 # Oracle configuration, only available when VECTOR_STORE is `oracle`
-ORACLE_HOST=oracle
-ORACLE_PORT=1521
 ORACLE_USER=dify
 ORACLE_PASSWORD=dify
-ORACLE_DATABASE=FREEPDB1
+ORACLE_DSN=oracle:1521/FREEPDB1
+ORACLE_CONFIG_DIR=/app/api/storage/wallet
+ORACLE_WALLET_LOCATION=/app/api/storage/wallet
+ORACLE_WALLET_PASSWORD=dify
+ORACLE_IS_AUTONOMOUS=false
 
 # relyt configurations, only available when VECTOR_STORE is `relyt`
 RELYT_HOST=db

docker/docker-compose-template.yaml

@@ -2,7 +2,7 @@ x-shared-env: &shared-api-worker-env
 services:
   # API service
   api:
-    image: langgenius/dify-api:0.15.3
+    image: langgenius/dify-api:1.0.0
     restart: always
     environment:
       # Use the shared environment variables.
@@ -12,6 +12,8 @@ services:
       SENTRY_DSN: ${API_SENTRY_DSN:-}
       SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0}
       SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0}
+      PLUGIN_REMOTE_INSTALL_HOST: ${EXPOSE_PLUGIN_DEBUGGING_HOST:-localhost}
+      PLUGIN_REMOTE_INSTALL_PORT: ${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}
       PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800}
       INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1}
     depends_on:
@@ -27,7 +29,7 @@ services:
   # worker service
   # The Celery worker for processing the queue.
   worker:
-    image: langgenius/dify-api:0.15.3
+    image: langgenius/dify-api:1.0.0
     restart: always
     environment:
       # Use the shared environment variables.
@@ -51,7 +53,7 @@ services:
 
   # Frontend web application.
   web:
-    image: langgenius/dify-web:0.15.3
+    image: langgenius/dify-web:1.0.0
     restart: always
     environment:
       CONSOLE_API_URL: ${CONSOLE_API_URL:-}
@@ -130,7 +132,7 @@ services:
 
   # plugin daemon
   plugin_daemon:
-    image: langgenius/dify-plugin-daemon:0.0.2-local
+    image: langgenius/dify-plugin-daemon:0.0.3-local
     restart: always
     environment:
       # Use the shared environment variables.
@@ -142,8 +144,8 @@ services:
       PPROF_ENABLED: ${PLUGIN_PPROF_ENABLED:-false}
       DIFY_INNER_API_URL: ${PLUGIN_DIFY_INNER_API_URL:-http://api:5001}
       DIFY_INNER_API_KEY: ${INNER_API_KEY_FOR_PLUGIN:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1}
-      PLUGIN_REMOTE_INSTALLING_HOST: ${PLUGIN_REMOTE_INSTALL_HOST:-0.0.0.0}
-      PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_REMOTE_INSTALL_PORT:-5003}
+      PLUGIN_REMOTE_INSTALLING_HOST: ${PLUGIN_DEBUGGING_HOST:-0.0.0.0}
+      PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_DEBUGGING_PORT:-5003}
       PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd}
       FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true}
     ports:

docker/docker-compose.middleware.yaml

@@ -66,7 +66,7 @@ services:
 
   # plugin daemon
   plugin_daemon:
-    image: langgenius/dify-plugin-daemon:0.0.2-local
+    image: langgenius/dify-plugin-daemon:0.0.3-local
     restart: always
     environment:
       # Use the shared environment variables.

docker/docker-compose.yaml

@@ -197,11 +197,13 @@ x-shared-env: &shared-api-worker-env
   CHROMA_DATABASE: ${CHROMA_DATABASE:-default_database}
   CHROMA_AUTH_PROVIDER: ${CHROMA_AUTH_PROVIDER:-chromadb.auth.token_authn.TokenAuthClientProvider}
   CHROMA_AUTH_CREDENTIALS: ${CHROMA_AUTH_CREDENTIALS:-}
-  ORACLE_HOST: ${ORACLE_HOST:-oracle}
-  ORACLE_PORT: ${ORACLE_PORT:-1521}
   ORACLE_USER: ${ORACLE_USER:-dify}
   ORACLE_PASSWORD: ${ORACLE_PASSWORD:-dify}
-  ORACLE_DATABASE: ${ORACLE_DATABASE:-FREEPDB1}
+  ORACLE_DSN: ${ORACLE_DSN:-oracle:1521/FREEPDB1}
+  ORACLE_CONFIG_DIR: ${ORACLE_CONFIG_DIR:-/app/api/storage/wallet}
+  ORACLE_WALLET_LOCATION: ${ORACLE_WALLET_LOCATION:-/app/api/storage/wallet}
+  ORACLE_WALLET_PASSWORD: ${ORACLE_WALLET_PASSWORD:-dify}
+  ORACLE_IS_AUTONOMOUS: ${ORACLE_IS_AUTONOMOUS:-false}
   RELYT_HOST: ${RELYT_HOST:-db}
   RELYT_PORT: ${RELYT_PORT:-5432}
   RELYT_USER: ${RELYT_USER:-postgres}
@@ -414,7 +416,7 @@ x-shared-env: &shared-api-worker-env
 services:
   # API service
   api:
-    image: langgenius/dify-api:0.15.3
+    image: langgenius/dify-api:1.0.0
     restart: always
     environment:
       # Use the shared environment variables.
@@ -424,6 +426,8 @@ services:
       SENTRY_DSN: ${API_SENTRY_DSN:-}
       SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0}
       SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0}
+      PLUGIN_REMOTE_INSTALL_HOST: ${EXPOSE_PLUGIN_DEBUGGING_HOST:-localhost}
+      PLUGIN_REMOTE_INSTALL_PORT: ${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}
       PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800}
       INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1}
     depends_on:
@@ -439,7 +443,7 @@ services:
   # worker service
   # The Celery worker for processing the queue.
   worker:
-    image: langgenius/dify-api:0.15.3
+    image: langgenius/dify-api:1.0.0
     restart: always
     environment:
       # Use the shared environment variables.
@@ -463,7 +467,7 @@ services:
 
   # Frontend web application.
   web:
-    image: langgenius/dify-web:0.15.3
+    image: langgenius/dify-web:1.0.0
     restart: always
     environment:
       CONSOLE_API_URL: ${CONSOLE_API_URL:-}
@@ -542,7 +546,7 @@ services:
 
   # plugin daemon
   plugin_daemon:
-    image: langgenius/dify-plugin-daemon:0.0.2-local
+    image: langgenius/dify-plugin-daemon:0.0.3-local
     restart: always
     environment:
       # Use the shared environment variables.
@@ -554,8 +558,8 @@ services:
       PPROF_ENABLED: ${PLUGIN_PPROF_ENABLED:-false}
       DIFY_INNER_API_URL: ${PLUGIN_DIFY_INNER_API_URL:-http://api:5001}
       DIFY_INNER_API_KEY: ${INNER_API_KEY_FOR_PLUGIN:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1}
-      PLUGIN_REMOTE_INSTALLING_HOST: ${PLUGIN_REMOTE_INSTALL_HOST:-0.0.0.0}
-      PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_REMOTE_INSTALL_PORT:-5003}
+      PLUGIN_REMOTE_INSTALLING_HOST: ${PLUGIN_DEBUGGING_HOST:-0.0.0.0}
+      PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_DEBUGGING_PORT:-5003}
       PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd}
       FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true}
     ports:

docker/nginx/conf.d/default.conf.template

@@ -29,7 +29,7 @@ server {
       include proxy.conf;
     }
 
-    location /e {
+    location /e/ {
       proxy_pass http://plugin_daemon:5002;
       proxy_set_header Dify-Hook-Url $scheme://$host$request_uri;
       include proxy.conf;

docker/startupscripts/init_user.script

@@ -5,6 +5,6 @@ create user dify identified by dify DEFAULT TABLESPACE users quota unlimited on
 grant DB_DEVELOPER_ROLE to dify;
 
 BEGIN
-CTX_DDL.CREATE_PREFERENCE('my_chinese_vgram_lexer','CHINESE_VGRAM_LEXER');
+CTX_DDL.CREATE_PREFERENCE('dify.multilingual_lexer','CHINESE_VGRAM_LEXER');
 END;
 /

web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx

@@ -94,7 +94,7 @@ const AppDetailLayout: FC<IAppDetailLayoutProps> = (props) => {
       },
     ]
     return navs
-  }, [t])
+  }, [])
 
   useEffect(() => {
     if (appDetail) {
@@ -120,7 +120,7 @@ const AppDetailLayout: FC<IAppDetailLayoutProps> = (props) => {
     }).finally(() => {
       setIsLoadingAppDetail(false)
     })
-  }, [appId, router, setAppDetail])
+  }, [appId, pathname])
 
   useEffect(() => {
     if (!appDetailRes || isLoadingCurrentWorkspace || isLoadingAppDetail)
@@ -148,7 +148,7 @@ const AppDetailLayout: FC<IAppDetailLayoutProps> = (props) => {
       }
     }
   // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [appDetailRes, appId, getNavigations, isCurrentWorkspaceEditor, isLoadingAppDetail, isLoadingCurrentWorkspace, router, setAppDetail, systemFeatures.enable_web_sso_switch_component])
+  }, [appDetailRes, isCurrentWorkspaceEditor, isLoadingAppDetail, isLoadingCurrentWorkspace, systemFeatures.enable_web_sso_switch_component])
 
   useUnmount(() => {
     setAppDetail()

web/app/components/app-sidebar/navLink.tsx

@@ -3,13 +3,13 @@
 import { useSelectedLayoutSegment } from 'next/navigation'
 import Link from 'next/link'
 import classNames from '@/utils/classnames'
+import type { RemixiconComponentType } from '@remixicon/react'
 
 export type NavIcon = React.ComponentType<
 React.PropsWithoutRef<React.ComponentProps<'svg'>> & {
   title?: string | undefined
   titleId?: string | undefined
-}
->
+}> | RemixiconComponentType
 
 export type NavLinkProps = {
   name: string

web/app/components/app/annotation/batch-add-annotation-modal/csv-downloader.tsx

@@ -42,7 +42,7 @@ const CSVDownload: FC = () => {
               <td className='h-9 pl-3 pr-2 border-b border-divider-regular'>{t('appAnnotation.batchModal.answer')}</td>
             </tr>
           </thead>
-          <tbody className='text-gray-700'>
+          <tbody className='text-text-secondary'>
             <tr>
               <td className='h-9 pl-3 pr-2 border-b border-divider-subtle text-[13px]'>{t('appAnnotation.batchModal.question')} 1</td>
               <td className='h-9 pl-3 pr-2 border-b border-divider-subtle text-[13px]'>{t('appAnnotation.batchModal.answer')} 1</td>

web/app/components/app/app-publisher/index.tsx

@@ -219,7 +219,7 @@ const AppPublisher = ({
               )}
             <SuggestedAction
               onClick={() => {
-                handleOpenInExplore()
+                publishedAt && handleOpenInExplore()
               }}
               disabled={!publishedAt}
               icon={<RiPlanetLine className='w-4 h-4' />}

web/app/components/app/configuration/dataset-config/select-dataset/index.tsx

@@ -132,11 +132,11 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
                   toggleSelect(item)
                 }}
               >
-                <div className='mr-1 flex items-center'>
+                <div className='mr-1 flex items-center overflow-hidden'>
                   <div className={cn('mr-2', !item.embedding_available && 'opacity-30')}>
                     <TypeIcon type="upload_file" size='md' />
                   </div>
-                  <div className={cn('max-w-[200px] text-[13px] font-medium text-text-secondary overflow-hidden text-ellipsis whitespace-nowrap', !item.embedding_available && 'opacity-30 !max-w-[120px]')}>{item.name}</div>
+                  <div className={cn('max-w-[200px] text-[13px] font-medium text-text-secondary truncate', !item.embedding_available && 'opacity-30 !max-w-[120px]')}>{item.name}</div>
                   {!item.embedding_available && (
                     <span className='ml-1 shrink-0 px-1 border border-divider-deep rounded-md text-text-tertiary text-xs font-normal leading-[18px]'>{t('dataset.unavailable')}</span>
                   )}
@@ -144,13 +144,14 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
                 {
                   item.indexing_technique && (
                     <Badge
+                      className='shrink-0'
                       text={formatIndexingTechniqueAndMethod(item.indexing_technique, item.retrieval_model_dict?.search_method)}
                     />
                   )
                 }
                 {
                   item.provider === 'external' && (
-                    <Badge text={t('dataset.externalTag')} />
+                    <Badge className='shrink-0' text={t('dataset.externalTag')} />
                   )
                 }
               </div>

web/app/components/app/configuration/debug/debug-with-multiple-model/text-generation-item.tsx

@@ -124,18 +124,9 @@ const TextGenerationItem: FC<TextGenerationItemProps> = ({
       doSend(v.payload.message, v.payload.files)
   })
 
-  const varList = modelConfig.configs.prompt_variables.map((item: any) => {
-    return {
-      label: item.key,
-      value: inputs[item.key],
-    }
-  })
-
   return (
     <TextGeneration
       className='flex flex-col h-full overflow-y-auto border-none'
-      innerClassName='grow flex flex-col'
-      contentClassName='grow'
       content={completion}
       isLoading={!completion && isResponding}
       isResponding={isResponding}
@@ -144,8 +135,7 @@ const TextGenerationItem: FC<TextGenerationItemProps> = ({
       messageId={messageId}
       isError={false}
       onRetry={() => { }}
-      appId={appId}
-      varList={varList}
+      inSidePanel
     />
   )
 }

web/app/components/app/configuration/debug/index.tsx

@@ -516,9 +516,6 @@ const Debug: FC<IDebug> = ({
                         messageId={messageId}
                         isError={false}
                         onRetry={() => { }}
-                        supportAnnotation
-                        appId={appId}
-                        varList={varList}
                         siteInfo={null}
                       />
                     </div>

web/app/components/app/configuration/index.tsx

@@ -94,7 +94,7 @@ const Configuration: FC = () => {
   })))
   const { data: fileUploadConfigResponse } = useSWR({ url: '/files/upload' }, fetchFileUploadConfig)
 
-  const latestPublishedAt = useMemo(() => appDetail?.model_config.updated_at, [appDetail])
+  const latestPublishedAt = useMemo(() => appDetail?.model_config?.updated_at, [appDetail])
   const [formattingChanged, setFormattingChanged] = useState(false)
   const { setShowAccountSettingModal } = useModalContext()
   const [hasFetchedDetail, setHasFetchedDetail] = useState(false)

web/app/components/app/create-app-dialog/app-list/index.tsx

@@ -128,7 +128,7 @@ const Apps = ({
     icon_background,
     description,
   }) => {
-    const { export_data } = await fetchAppDetail(
+    const { export_data, mode } = await fetchAppDetail(
       currApp?.app.id as string,
     )
     try {
@@ -151,7 +151,7 @@ const Apps = ({
       if (app.app_id)
         await handleCheckPluginDependencies(app.app_id)
       localStorage.setItem(NEED_REFRESH_APP_LIST_KEY, '1')
-      getRedirection(isCurrentWorkspaceEditor, { id: app.app_id }, push)
+      getRedirection(isCurrentWorkspaceEditor, { id: app.app_id, mode }, push)
     }
     catch (e) {
       Toast.notify({ type: 'error', message: t('app.newApp.appCreateFailed') })