Merge branch 'feat/upgrade-knowledge-metabase' into deploy/dev

# Conflicts:
#	api/poetry.lock

.gitignore

@@ -183,6 +183,7 @@ docker/nginx/conf.d/default.conf
 docker/nginx/ssl/*
 !docker/nginx/ssl/.gitkeep
 docker/middleware.env
+docker/docker-compose.override.yaml
 
 sdks/python-client/build
 sdks/python-client/dist

README.md

@@ -40,6 +40,7 @@
 
 <p align="center">
   <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-d9d9d9"></a>
+  <a href="./README_TW.md"><img alt="繁體中文文件" src="https://img.shields.io/badge/繁體中文-d9d9d9"></a>
   <a href="./README_CN.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-d9d9d9"></a>
   <a href="./README_JA.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-d9d9d9"></a>
   <a href="./README_ES.md"><img alt="README en Español" src="https://img.shields.io/badge/Español-d9d9d9"></a>
@@ -53,14 +54,14 @@
   <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </p>
 
-
-Dify is an open-source LLM app development platform. Its intuitive interface combines agentic AI workflow, RAG pipeline, agent capabilities, model management, observability features and more, letting you quickly go from prototype to production. 
+Dify is an open-source LLM app development platform. Its intuitive interface combines agentic AI workflow, RAG pipeline, agent capabilities, model management, observability features and more, letting you quickly go from prototype to production.
 
 ## Quick start
+
 > Before installing Dify, make sure your machine meets the following minimum system requirements:
-> 
->- CPU >= 2 Core
->- RAM >= 4 GiB
+>
+> - CPU >= 2 Core
+> - RAM >= 4 GiB
 
 </br>
 
@@ -76,41 +77,40 @@ docker compose up -d
 After running, you can access the Dify dashboard in your browser at [http://localhost/install](http://localhost/install) and start the initialization process.
 
 #### Seeking help
+
 Please refer to our [FAQ](https://docs.dify.ai/getting-started/install-self-hosted/faqs) if you encounter problems setting up Dify. Reach out to [the community and us](#community--contact) if you are still having issues.
 
 > If you'd like to contribute to Dify or do additional development, refer to our [guide to deploying from source code](https://docs.dify.ai/getting-started/install-self-hosted/local-source-code)
 
 ## Key features
-**1. Workflow**: 
-  Build and test powerful AI workflows on a visual canvas, leveraging all the following features and beyond.
 
+**1. Workflow**:
+Build and test powerful AI workflows on a visual canvas, leveraging all the following features and beyond.
 
-  https://github.com/langgenius/dify/assets/13230914/356df23e-1604-483d-80a6-9517ece318aa
+https://github.com/langgenius/dify/assets/13230914/356df23e-1604-483d-80a6-9517ece318aa
 
-
-
-**2. Comprehensive model support**: 
-  Seamless integration with hundreds of proprietary / open-source LLMs from dozens of inference providers and self-hosted solutions, covering GPT, Mistral, Llama3, and any OpenAI API-compatible models. A full list of supported model providers can be found [here](https://docs.dify.ai/getting-started/readme/model-providers).
+**2. Comprehensive model support**:
+Seamless integration with hundreds of proprietary / open-source LLMs from dozens of inference providers and self-hosted solutions, covering GPT, Mistral, Llama3, and any OpenAI API-compatible models. A full list of supported model providers can be found [here](https://docs.dify.ai/getting-started/readme/model-providers).
 
 ![providers-v5](https://github.com/langgenius/dify/assets/13230914/5a17bdbe-097a-4100-8363-40255b70f6e3)
 
+**3. Prompt IDE**:
+Intuitive interface for crafting prompts, comparing model performance, and adding additional features such as text-to-speech to a chat-based app.
 
-**3. Prompt IDE**: 
-  Intuitive interface for crafting prompts, comparing model performance, and adding additional features such as text-to-speech to a chat-based app. 
+**4. RAG Pipeline**:
+Extensive RAG capabilities that cover everything from document ingestion to retrieval, with out-of-box support for text extraction from PDFs, PPTs, and other common document formats.
 
-**4. RAG Pipeline**: 
-  Extensive RAG capabilities that cover everything from document ingestion to retrieval, with out-of-box support for text extraction from PDFs, PPTs, and other common document formats.
+**5. Agent capabilities**:
+You can define agents based on LLM Function Calling or ReAct, and add pre-built or custom tools for the agent. Dify provides 50+ built-in tools for AI agents, such as Google Search, DALL·E, Stable Diffusion and WolframAlpha.
 
-**5. Agent capabilities**: 
-  You can define agents based on LLM Function Calling or ReAct, and add pre-built or custom tools for the agent. Dify provides 50+ built-in tools for AI agents, such as Google Search, DALL·E, Stable Diffusion and WolframAlpha.
+**6. LLMOps**:
+Monitor and analyze application logs and performance over time. You could continuously improve prompts, datasets, and models based on production data and annotations.
 
-**6. LLMOps**: 
-  Monitor and analyze application logs and performance over time. You could continuously improve prompts, datasets, and models based on production data and annotations.
-
-**7. Backend-as-a-Service**: 
-  All of Dify's offerings come with corresponding APIs, so you could effortlessly integrate Dify into your own business logic.
+**7. Backend-as-a-Service**:
+All of Dify's offerings come with corresponding APIs, so you could effortlessly integrate Dify into your own business logic.
 
 ## Feature Comparison
+
 <table style="width: 100%;">
   <tr>
     <th align="center">Feature</th>
@@ -180,24 +180,22 @@ Please refer to our [FAQ](https://docs.dify.ai/getting-started/install-self-host
 ## Using Dify
 
 - **Cloud </br>**
-We host a [Dify Cloud](https://dify.ai) service for anyone to try with zero setup. It provides all the capabilities of the self-deployed version, and includes 200 free GPT-4 calls in the sandbox plan.
+  We host a [Dify Cloud](https://dify.ai) service for anyone to try with zero setup. It provides all the capabilities of the self-deployed version, and includes 200 free GPT-4 calls in the sandbox plan.
 
 - **Self-hosting Dify Community Edition</br>**
-Quickly get Dify running in your environment with this [starter guide](#quick-start).
-Use our [documentation](https://docs.dify.ai) for further references and more in-depth instructions.
+  Quickly get Dify running in your environment with this [starter guide](#quick-start).
+  Use our [documentation](https://docs.dify.ai) for further references and more in-depth instructions.
 
 - **Dify for enterprise / organizations</br>**
-We provide additional enterprise-centric features. [Log your questions for us through this chatbot](https://udify.app/chat/22L1zSxg6yW1cWQg) or [send us an email](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry) to discuss enterprise needs. </br>
+  We provide additional enterprise-centric features. [Log your questions for us through this chatbot](https://udify.app/chat/22L1zSxg6yW1cWQg) or [send us an email](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry) to discuss enterprise needs. </br>
   > For startups and small businesses using AWS, check out [Dify Premium on AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6) and deploy it to your own AWS VPC with one-click. It's an affordable AMI offering with the option to create apps with custom logo and branding.
 
-
 ## Staying ahead
 
 Star Dify on GitHub and be instantly notified of new releases.
 
 ![star-us](https://github.com/langgenius/dify/assets/13230914/b823edc1-6388-4e25-ad45-2f6b187adbb4)
 
-
 ## Advanced Setup
 
 If you need to customize the configuration, please refer to the comments in our [.env.example](docker/.env.example) file and update the corresponding values in your `.env` file. Additionally, you might need to make adjustments to the `docker-compose.yaml` file itself, such as changing image versions, port mappings, or volume mounts, based on your specific deployment environment and requirements. After making any changes, please re-run `docker-compose up -d`. You can find the full list of available environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
@@ -213,32 +211,34 @@ If you'd like to configure a highly-available setup, there are community-contrib
 Deploy Dify to Cloud Platform with a single click using [terraform](https://www.terraform.io/)
 
 ##### Azure Global
+
 - [Azure Terraform by @nikawang](https://github.com/nikawang/dify-azure-terraform)
 
 ##### Google Cloud
+
 - [Google Cloud Terraform by @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
 #### Using AWS CDK for Deployment
 
 Deploy Dify to AWS with [CDK](https://aws.amazon.com/cdk/)
 
-##### AWS 
+##### AWS
+
 - [AWS CDK by @KevinZhao](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
 
 ## Contributing
 
-For those who'd like to contribute code, see our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). 
+For those who'd like to contribute code, see our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
 At the same time, please consider supporting Dify by sharing it on social media and at events and conferences.
 
-
 > We are looking for contributors to help with translating Dify to languages other than Mandarin or English. If you are interested in helping, please see the [i18n README](https://github.com/langgenius/dify/blob/main/web/i18n/README.md) for more information, and leave us a comment in the `global-users` channel of our [Discord Community Server](https://discord.gg/8Tpq4AcN9c).
 
 ## Community & contact
 
-* [Github Discussion](https://github.com/langgenius/dify/discussions). Best for: sharing feedback and asking questions.
-* [GitHub Issues](https://github.com/langgenius/dify/issues). Best for: bugs you encounter using Dify.AI, and feature proposals. See our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
-* [Discord](https://discord.gg/FngNHpbcY7). Best for: sharing your applications and hanging out with the community.
-* [X(Twitter)](https://twitter.com/dify_ai). Best for: sharing your applications and hanging out with the community.
+- [Github Discussion](https://github.com/langgenius/dify/discussions). Best for: sharing feedback and asking questions.
+- [GitHub Issues](https://github.com/langgenius/dify/issues). Best for: bugs you encounter using Dify.AI, and feature proposals. See our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
+- [Discord](https://discord.gg/FngNHpbcY7). Best for: sharing your applications and hanging out with the community.
+- [X(Twitter)](https://twitter.com/dify_ai). Best for: sharing your applications and hanging out with the community.
 
 **Contributors**
 
@@ -250,7 +250,6 @@ At the same time, please consider supporting Dify by sharing it on social media
 
 [![Star History Chart](https://api.star-history.com/svg?repos=langgenius/dify&type=Date)](https://star-history.com/#langgenius/dify&Date)
 
-
 ## Security disclosure
 
 To protect your privacy, please avoid posting security issues on GitHub. Instead, send your questions to security@dify.ai and we will provide you with a more detailed answer.
@@ -258,4 +257,3 @@ To protect your privacy, please avoid posting security issues on GitHub. Instead
 ## License
 
 This repository is available under the [Dify Open Source License](LICENSE), which is essentially Apache 2.0 with a few additional restrictions.
-

api/controllers/console/app/workflow_app_log.py

@@ -1,13 +1,18 @@
+from datetime import datetime
+
 from flask_restful import Resource, marshal_with, reqparse  # type: ignore
 from flask_restful.inputs import int_range  # type: ignore
+from sqlalchemy.orm import Session
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
+from extensions.ext_database import db
 from fields.workflow_app_log_fields import workflow_app_log_pagination_fields
 from libs.login import login_required
 from models import App
 from models.model import AppMode
+from models.workflow import WorkflowRunStatus
 from services.workflow_app_service import WorkflowAppService
 
 
@@ -24,17 +29,38 @@ class WorkflowAppLogApi(Resource):
         parser = reqparse.RequestParser()
         parser.add_argument("keyword", type=str, location="args")
         parser.add_argument("status", type=str, choices=["succeeded", "failed", "stopped"], location="args")
+        parser.add_argument(
+            "created_at__before", type=str, location="args", help="Filter logs created before this timestamp"
+        )
+        parser.add_argument(
+            "created_at__after", type=str, location="args", help="Filter logs created after this timestamp"
+        )
         parser.add_argument("page", type=int_range(1, 99999), default=1, location="args")
         parser.add_argument("limit", type=int_range(1, 100), default=20, location="args")
         args = parser.parse_args()
 
+        args.status = WorkflowRunStatus(args.status) if args.status else None
+        if args.created_at__before:
+            args.created_at__before = datetime.fromisoformat(args.created_at__before.replace("Z", "+00:00"))
+
+        if args.created_at__after:
+            args.created_at__after = datetime.fromisoformat(args.created_at__after.replace("Z", "+00:00"))
+
         # get paginate workflow app logs
         workflow_app_service = WorkflowAppService()
-        workflow_app_log_pagination = workflow_app_service.get_paginate_workflow_app_logs(
-            app_model=app_model, args=args
-        )
+        with Session(db.engine) as session:
+            workflow_app_log_pagination = workflow_app_service.get_paginate_workflow_app_logs(
+                session=session,
+                app_model=app_model,
+                keyword=args.keyword,
+                status=args.status,
+                created_at_before=args.created_at__before,
+                created_at_after=args.created_at__after,
+                page=args.page,
+                limit=args.limit,
+            )
 
-        return workflow_app_log_pagination
+            return workflow_app_log_pagination
 
 
 api.add_resource(WorkflowAppLogApi, "/apps/<uuid:app_id>/workflow-app-logs")

api/controllers/console/error.py

@@ -103,7 +103,13 @@ class AccountInFreezeError(BaseHTTPException):
     )
 
 
-class CompilanceRateLimitError(BaseHTTPException):
-    error_code = "compilance_rate_limit"
-    description = "Rate limit exceeded for downloading compliance report."
+class EducationVerifyLimitError(BaseHTTPException):
+    error_code = "education_verify_limit"
+    description = "Rate limit exceeded"
+    code = 429
+
+
+class EducationActivateLimitError(BaseHTTPException):
+    error_code = "education_activate_limit"
+    description = "Rate limit exceeded"
     code = 429

api/controllers/console/workspace/account.py

@@ -15,7 +15,13 @@ from controllers.console.workspace.error import (
     InvalidInvitationCodeError,
     RepeatPasswordNotMatchError,
 )
-from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required
+from controllers.console.wraps import (
+    account_initialization_required,
+    cloud_edition_billing_enabled,
+    enterprise_license_required,
+    only_edition_cloud,
+    setup_required,
+)
 from extensions.ext_database import db
 from fields.member_fields import account_fields
 from libs.helper import TimestampField, timezone
@@ -292,6 +298,78 @@ class AccountDeleteUpdateFeedbackApi(Resource):
         return {"result": "success"}
 
 
+class EducationVerifyApi(Resource):
+    verify_fields = {
+        "token": fields.String,
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @only_edition_cloud
+    @cloud_edition_billing_enabled
+    @marshal_with(verify_fields)
+    def get(self):
+        account = current_user
+
+        return BillingService.EducationIdentity.verify(account.id, account.email)
+
+
+class EducationApi(Resource):
+    status_fields = {
+        "result": fields.Boolean,
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @only_edition_cloud
+    @cloud_edition_billing_enabled
+    def post(self):
+        account = current_user
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("token", type=str, required=True, location="json")
+        parser.add_argument("institution", type=str, required=True, location="json")
+        args = parser.parse_args()
+
+        return BillingService.EducationIdentity.activate(account, args["token"], args["institution"])
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @only_edition_cloud
+    @cloud_edition_billing_enabled
+    @marshal_with(status_fields)
+    def get(self):
+        account = current_user
+
+        return BillingService.EducationIdentity.is_active(account.id)
+
+
+class EducationAutoCompleteApi(Resource):
+    data_fields = {
+        "data": fields.List(fields.String),
+        "curr_page": fields.Integer,
+        "has_next": fields.Boolean,
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @only_edition_cloud
+    @cloud_edition_billing_enabled
+    @marshal_with(data_fields)
+    def get(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument("keywords", type=str, required=True, location="args")
+        parser.add_argument("page", type=int, required=False, location="args", default=0)
+        parser.add_argument("limit", type=int, required=False, location="args", default=20)
+        args = parser.parse_args()
+
+        return BillingService.EducationIdentity.autocomplete(args["keywords"], args["page"], args["limit"])
+
+
 # Register API resources
 api.add_resource(AccountInitApi, "/account/init")
 api.add_resource(AccountProfileApi, "/account/profile")
@@ -305,5 +383,8 @@ api.add_resource(AccountIntegrateApi, "/account/integrates")
 api.add_resource(AccountDeleteVerifyApi, "/account/delete/verify")
 api.add_resource(AccountDeleteApi, "/account/delete")
 api.add_resource(AccountDeleteUpdateFeedbackApi, "/account/delete/feedback")
+api.add_resource(EducationVerifyApi, "/account/education/verify")
+api.add_resource(EducationApi, "/account/education")
+api.add_resource(EducationAutoCompleteApi, "/account/education/autocomplete")
 # api.add_resource(AccountEmailApi, '/account/email')
 # api.add_resource(AccountEmailVerifyApi, '/account/email-verify')

api/controllers/console/wraps.py

@@ -51,6 +51,17 @@ def only_edition_self_hosted(view):
     return decorated
 
 
+def cloud_edition_billing_enabled(view):
+    @wraps(view)
+    def decorated(*args, **kwargs):
+        features = FeatureService.get_features(current_user.current_tenant_id)
+        if not features.billing.enabled:
+            abort(403, "Billing feature is not enabled.")
+        return view(*args, **kwargs)
+
+    return decorated
+
+
 def cloud_edition_billing_resource_check(resource: str):
     def interceptor(view):
         @wraps(view)

api/controllers/service_api/app/message.py

@@ -70,7 +70,7 @@ class MessageListApi(Resource):
 
         try:
             return MessageService.pagination_by_first_id(
-                app_model, end_user, args["conversation_id"], args["first_id"], args["limit"], "desc"
+                app_model, end_user, args["conversation_id"], args["first_id"], args["limit"]
             )
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")

api/controllers/service_api/app/workflow.py

@@ -1,7 +1,9 @@
 import logging
+from datetime import datetime
 
 from flask_restful import Resource, fields, marshal_with, reqparse  # type: ignore
 from flask_restful.inputs import int_range  # type: ignore
+from sqlalchemy.orm import Session
 from werkzeug.exceptions import InternalServerError
 
 from controllers.service_api import api
@@ -25,7 +27,7 @@ from extensions.ext_database import db
 from fields.workflow_app_log_fields import workflow_app_log_pagination_fields
 from libs import helper
 from models.model import App, AppMode, EndUser
-from models.workflow import WorkflowRun
+from models.workflow import WorkflowRun, WorkflowRunStatus
 from services.app_generate_service import AppGenerateService
 from services.workflow_app_service import WorkflowAppService
 
@@ -125,17 +127,34 @@ class WorkflowAppLogApi(Resource):
         parser = reqparse.RequestParser()
         parser.add_argument("keyword", type=str, location="args")
         parser.add_argument("status", type=str, choices=["succeeded", "failed", "stopped"], location="args")
+        parser.add_argument("created_at__before", type=str, location="args")
+        parser.add_argument("created_at__after", type=str, location="args")
         parser.add_argument("page", type=int_range(1, 99999), default=1, location="args")
         parser.add_argument("limit", type=int_range(1, 100), default=20, location="args")
         args = parser.parse_args()
 
+        args.status = WorkflowRunStatus(args.status) if args.status else None
+        if args.created_at__before:
+            args.created_at__before = datetime.fromisoformat(args.created_at__before.replace("Z", "+00:00"))
+
+        if args.created_at__after:
+            args.created_at__after = datetime.fromisoformat(args.created_at__after.replace("Z", "+00:00"))
+
         # get paginate workflow app logs
         workflow_app_service = WorkflowAppService()
-        workflow_app_log_pagination = workflow_app_service.get_paginate_workflow_app_logs(
-            app_model=app_model, args=args
-        )
+        with Session(db.engine) as session:
+            workflow_app_log_pagination = workflow_app_service.get_paginate_workflow_app_logs(
+                session=session,
+                app_model=app_model,
+                keyword=args.keyword,
+                status=args.status,
+                created_at_before=args.created_at__before,
+                created_at_after=args.created_at__after,
+                page=args.page,
+                limit=args.limit,
+            )
 
-        return workflow_app_log_pagination
+            return workflow_app_log_pagination
 
 
 api.add_resource(WorkflowRunApi, "/workflows/run")

api/core/app/app_config/entities.py

@@ -140,12 +140,12 @@ SupportedComparisonOperator = Literal[
     # for string or array
     "contains",
     "not contains",
-    "starts with",
-    "ends with",
+    "start with",
+    "end with",
     "is",
     "is not",
     "empty",
-    "is not empty",
+    "not empty",
     # for number
     "=",
     "≠",
@@ -173,7 +173,7 @@ class Condition(BaseModel):
 
     name: str
     comparison_operator: SupportedComparisonOperator
-    value: str | Sequence[str] | None = None
+    value: str | Sequence[str] | None | int | float = None
 
 
 class MetadataFilteringCondition(BaseModel):
@@ -223,7 +223,7 @@ class DatasetRetrieveConfigEntity(BaseModel):
     reranking_enabled: Optional[bool] = True
     metadata_filtering_mode: Optional[Literal["disabled", "automatic", "manual"]] = "disabled"
     metadata_model_config: Optional[ModelConfig] = None
-    metadata_filtering_conditions: Optional[list[MetadataFilteringCondition]] = None
+    metadata_filtering_conditions: Optional[MetadataFilteringCondition] = None
 
 
 class DatasetEntity(BaseModel):

api/core/file/models.py

@@ -97,32 +97,18 @@ class File(BaseModel):
         return text
 
     def generate_url(self) -> Optional[str]:
-        if self.type == FileType.IMAGE:
-            if self.transfer_method == FileTransferMethod.REMOTE_URL:
-                return self.remote_url
-            elif self.transfer_method == FileTransferMethod.LOCAL_FILE:
-                if self.related_id is None:
-                    raise ValueError("Missing file related_id")
-                return helpers.get_signed_file_url(upload_file_id=self.related_id)
-            elif self.transfer_method == FileTransferMethod.TOOL_FILE:
-                assert self.related_id is not None
-                assert self.extension is not None
-                return ToolFileParser.get_tool_file_manager().sign_file(
-                    tool_file_id=self.related_id, extension=self.extension
-                )
-        else:
-            if self.transfer_method == FileTransferMethod.REMOTE_URL:
-                return self.remote_url
-            elif self.transfer_method == FileTransferMethod.LOCAL_FILE:
-                if self.related_id is None:
-                    raise ValueError("Missing file related_id")
-                return helpers.get_signed_file_url(upload_file_id=self.related_id)
-            elif self.transfer_method == FileTransferMethod.TOOL_FILE:
-                assert self.related_id is not None
-                assert self.extension is not None
-                return ToolFileParser.get_tool_file_manager().sign_file(
-                    tool_file_id=self.related_id, extension=self.extension
-                )
+        if self.transfer_method == FileTransferMethod.REMOTE_URL:
+            return self.remote_url
+        elif self.transfer_method == FileTransferMethod.LOCAL_FILE:
+            if self.related_id is None:
+                raise ValueError("Missing file related_id")
+            return helpers.get_signed_file_url(upload_file_id=self.related_id)
+        elif self.transfer_method == FileTransferMethod.TOOL_FILE:
+            assert self.related_id is not None
+            assert self.extension is not None
+            return ToolFileParser.get_tool_file_manager().sign_file(
+                tool_file_id=self.related_id, extension=self.extension
+            )
 
     def to_plugin_parameter(self) -> dict[str, Any]:
         return {

api/core/plugin/entities/plugin.py

@@ -5,6 +5,7 @@ from collections.abc import Mapping
 from typing import Any, Optional
 
 from pydantic import BaseModel, Field, model_validator
+from werkzeug.exceptions import NotFound
 
 from core.agent.plugin_entities import AgentStrategyProviderEntity
 from core.model_runtime.entities.provider_entities import ProviderEntity
@@ -153,6 +154,8 @@ class GenericProviderID:
         return f"{self.organization}/{self.plugin_name}/{self.provider_name}"
 
     def __init__(self, value: str, is_hardcoded: bool = False) -> None:
+        if not value:
+            raise NotFound("plugin not found, please add plugin")
         # check if the value is a valid plugin id with format: $organization/$plugin_name/$provider_name
         if not re.match(r"^[a-z0-9_-]+\/[a-z0-9_-]+\/[a-z0-9_-]+$", value):
             # check if matches [a-z0-9_-]+, if yes, append with langgenius/$value/$value

api/core/rag/retrieval/dataset_retrieval.py

@@ -51,7 +51,7 @@ from core.rag.retrieval.template_prompts import (
 )
 from core.tools.utils.dataset_retriever.dataset_retriever_base_tool import DatasetRetrieverBaseTool
 from extensions.ext_database import db
-from models.dataset import ChildChunk, Dataset, DatasetQuery, DocumentSegment
+from models.dataset import ChildChunk, Dataset, DatasetMetadata, DatasetQuery, DocumentSegment
 from models.dataset import Document as DatasetDocument
 from services.external_knowledge_service import ExternalDatasetService
 
@@ -395,6 +395,7 @@ class DatasetRetrieval:
         weights: Optional[dict[str, Any]] = None,
         reranking_enable: bool = True,
         message_id: Optional[str] = None,
+        metadata_filter_document_ids: Optional[dict[str, list[str]]] = None,
     ):
         if not available_datasets:
             return []
@@ -434,6 +435,11 @@ class DatasetRetrieval:
 
         for dataset in available_datasets:
             index_type = dataset.indexing_technique
+            document_ids_filter = None
+            if metadata_filter_document_ids:
+                document_ids = metadata_filter_document_ids.get(dataset.id, [])
+                if document_ids:
+                    document_ids_filter = document_ids
             retrieval_thread = threading.Thread(
                 target=self._retriever,
                 kwargs={
@@ -442,6 +448,7 @@ class DatasetRetrieval:
                     "query": query,
                     "top_k": top_k,
                     "all_documents": all_documents,
+                    "document_ids_filter": document_ids_filter,
                 },
             )
             threads.append(retrieval_thread)
@@ -537,7 +544,8 @@ class DatasetRetrieval:
             db.session.add_all(dataset_queries)
         db.session.commit()
 
-    def _retriever(self, flask_app: Flask, dataset_id: str, query: str, top_k: int, all_documents: list):
+    def _retriever(self, flask_app: Flask, dataset_id: str, query: str, top_k: int, all_documents: list,
+                    document_ids_filter: Optional[list[str]] = None):
         with flask_app.app_context():
             dataset = db.session.query(Dataset).filter(Dataset.id == dataset_id).first()
 
@@ -590,6 +598,7 @@ class DatasetRetrieval:
                             else None,
                             reranking_mode=retrieval_model.get("reranking_mode") or "reranking_model",
                             weights=retrieval_model.get("weights", None),
+                            document_ids_filter=document_ids_filter,
                         )
 
                         all_documents.extend(documents)
@@ -789,11 +798,11 @@ class DatasetRetrieval:
         metadata_filtering_conditions: Optional[MetadataFilteringCondition],
         inputs: dict,
     ) -> Optional[dict[str, list[str]]]:
-        document_query = db.session.query(Document).filter(
-            Document.dataset_id.in_(dataset_ids),
-            Document.indexing_status == "completed",
-            Document.enabled == True,
-            Document.archived == False,
+        document_query = db.session.query(DatasetDocument).filter(
+            DatasetDocument.dataset_id.in_(dataset_ids),
+            DatasetDocument.indexing_status == "completed",
+            DatasetDocument.enabled == True,
+            DatasetDocument.archived == False,
         )
         if metadata_filtering_mode == "disabled":
             return None
@@ -803,18 +812,20 @@ class DatasetRetrieval:
             )
             if automatic_metadata_filters:
                 for filter in automatic_metadata_filters:
-                    self._process_metadata_filter_func(
+                    document_query = self._process_metadata_filter_func(
                         filter.get("condition"), filter.get("metadata_name"), filter.get("value"), document_query
                     )
         elif metadata_filtering_mode == "manual":
-            for condition in metadata_filtering_conditions.conditions:
-                metadata_name = condition.name
-                expected_value = condition.value
-                if isinstance(expected_value, str):
-                    expected_value = self._replace_metadata_filter_value(expected_value, inputs)
-                self._process_metadata_filter_func(
-                    condition.comparison_operator, metadata_name, expected_value, document_query
-                )
+            if metadata_filtering_conditions:
+                for condition in metadata_filtering_conditions.conditions:
+                    metadata_name = condition.name
+                    expected_value = condition.value
+                    if expected_value:
+                        if isinstance(expected_value, str):
+                            expected_value = self._replace_metadata_filter_value(expected_value, inputs)
+                        document_query = self._process_metadata_filter_func(
+                            condition.comparison_operator, metadata_name, expected_value, document_query
+                        )
         else:
             raise ValueError("Invalid metadata filtering mode")
         documnents = document_query.all()
@@ -834,10 +845,10 @@ class DatasetRetrieval:
 
     def _automatic_metadata_filter_func(
         self, dataset_ids: list, query: str, tenant_id: str, user_id: str, metadata_model_config: ModelConfig
-    ) -> list[dict[str, Any]]:
+    ) -> Optional[list[dict[str, Any]]]:
         # get all metadata field
         metadata_fields = db.session.query(DatasetMetadata).filter(DatasetMetadata.dataset_id.in_(dataset_ids)).all()
-        all_metadata_fields = [metadata_field.field_name for metadata_field in metadata_fields]
+        all_metadata_fields = [metadata_field.name for metadata_field in metadata_fields]
         # get metadata model config
         if metadata_model_config is None:
             raise ValueError("metadata_model_config is required")
@@ -847,7 +858,6 @@ class DatasetRetrieval:
 
         # fetch prompt messages
         prompt_messages, stop = self._get_prompt_template(
-            model_instance=model_instance,
             model_config=model_config,
             mode=metadata_model_config.mode,
             metadata_fields=all_metadata_fields,
@@ -888,34 +898,41 @@ class DatasetRetrieval:
             return None
         return automatic_metadata_filters
 
-    def _process_metadata_filter_func(*, condition: str, metadata_name: str, value: str, query):
+    def _process_metadata_filter_func(self, condition: str, metadata_name: str, value: str, query):
         match condition:
             case "contains":
-                query = query.filter(Document.doc_metadata[metadata_name].like(f"%{value}%"))
+                query = query.filter(DatasetDocument.doc_metadata[metadata_name].like(f'"%{value}%"'))
             case "not contains":
-                query = query.filter(Document.doc_metadata[metadata_name].notlike(f"%{value}%"))
+                query = query.filter(DatasetDocument.doc_metadata[metadata_name].notlike(f'"%{value}%"'))
             case "start with":
-                query = query.filter(Document.doc_metadata[metadata_name].like(f"{value}%"))
+                query = query.filter(DatasetDocument.doc_metadata[metadata_name].like(f'"{value}%"'))
             case "end with":
-                query = query.filter(Document.doc_metadata[metadata_name].like(f"%{value}"))
-            case "is", "=":
-                query = query.filter(Document.doc_metadata[metadata_name] == value)
-            case "is not", "≠":
-                query = query.filter(Document.doc_metadata[metadata_name] != value)
+                query = query.filter(DatasetDocument.doc_metadata[metadata_name].like(f'"%{value}"'))
+            case "is" | "=":
+                if isinstance(value, str):
+                    query = query.filter(DatasetDocument.doc_metadata[metadata_name] == f'"{value}"')
+                else:
+                    query = query.filter(DatasetDocument.doc_metadata[metadata_name] == value)
+            case "is not" | "≠":
+                if isinstance(value, str):
+                    query = query.filter(DatasetDocument.doc_metadata[metadata_name] != f'"{value}"')
+                else:
+                    query = query.filter(DatasetDocument.doc_metadata[metadata_name] != value)
             case "is empty":
-                query = query.filter(Document.doc_metadata[metadata_name].is_(None))
+                query = query.filter(DatasetDocument.doc_metadata[metadata_name].is_(None))
             case "is not empty":
-                query = query.filter(Document.doc_metadata[metadata_name].isnot(None))
-            case "before", "<":
-                query = query.filter(Document.doc_metadata[metadata_name] < value)
-            case "after", ">":
-                query = query.filter(Document.doc_metadata[metadata_name] > value)
-            case "≤", ">=":
-                query = query.filter(Document.doc_metadata[metadata_name] <= value)
-            case "≥", ">=":
-                query = query.filter(Document.doc_metadata[metadata_name] >= value)
+                query = query.filter(DatasetDocument.doc_metadata[metadata_name].isnot(None))
+            case "before" | "<":
+                query = query.filter(DatasetDocument.doc_metadata[metadata_name] < value)
+            case "after" | ">":
+                query = query.filter(DatasetDocument.doc_metadata[metadata_name] > value)
+            case "≤" | ">=":
+                query = query.filter(DatasetDocument.doc_metadata[metadata_name] <= value)
+            case "≥" | ">=":
+                query = query.filter(DatasetDocument.doc_metadata[metadata_name] >= value)
             case _:
                 pass
+        return query
 
     def _fetch_model_config(
         self, tenant_id: str, model: ModelConfig

api/core/tools/docs/en_US/advanced_scale_out.md

@@ -55,7 +55,7 @@ If you need to return a text message, you can use the following interface.
 If you need to return the raw data of a file, such as images, audio, video, PPT, Word, Excel, etc., you can use the following interface.
 
 - `blob` The raw data of the file, of bytes type
-- `meta` The metadata of the file, if you know the type of the file, it is best to pass a `mime_type`, otherwise Dify will use `octet/stream` as the default type
+- `meta` The metadata of the file, if you know the type of the file, it is best to pass a `mime_type`, otherwise Dify will use `application/octet-stream` as the default type
 
 ```python
     def create_blob_message(self, blob: bytes, meta: dict = None, save_as: str = '') -> ToolInvokeMessage:

api/core/tools/docs/ja_JP/advanced_scale_out.md

@@ -58,7 +58,7 @@ Difyは`テキスト`、`リンク`、`画像`、`ファイルBLOB`、`JSON`な
 画像、音声、動画、PPT、Word、Excelなどのファイルの生データを返す必要がある場合は、以下のインターフェースを使用できます。
 
 - `blob` ファイルの生データ（bytes型）
-- `meta` ファイルのメタデータ。ファイルの種類が分かっている場合は、`mime_type`を渡すことをお勧めします。そうでない場合、Difyはデフォルトタイプとして`octet/stream`を使用します。
+- `meta` ファイルのメタデータ。ファイルの種類が分かっている場合は、`mime_type`を渡すことをお勧めします。そうでない場合、Difyはデフォルトタイプとして`application/octet-stream`を使用します。
 
 ```python
     def create_blob_message(self, blob: bytes, meta: dict = None, save_as: str = '') -> ToolInvokeMessage:

api/core/tools/docs/zh_Hans/advanced_scale_out.md

@@ -58,7 +58,7 @@ Dify支持`文本` `链接` `图片` `文件BLOB` `JSON` 等多种消息类型
 如果你需要返回文件的原始数据，如图片、音频、视频、PPT、Word、Excel等，可以使用以下接口。
 
 - `blob` 文件的原始数据，bytes类型
-- `meta` 文件的元数据，如果你知道该文件的类型，最好传递一个`mime_type`，否则Dify将使用`octet/stream`作为默认类型
+- `meta` 文件的元数据，如果你知道该文件的类型，最好传递一个`mime_type`，否则Dify将使用`application/octet-stream`作为默认类型
 
 ```python
     def create_blob_message(self, blob: bytes, meta: dict = None, save_as: str = '') -> ToolInvokeMessage:

api/core/tools/tool_engine.py

@@ -290,14 +290,16 @@ class ToolEngine:
                     raise ValueError("missing meta data")
 
                 yield ToolInvokeMessageBinary(
-                    mimetype=response.meta.get("mime_type", "octet/stream"),
+                    mimetype=response.meta.get("mime_type", "application/octet-stream"),
                     url=cast(ToolInvokeMessage.TextMessage, response.message).text,
                 )
             elif response.type == ToolInvokeMessage.MessageType.LINK:
                 # check if there is a mime type in meta
                 if response.meta and "mime_type" in response.meta:
                     yield ToolInvokeMessageBinary(
-                        mimetype=response.meta.get("mime_type", "octet/stream") if response.meta else "octet/stream",
+                        mimetype=response.meta.get("mime_type", "application/octet-stream")
+                        if response.meta
+                        else "application/octet-stream",
                         url=cast(ToolInvokeMessage.TextMessage, response.message).text,
                     )
 

api/core/tools/tool_file_manager.py

@@ -101,7 +101,7 @@ class ToolFileManager:
         except httpx.TimeoutException:
             raise ValueError(f"timeout when downloading file from {file_url}")
 
-        mimetype = guess_type(file_url)[0] or "octet/stream"
+        mimetype = guess_type(file_url)[0] or "application/octet-stream"
         extension = guess_extension(mimetype) or ".bin"
         unique_name = uuid4().hex
         filename = f"{unique_name}{extension}"

api/core/tools/utils/message_transformer.py

@@ -58,7 +58,7 @@ class ToolFileMessageTransformer:
                 # get mime type and save blob to storage
                 meta = message.meta or {}
 
-                mimetype = meta.get("mime_type", "octet/stream")
+                mimetype = meta.get("mime_type", "application/octet-stream")
                 # if message is str, encode it to bytes
 
                 if not isinstance(message.message, ToolInvokeMessage.BlobMessage):

api/core/variables/segments.py

@@ -136,7 +136,7 @@ class ArrayStringSegment(ArraySegment):
 
     @property
     def text(self) -> str:
-        return json.dumps(self.value)
+        return json.dumps(self.value, ensure_ascii=False)
 
 
 class ArrayNumberSegment(ArraySegment):

api/core/workflow/nodes/knowledge_retrieval/entities.py

@@ -79,12 +79,12 @@ SupportedComparisonOperator = Literal[
     # for string or array
     "contains",
     "not contains",
-    "starts with",
-    "ends with",
+    "start with",
+    "end with",
     "is",
     "is not",
     "empty",
-    "is not empty",
+    "not empty",
     # for number
     "=",
     "≠",

api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py

@@ -214,6 +214,7 @@ class KnowledgeRetrievalNode(LLMNode):
                 reranking_model=reranking_model,
                 weights=weights,
                 reranking_enable=node_data.multiple_retrieval_config.reranking_enable,
+                metadata_filter_document_ids=metadata_filter_document_ids,
             )
         dify_documents = [item for item in all_documents if item.provider == "dify"]
         external_documents = [item for item in all_documents if item.provider == "external"]
@@ -296,18 +297,20 @@ class KnowledgeRetrievalNode(LLMNode):
             automatic_metadata_filters = self._automatic_metadata_filter_func(dataset_ids, query, node_data)
             if automatic_metadata_filters:
                 for filter in automatic_metadata_filters:
-                    self._process_metadata_filter_func(
+                    document_query = self._process_metadata_filter_func(
                         filter.get("condition"), filter.get("metadata_name"), filter.get("value"), document_query
                     )
         elif node_data.metadata_filtering_mode == "manual":
-            for condition in node_data.metadata_filtering_conditions.conditions:
-                metadata_name = condition.name
-                expected_value = condition.value
-                if isinstance(expected_value, str):
-                    expected_value = self.graph_runtime_state.variable_pool.convert_template(expected_value).text
-                self._process_metadata_filter_func(
-                    condition.comparison_operator, metadata_name, expected_value, document_query
-                )
+            if node_data.metadata_filtering_conditions:
+                for condition in node_data.metadata_filtering_conditions.conditions:
+                    metadata_name = condition.name
+                    expected_value = condition.value
+                    if expected_value:
+                        if isinstance(expected_value, str):
+                            expected_value = self.graph_runtime_state.variable_pool.convert_template(expected_value).text
+                        document_query = self._process_metadata_filter_func(
+                            condition.comparison_operator, metadata_name, expected_value, document_query
+                        )
         else:
             raise ValueError("Invalid metadata filtering mode")
         documnents = document_query.all()
@@ -383,17 +386,23 @@ class KnowledgeRetrievalNode(LLMNode):
     def _process_metadata_filter_func(self, condition: str, metadata_name: str, value: str, query):
         match condition:
             case "contains":
-                query = query.filter(Document.doc_metadata[metadata_name].like(f"%{value}%"))
+                query = query.filter(Document.doc_metadata[metadata_name].like(f'"%{value}%"'))
             case "not contains":
-                query = query.filter(Document.doc_metadata[metadata_name].notlike(f"%{value}%"))
+                query = query.filter(Document.doc_metadata[metadata_name].notlike(f'"%{value}%"'))
             case "start with":
-                query = query.filter(Document.doc_metadata[metadata_name].like(f"{value}%"))
+                query = query.filter(Document.doc_metadata[metadata_name].like(f'"{value}%"'))
             case "end with":
-                query = query.filter(Document.doc_metadata[metadata_name].like(f"%{value}"))
+                query = query.filter(Document.doc_metadata[metadata_name].like(f'"%{value}"'))
             case "=" | "is":
-                query = query.filter(Document.doc_metadata[metadata_name] == value)
+                if isinstance(value, str):
+                    query = query.filter(Document.doc_metadata[metadata_name] == f'"{value}"')
+                else:
+                    query = query.filter(Document.doc_metadata[metadata_name] == value)
             case "is not" | "≠":
-                query = query.filter(Document.doc_metadata[metadata_name] != value)
+                if isinstance(value, str):
+                    query = query.filter(Document.doc_metadata[metadata_name] != f'"{value}"')
+                else:
+                    query = query.filter(Document.doc_metadata[metadata_name] != value)
             case "is empty":
                 query = query.filter(Document.doc_metadata[metadata_name].is_(None))
             case "is not empty":
@@ -408,7 +417,7 @@ class KnowledgeRetrievalNode(LLMNode):
                 query = query.filter(Document.doc_metadata[metadata_name] >= value)
             case _:
                 pass
-
+        return query
     @classmethod
     def _extract_variable_selector_to_variable_mapping(
         cls,

api/factories/file_factory.py

@@ -7,7 +7,7 @@ import httpx
 from sqlalchemy import select
 
 from constants import AUDIO_EXTENSIONS, DOCUMENT_EXTENSIONS, IMAGE_EXTENSIONS, VIDEO_EXTENSIONS
-from core.file import File, FileBelongsTo, FileTransferMethod, FileType, FileUploadConfig
+from core.file import File, FileBelongsTo, FileTransferMethod, FileType, FileUploadConfig, helpers
 from core.helper import ssrf_proxy
 from extensions.ext_database import db
 from models import MessageFile, ToolFile, UploadFile
@@ -158,6 +158,39 @@ def _build_from_remote_url(
     tenant_id: str,
     transfer_method: FileTransferMethod,
 ) -> File:
+    upload_file_id = mapping.get("upload_file_id")
+    if upload_file_id:
+        try:
+            uuid.UUID(upload_file_id)
+        except ValueError:
+            raise ValueError("Invalid upload file id format")
+        stmt = select(UploadFile).where(
+            UploadFile.id == upload_file_id,
+            UploadFile.tenant_id == tenant_id,
+        )
+
+        upload_file = db.session.scalar(stmt)
+        if upload_file is None:
+            raise ValueError("Invalid upload file")
+
+        file_type = FileType(mapping.get("type", "custom"))
+        file_type = _standardize_file_type(
+            file_type, extension="." + upload_file.extension, mime_type=upload_file.mime_type
+        )
+
+        return File(
+            id=mapping.get("id"),
+            filename=upload_file.name,
+            extension="." + upload_file.extension,
+            mime_type=upload_file.mime_type,
+            tenant_id=tenant_id,
+            type=file_type,
+            transfer_method=transfer_method,
+            remote_url=helpers.get_signed_file_url(upload_file_id=str(upload_file_id)),
+            related_id=mapping.get("upload_file_id"),
+            size=upload_file.size,
+            storage_key=upload_file.key,
+        )
     url = mapping.get("url") or mapping.get("remote_url")
     if not url:
         raise ValueError("Invalid file url")

api/fields/workflow_app_log_fields.py

@@ -17,8 +17,8 @@ workflow_app_log_partial_fields = {
 
 workflow_app_log_pagination_fields = {
     "page": fields.Integer,
-    "limit": fields.Integer(attribute="per_page"),
+    "limit": fields.Integer,
     "total": fields.Integer,
-    "has_more": fields.Boolean(attribute="has_next"),
-    "data": fields.List(fields.Nested(workflow_app_log_partial_fields), attribute="items"),
+    "has_more": fields.Boolean,
+    "data": fields.List(fields.Nested(workflow_app_log_partial_fields)),
 }

api/libs/login.py

@@ -77,7 +77,7 @@ def login_required(func):
                             )
                             if tenant_account_join:
                                 tenant, ta = tenant_account_join
-                                account = Account.query.filter_by(id=ta.account_id).first()
+                                account = db.session.query(Account).filter_by(id=ta.account_id).first()
                                 # Login admin
                                 if account:
                                     account.current_tenant = tenant

api/models/model.py

@@ -1081,19 +1081,19 @@ class Message(db.Model):  # type: ignore[name-defined]
 
         files = []
         for message_file in message_files:
-            if message_file.transfer_method == "local_file":
+            if message_file.transfer_method == FileTransferMethod.LOCAL_FILE.value:
                 if message_file.upload_file_id is None:
                     raise ValueError(f"MessageFile {message_file.id} is a local file but has no upload_file_id")
                 file = file_factory.build_from_mapping(
                     mapping={
                         "id": message_file.id,
-                        "upload_file_id": message_file.upload_file_id,
-                        "transfer_method": message_file.transfer_method,
                         "type": message_file.type,
+                        "transfer_method": message_file.transfer_method,
+                        "upload_file_id": message_file.upload_file_id,
                     },
                     tenant_id=current_app.tenant_id,
                 )
-            elif message_file.transfer_method == "remote_url":
+            elif message_file.transfer_method == FileTransferMethod.REMOTE_URL.value:
                 if message_file.url is None:
                     raise ValueError(f"MessageFile {message_file.id} is a remote url but has no url")
                 file = file_factory.build_from_mapping(
@@ -1101,11 +1101,12 @@ class Message(db.Model):  # type: ignore[name-defined]
                         "id": message_file.id,
                         "type": message_file.type,
                         "transfer_method": message_file.transfer_method,
+                        "upload_file_id": message_file.upload_file_id,
                         "url": message_file.url,
                     },
                     tenant_id=current_app.tenant_id,
                 )
-            elif message_file.transfer_method == "tool_file":
+            elif message_file.transfer_method == FileTransferMethod.TOOL_FILE.value:
                 if message_file.upload_file_id is None:
                     assert message_file.url is not None
                     message_file.upload_file_id = message_file.url.split("/")[-1].split(".")[0]

api/models/workflow.py

@@ -354,19 +354,6 @@ class WorkflowRunStatus(StrEnum):
     STOPPED = "stopped"
     PARTIAL_SUCCESSED = "partial-succeeded"
 
-    @classmethod
-    def value_of(cls, value: str) -> "WorkflowRunStatus":
-        """
-        Get value of given mode.
-
-        :param value: mode value
-        :return: mode
-        """
-        for mode in cls:
-            if mode.value == value:
-                return mode
-        raise ValueError(f"invalid workflow run status value {value}")
-
 
 class WorkflowRun(Base):
     """

api/pyproject.toml

@@ -50,7 +50,7 @@ oci = "~2.135.1"
 openai = "~1.61.0"
 openpyxl = "~3.1.5"
 opik = "~1.3.4"
-pandas = { version = "~2.2.2", extras = ["performance", "excel", "output-formatting"] }
+pandas = { version = "~2.2.2", extras = ["performance", "excel"] }
 pandas-stubs = "~2.2.3.241009"
 psycogreen = "~1.0.2"
 psycopg2-binary = "~2.9.6"
@@ -151,7 +151,6 @@ pytest-benchmark = "~4.0.0"
 pytest-env = "~1.1.3"
 pytest-mock = "~3.14.0"
 types-beautifulsoup4 = "~4.12.0.20241020"
-types-deprecated = "~1.2.15.20250304"
 types-flask-cors = "~5.0.0.20240902"
 types-flask-migrate = "~4.1.0.20250112"
 types-html5lib = "~1.1.11.20241018"
@@ -175,4 +174,4 @@ types-tqdm = "~4.67.0.20241221"
 optional = true
 [tool.poetry.group.lint.dependencies]
 dotenv-linter = "~0.5.0"
-ruff = "~0.9.9"
+ruff = "~0.9.2"

api/services/billing_service.py

@@ -6,7 +6,7 @@ from tenacity import retry, retry_if_exception_type, stop_before_delay, wait_fix
 
 from extensions.ext_database import db
 from libs.helper import RateLimiter
-from models.account import TenantAccountJoin, TenantAccountRole
+from models.account import Account, TenantAccountJoin, TenantAccountRole
 
 
 class BillingService:
@@ -95,6 +95,47 @@ class BillingService:
         json = {"email": email, "feedback": feedback}
         return cls._send_request("POST", "/account/delete-feedback", json=json)
 
+    class EducationIdentity:
+        verification_rate_limit = RateLimiter(prefix="edu_verification_rate_limit", max_attempts=10, time_window=60)
+        activation_rate_limit = RateLimiter(prefix="edu_activation_rate_limit", max_attempts=10, time_window=60)
+
+        @classmethod
+        def verify(cls, account_id: str, account_email: str):
+            if cls.verification_rate_limit.is_rate_limited(account_email):
+                from controllers.console.error import EducationVerifyLimitError
+
+                raise EducationVerifyLimitError()
+
+            cls.verification_rate_limit.increment_rate_limit(account_email)
+
+            params = {"account_id": account_id}
+            return BillingService._send_request("GET", "/education/verify", params=params)
+
+        @classmethod
+        def is_active(cls, account_id: str):
+            params = {"account_id": account_id}
+            return BillingService._send_request("GET", "/education/status", params=params)
+
+        @classmethod
+        def activate(cls, account: Account, token: str, institution: str):
+            if cls.activation_rate_limit.is_rate_limited(account.email):
+                from controllers.console.error import EducationActivateLimitError
+
+                raise EducationActivateLimitError()
+
+            cls.activation_rate_limit.increment_rate_limit(account.email)
+            json = {
+                "account_id": account.id,
+                "institution": institution,
+                "token": token,
+            }
+            return BillingService._send_request("POST", "/education/", json=json)
+
+        @classmethod
+        def autocomplete(cls, keywords: str, page: int = 0, limit: int = 20):
+            params = {"keywords": keywords, "page": page, "limit": limit}
+            return BillingService._send_request("GET", "/education/autocomplete", params=params)
+
     @classmethod
     def get_compliance_download_link(
         cls,

api/services/feature_service.py

@@ -10,6 +10,7 @@ from services.enterprise.enterprise_service import EnterpriseService
 class SubscriptionModel(BaseModel):
     plan: str = "sandbox"
     interval: str = ""
+    education: bool = False
 
 
 class BillingModel(BaseModel):
@@ -119,6 +120,7 @@ class FeatureService:
         features.billing.enabled = billing_info["enabled"]
         features.billing.subscription.plan = billing_info["subscription"]["plan"]
         features.billing.subscription.interval = billing_info["subscription"]["interval"]
+        features.billing.subscription.education = billing_info["subscription"]["education"] if "education" in billing_info["subscription"] else False
 
         if "members" in billing_info:
             features.members.size = billing_info["members"]["size"]

api/services/workflow_app_service.py

@@ -1,30 +1,46 @@
 import uuid
+from datetime import datetime
 
-from flask_sqlalchemy.pagination import Pagination
-from sqlalchemy import and_, or_
+from sqlalchemy import and_, func, or_, select
+from sqlalchemy.orm import Session
 
-from extensions.ext_database import db
 from models import App, EndUser, WorkflowAppLog, WorkflowRun
 from models.enums import CreatedByRole
 from models.workflow import WorkflowRunStatus
 
 
 class WorkflowAppService:
-    def get_paginate_workflow_app_logs(self, app_model: App, args: dict) -> Pagination:
+    def get_paginate_workflow_app_logs(
+        self,
+        *,
+        session: Session,
+        app_model: App,
+        keyword: str | None = None,
+        status: WorkflowRunStatus | None = None,
+        created_at_before: datetime | None = None,
+        created_at_after: datetime | None = None,
+        page: int = 1,
+        limit: int = 20,
+    ) -> dict:
         """
-        Get paginate workflow app logs
-        :param app: app model
-        :param args: request args
-        :return:
+        Get paginate workflow app logs using SQLAlchemy 2.0 style
+        :param session: SQLAlchemy session
+        :param app_model: app model
+        :param keyword: search keyword
+        :param status: filter by status
+        :param created_at_before: filter logs created before this timestamp
+        :param created_at_after: filter logs created after this timestamp
+        :param page: page number
+        :param limit: items per page
+        :return: Pagination object
         """
-        query = db.select(WorkflowAppLog).where(
+        # Build base statement using SQLAlchemy 2.0 style
+        stmt = select(WorkflowAppLog).where(
             WorkflowAppLog.tenant_id == app_model.tenant_id, WorkflowAppLog.app_id == app_model.id
         )
 
-        status = WorkflowRunStatus.value_of(args.get("status", "")) if args.get("status") else None
-        keyword = args["keyword"]
         if keyword or status:
-            query = query.join(WorkflowRun, WorkflowRun.id == WorkflowAppLog.workflow_run_id)
+            stmt = stmt.join(WorkflowRun, WorkflowRun.id == WorkflowAppLog.workflow_run_id)
 
         if keyword:
             keyword_like_val = f"%{keyword[:30].encode('unicode_escape').decode('utf-8')}%".replace(r"\u", r"\\u")
@@ -40,20 +56,40 @@ class WorkflowAppService:
             if keyword_uuid:
                 keyword_conditions.append(WorkflowRun.id == keyword_uuid)
 
-            query = query.outerjoin(
+            stmt = stmt.outerjoin(
                 EndUser,
                 and_(WorkflowRun.created_by == EndUser.id, WorkflowRun.created_by_role == CreatedByRole.END_USER),
-            ).filter(or_(*keyword_conditions))
+            ).where(or_(*keyword_conditions))
 
         if status:
-            # join with workflow_run and filter by status
-            query = query.filter(WorkflowRun.status == status.value)
+            stmt = stmt.where(WorkflowRun.status == status)
 
-        query = query.order_by(WorkflowAppLog.created_at.desc())
+        # Add time-based filtering
+        if created_at_before:
+            stmt = stmt.where(WorkflowAppLog.created_at <= created_at_before)
 
-        pagination = db.paginate(query, page=args["page"], per_page=args["limit"], error_out=False)
+        if created_at_after:
+            stmt = stmt.where(WorkflowAppLog.created_at >= created_at_after)
 
-        return pagination
+        stmt = stmt.order_by(WorkflowAppLog.created_at.desc())
+
+        # Get total count using the same filters
+        count_stmt = select(func.count()).select_from(stmt.subquery())
+        total = session.scalar(count_stmt) or 0
+
+        # Apply pagination limits
+        offset_stmt = stmt.offset((page - 1) * limit).limit(limit)
+
+        # Execute query and get items
+        items = list(session.scalars(offset_stmt).all())
+
+        return {
+            "page": page,
+            "limit": limit,
+            "total": total,
+            "has_more": total > page * limit,
+            "data": items,
+        }
 
     @staticmethod
     def _safe_parse_uuid(value: str):

docker/.env.example

@@ -344,7 +344,7 @@ TENCENT_COS_SCHEME=your-scheme
 
 # Oracle Storage Configuration
 #
-OCI_ENDPOINT=https://objectstorage.us-ashburn-1.oraclecloud.com
+OCI_ENDPOINT=https://your-object-storage-namespace.compat.objectstorage.us-ashburn-1.oraclecloud.com
 OCI_BUCKET_NAME=your-bucket-name
 OCI_ACCESS_KEY=your-access-key
 OCI_SECRET_KEY=your-secret-key
@@ -968,3 +968,6 @@ MARKETPLACE_ENABLED=true
 MARKETPLACE_API_URL=https://marketplace.dify.ai
 
 FORCE_VERIFYING_SIGNATURE=true
+
+PLUGIN_PYTHON_ENV_INIT_TIMEOUT=120
+PLUGIN_MAX_EXECUTION_TIMEOUT=600

docker/docker-compose-template.yaml

@@ -149,6 +149,8 @@ services:
       PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_DEBUGGING_PORT:-5003}
       PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd}
       FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true}
+      PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120}
+      PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600}
     ports:
       - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}"
     volumes:

docker/docker-compose.middleware.yaml

@@ -88,6 +88,8 @@ services:
       PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_DEBUGGING_PORT:-5003}
       PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd}
       FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true}
+      PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120}
+      PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600}
     ports:
       - "${EXPOSE_PLUGIN_DAEMON_PORT:-5002}:${PLUGIN_DAEMON_PORT:-5002}"
       - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}"

docker/docker-compose.yaml

@@ -105,7 +105,7 @@ x-shared-env: &shared-api-worker-env
   TENCENT_COS_SECRET_ID: ${TENCENT_COS_SECRET_ID:-your-secret-id}
   TENCENT_COS_REGION: ${TENCENT_COS_REGION:-your-region}
   TENCENT_COS_SCHEME: ${TENCENT_COS_SCHEME:-your-scheme}
-  OCI_ENDPOINT: ${OCI_ENDPOINT:-https://objectstorage.us-ashburn-1.oraclecloud.com}
+  OCI_ENDPOINT: ${OCI_ENDPOINT:-https://your-object-storage-namespace.compat.objectstorage.us-ashburn-1.oraclecloud.com}
   OCI_BUCKET_NAME: ${OCI_BUCKET_NAME:-your-bucket-name}
   OCI_ACCESS_KEY: ${OCI_ACCESS_KEY:-your-access-key}
   OCI_SECRET_KEY: ${OCI_SECRET_KEY:-your-secret-key}
@@ -413,6 +413,8 @@ x-shared-env: &shared-api-worker-env
   MARKETPLACE_ENABLED: ${MARKETPLACE_ENABLED:-true}
   MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai}
   FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true}
+  PLUGIN_PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120}
+  PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600}
 
 services:
   # API service
@@ -564,6 +566,8 @@ services:
       PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_DEBUGGING_PORT:-5003}
       PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd}
       FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true}
+      PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120}
+      PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600}
     ports:
       - "${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003}"
     volumes:

docker/middleware.env.example

@@ -114,4 +114,7 @@ PLUGIN_DIFY_INNER_API_URL=http://api:5001
 MARKETPLACE_ENABLED=true
 MARKETPLACE_API_URL=https://marketplace.dify.ai
 
-FORCE_VERIFYING_SIGNATURE=true
+FORCE_VERIFYING_SIGNATURE=true
+
+PLUGIN_PYTHON_ENV_INIT_TIMEOUT=120
+PLUGIN_MAX_EXECUTION_TIMEOUT=600

docker/nginx/conf.d/default.conf.template

@@ -4,6 +4,19 @@ server {
     listen ${NGINX_PORT};
     server_name ${NGINX_SERVER_NAME};
 
+    # Rule 1: Handle application entry points (preserve /app/{id})
+    location ~ ^/app/[a-f0-9-]+$ {
+      proxy_pass http://api:5001;
+      include proxy.conf;
+    }
+
+    # Rule 2: Handle static resource requests (remove /app/{id} prefix)
+    location ~ ^/app/[a-f0-9-]+/(console/api/.*)$ {
+      rewrite ^/app/[a-f0-9-]+/(.*)$ /$1 break;
+      proxy_pass http://api:5001;
+      include proxy.conf;
+    }
+
     location /console/api {
       proxy_pass http://api:5001;
       include proxy.conf;

web/app/(commonLayout)/datasets/Datasets.tsx

@@ -1,6 +1,6 @@
 'use client'
 
-import { useEffect, useRef } from 'react'
+import { useCallback, useEffect, useRef } from 'react'
 import useSWRInfinite from 'swr/infinite'
 import { debounce } from 'lodash-es'
 import { useTranslation } from 'react-i18next'
@@ -62,21 +62,28 @@ const Datasets = ({
   useEffect(() => {
     loadingStateRef.current = isLoading
     document.title = `${t('dataset.knowledge')} - Dify`
-  }, [isLoading])
+  }, [isLoading, t])
 
-  useEffect(() => {
-    const onScroll = debounce(() => {
-      if (!loadingStateRef.current) {
-        const { scrollTop, clientHeight } = containerRef.current!
-        const anchorOffset = anchorRef.current!.offsetTop
+  const onScroll = useCallback(
+    debounce(() => {
+      if (!loadingStateRef.current && containerRef.current && anchorRef.current) {
+        const { scrollTop, clientHeight } = containerRef.current
+        const anchorOffset = anchorRef.current.offsetTop
         if (anchorOffset - scrollTop - clientHeight < 100)
           setSize(size => size + 1)
       }
-    }, 50)
+    }, 50),
+    [setSize],
+  )
 
-    containerRef.current?.addEventListener('scroll', onScroll)
-    return () => containerRef.current?.removeEventListener('scroll', onScroll)
-  }, [])
+  useEffect(() => {
+    const currentContainer = containerRef.current
+    currentContainer?.addEventListener('scroll', onScroll)
+    return () => {
+      currentContainer?.removeEventListener('scroll', onScroll)
+      onScroll.cancel()
+    }
+  }, [onScroll])
 
   return (
     <nav className='grid content-start grid-cols-1 gap-4 px-12 pt-2 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4 grow shrink-0'>

web/app/components/app/configuration/dataset-config/index.tsx

@@ -4,6 +4,7 @@ import React, { useCallback, useMemo } from 'react'
 import { useTranslation } from 'react-i18next'
 import { useContext } from 'use-context-selector'
 import produce from 'immer'
+import { v4 as uuid4 } from 'uuid'
 import { useFormattingChangedDispatcher } from '../debug/hooks'
 import FeaturePanel from '../base/feature-panel'
 import OperationBtn from '../base/operation-btn'
@@ -150,6 +151,7 @@ const DatasetConfig: FC = () => {
       operator = ComparisonOperator.equal
 
     const newCondition = {
+      id: uuid4(),
       name,
       comparison_operator: operator,
     }
@@ -168,9 +170,9 @@ const DatasetConfig: FC = () => {
     setDatasetConfigs(newInputs)
   }, [setDatasetConfigs, datasetConfigsRef])
 
-  const handleRemoveCondition = useCallback<HandleRemoveCondition>((name) => {
+  const handleRemoveCondition = useCallback<HandleRemoveCondition>((id) => {
     const conditions = datasetConfigsRef.current!.metadata_filtering_conditions?.conditions || []
-    const index = conditions.findIndex(c => c.name === name)
+    const index = conditions.findIndex(c => c.id === id)
     const newInputs = produce(datasetConfigsRef.current!, (draft) => {
       if (index > -1)
         draft.metadata_filtering_conditions?.conditions.splice(index, 1)
@@ -178,10 +180,10 @@ const DatasetConfig: FC = () => {
     setDatasetConfigs(newInputs)
   }, [setDatasetConfigs, datasetConfigsRef])
 
-  const handleUpdateCondition = useCallback<HandleUpdateCondition>((name, newCondition) => {
+  const handleUpdateCondition = useCallback<HandleUpdateCondition>((id, newCondition) => {
     console.log(newCondition, 'newCondition')
     const conditions = datasetConfigsRef.current!.metadata_filtering_conditions?.conditions || []
-    const index = conditions.findIndex(c => c.name === name)
+    const index = conditions.findIndex(c => c.id === id)
     const newInputs = produce(datasetConfigsRef.current!, (draft) => {
       if (index > -1)
         draft.metadata_filtering_conditions!.conditions[index] = newCondition
@@ -256,6 +258,7 @@ const DatasetConfig: FC = () => {
       <div className='py-2 border-t border-t-divider-subtle'>
         <MetadataFilter
           metadataList={metadataList}
+          selectedDatasetsLoaded
           metadataFilterMode={datasetConfigs.metadata_filtering_mode}
           metadataFilteringConditions={datasetConfigs.metadata_filtering_conditions}
           handleAddCondition={handleAddCondition}

web/app/components/app/configuration/index.tsx

@@ -630,13 +630,14 @@ const Configuration: FC = () => {
               tools: modelConfig.agent_mode?.tools.filter((tool: any) => {
                 return !tool.dataset
               }).map((tool: any) => {
+                const toolInCollectionList = collectionList.find(c => tool.provider_id === c.id)
                 return {
                   ...tool,
                   isDeleted: res.deleted_tools?.some((deletedTool: any) => deletedTool.id === tool.id && deletedTool.tool_name === tool.tool_name),
-                  notAuthor: collectionList.find(c => tool.provider_id === c.id)?.is_team_authorization === false,
+                  notAuthor: toolInCollectionList?.is_team_authorization === false,
                   ...(tool.provider_type === 'builtin' ? {
-                    provider_id: correctToolProvider(tool.provider_name),
-                    provider_name: correctToolProvider(tool.provider_name),
+                    provider_id: correctToolProvider(tool.provider_name, !!toolInCollectionList),
+                    provider_name: correctToolProvider(tool.provider_name, !!toolInCollectionList),
                   } : {}),
                 }
               }),

web/app/components/app/workflow-log/filter.tsx

@@ -2,11 +2,29 @@
 import type { FC } from 'react'
 import React from 'react'
 import { useTranslation } from 'react-i18next'
+import dayjs from 'dayjs'
+import { RiCalendarLine } from '@remixicon/react'
+import quarterOfYear from 'dayjs/plugin/quarterOfYear'
 import type { QueryParam } from './index'
 import Chip from '@/app/components/base/chip'
 import Input from '@/app/components/base/input'
+dayjs.extend(quarterOfYear)
 
-interface IFilterProps {
+const today = dayjs()
+
+export const TIME_PERIOD_MAPPING: { [key: string]: { value: number; name: string } } = {
+  1: { value: 0, name: 'today' },
+  2: { value: 7, name: 'last7days' },
+  3: { value: 28, name: 'last4weeks' },
+  4: { value: today.diff(today.subtract(3, 'month'), 'day'), name: 'last3months' },
+  5: { value: today.diff(today.subtract(12, 'month'), 'day'), name: 'last12months' },
+  6: { value: today.diff(today.startOf('month'), 'day'), name: 'monthToDate' },
+  7: { value: today.diff(today.startOf('quarter'), 'day'), name: 'quarterToDate' },
+  8: { value: today.diff(today.startOf('year'), 'day'), name: 'yearToDate' },
+  9: { value: -1, name: 'allTime' },
+}
+
+type IFilterProps = {
   queryParams: QueryParam
   setQueryParams: (v: QueryParam) => void
 }
@@ -27,6 +45,17 @@ const Filter: FC<IFilterProps> = ({ queryParams, setQueryParams }: IFilterProps)
           { value: 'stopped', name: 'Stop' },
         ]}
       />
+      <Chip
+        className='min-w-[150px]'
+        panelClassName='w-[270px]'
+        leftIcon={<RiCalendarLine className='h-4 w-4 text-text-secondary' />}
+        value={queryParams.period}
+        onSelect={(item) => {
+          setQueryParams({ ...queryParams, period: item.value })
+        }}
+        onClear={() => setQueryParams({ ...queryParams, period: '9' })}
+        items={Object.entries(TIME_PERIOD_MAPPING).map(([k, v]) => ({ value: k, name: t(`appLog.filter.period.${v.name}`) }))}
+      />
       <Input
         wrapperClassName='w-[200px]'
         showLeftIcon

web/app/components/app/workflow-log/index.tsx

@@ -4,21 +4,30 @@ import React, { useState } from 'react'
 import useSWR from 'swr'
 import { usePathname } from 'next/navigation'
 import { useDebounce } from 'ahooks'
+import { omit } from 'lodash-es'
+import dayjs from 'dayjs'
+import utc from 'dayjs/plugin/utc'
+import timezone from 'dayjs/plugin/timezone'
 import { Trans, useTranslation } from 'react-i18next'
 import Link from 'next/link'
 import List from './list'
-import Filter from './filter'
+import Filter, { TIME_PERIOD_MAPPING } from './filter'
 import Pagination from '@/app/components/base/pagination'
 import Loading from '@/app/components/base/loading'
 import { fetchWorkflowLogs } from '@/service/log'
 import { APP_PAGE_LIMIT } from '@/config'
 import type { App, AppMode } from '@/types/app'
+import { useAppContext } from '@/context/app-context'
+
+dayjs.extend(utc)
+dayjs.extend(timezone)
 
 export type ILogsProps = {
   appDetail: App
 }
 
 export type QueryParam = {
+  period: string
   status?: string
   keyword?: string
 }
@@ -48,7 +57,8 @@ const EmptyElement: FC<{ appUrl: string }> = ({ appUrl }) => {
 
 const Logs: FC<ILogsProps> = ({ appDetail }) => {
   const { t } = useTranslation()
-  const [queryParams, setQueryParams] = useState<QueryParam>({ status: 'all' })
+  const { userProfile: { timezone } } = useAppContext()
+  const [queryParams, setQueryParams] = useState<QueryParam>({ status: 'all', period: '2' })
   const [currPage, setCurrPage] = React.useState<number>(0)
   const debouncedQueryParams = useDebounce(queryParams, { wait: 500 })
   const [limit, setLimit] = React.useState<number>(APP_PAGE_LIMIT)
@@ -58,6 +68,13 @@ const Logs: FC<ILogsProps> = ({ appDetail }) => {
     limit,
     ...(debouncedQueryParams.status !== 'all' ? { status: debouncedQueryParams.status } : {}),
     ...(debouncedQueryParams.keyword ? { keyword: debouncedQueryParams.keyword } : {}),
+    ...((debouncedQueryParams.period !== '9')
+      ? {
+        created_at__after: dayjs().subtract(TIME_PERIOD_MAPPING[debouncedQueryParams.period].value, 'day').startOf('day').tz(timezone).format('YYYY-MM-DDTHH:mm:ssZ'),
+        created_at__before: dayjs().endOf('day').tz(timezone).format('YYYY-MM-DDTHH:mm:ssZ'),
+      }
+      : {}),
+    ...omit(debouncedQueryParams, ['period', 'status']),
   }
 
   const getWebAppType = (appType: AppMode) => {

web/app/components/base/date-and-time-picker/date-picker/index.tsx

@@ -238,7 +238,7 @@ const DatePicker = ({
           </div>
         )}
       </PortalToFollowElemTrigger>
-      <PortalToFollowElemContent className='z-50'>
+      <PortalToFollowElemContent className={popupZIndexClassname}>
         <div className='w-[252px] mt-1 bg-components-panel-bg rounded-xl shadow-lg shadow-shadow-shadow-5 border-[0.5px] border-components-panel-border'>
           {/* Header */}
           {view === ViewType.date ? (

web/app/components/base/image-gallery/style.module.css

@@ -1,8 +1,8 @@
 .item {
-  height: 200px;
+  max-height: 200px;
   margin-right: 8px;
   margin-bottom: 8px;
-  object-fit: cover;
+  object-fit: contain;
   object-position: center;
   border-radius: 8px;
   cursor: pointer;

web/app/components/tools/provider/detail.tsx

@@ -337,7 +337,7 @@ const ProviderDetail = ({
           {/* Custom type */}
           {!isDetailLoading && (collection.type === CollectionType.custom) && (
             <div className='text-text-secondary system-sm-semibold-uppercase'>
-              <span className=''>{t('tools.includeToolNum', { num: toolList.length }).toLocaleUpperCase()}</span>
+              <span className=''>{t('tools.includeToolNum', { num: toolList.length, action: toolList.length > 1 ? 'actions' : 'action' }).toLocaleUpperCase()}</span>
             </div>
           )}
           {/* Workflow type */}

web/app/components/workflow/nodes/_base/components/output-vars.tsx

@@ -42,7 +42,7 @@ export const VarItem: FC<VarItemProps> = ({
     <div className='py-1'>
       <div className='flex leading-[18px] items-center'>
         <div className='code-sm-semibold text-text-secondary'>{name}</div>
-        <div className='ml-2 system-xs-regular text-text-tertiary'>{type}</div>
+        <div className='ml-2 system-xs-regular text-text-tertiary capitalize'>{type}</div>
       </div>
       <div className='mt-0.5 system-xs-regular text-text-tertiary'>
         {description}

web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/add-condition.tsx

@@ -68,7 +68,7 @@ const AddCondition = ({
             {
               filteredMetadataList?.map(metadata => (
                 <div
-                  key={metadata.id}
+                  key={metadata.name}
                   className='flex items-center px-3 h-6 rounded-md system-sm-medium text-text-secondary cursor-pointer hover:bg-state-base-hover'
                 >
                   <div className='mr-1 p-[1px]'>

web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-item.tsx

@@ -107,11 +107,11 @@ const ConditionItem = ({
 
   const handleValueMethodChange = useCallback((v: string) => {
     setLocalValueMethod(v)
-    onUpdateCondition?.(condition.name, { ...condition, value: undefined })
+    onUpdateCondition?.(condition.id, { ...condition, value: undefined })
   }, [condition, onUpdateCondition])
 
   const handleValueChange = useCallback((v: any) => {
-    onUpdateCondition?.(condition.name, { ...condition, value: v })
+    onUpdateCondition?.(condition.id, { ...condition, value: v })
   }, [condition, onUpdateCondition])
 
   return (

web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/condition-number.tsx

@@ -72,7 +72,7 @@ const ConditionNumber = ({
           <Input
             className='bg-transparent hover:bg-transparent outline-none border-none focus:shadow-none focus:bg-transparent'
             value={value}
-            onChange={e => onChange(e.target.value)}
+            onChange={e => onChange(Number(e.target.value))}
             placeholder={t('workflow.nodes.knowledgeRetrieval.metadata.panel.placeholder')}
             type='number'
           />

web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/condition-list/index.tsx

@@ -51,7 +51,7 @@ const ConditionList = ({
         {
           conditions.map(condition => (
             <ConditionItem
-              key={condition.name}
+              key={`${condition.id}`}
               disabled={disabled}
               condition={condition}
               onUpdateCondition={handleUpdateCondition}

web/app/components/workflow/nodes/knowledge-retrieval/components/metadata/metadata-trigger.tsx

@@ -2,6 +2,7 @@ import {
   useEffect,
   useState,
 } from 'react'
+import { unionBy } from 'lodash-es'
 import { useTranslation } from 'react-i18next'
 import { RiFilter3Line } from '@remixicon/react'
 import MetadataPanel from './metadata-panel'
@@ -15,21 +16,25 @@ import type { MetadataShape } from '@/app/components/workflow/nodes/knowledge-re
 
 const MetadataTrigger = ({
   metadataFilteringConditions,
-  metadataList = [],
+  metadataList: originalMetadataList = [],
   handleRemoveCondition,
+  selectedDatasetsLoaded,
   ...restProps
 }: MetadataShape) => {
   const { t } = useTranslation()
   const [open, setOpen] = useState(false)
   const conditions = metadataFilteringConditions?.conditions || []
 
+  const metadataList = unionBy(originalMetadataList, 'name').sort(a => a.id === 'built-in' ? 1 : -1)
   useEffect(() => {
-    conditions.forEach((condition) => {
-      if (!metadataList.find(metadata => metadata.name === condition.name))
-        handleRemoveCondition(condition.name)
-    })
+    if (selectedDatasetsLoaded) {
+      conditions.forEach((condition) => {
+        if (!metadataList.find(metadata => metadata.name === condition.name))
+          handleRemoveCondition(condition.id)
+      })
+    }
   // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [metadataList, handleRemoveCondition])
+  }, [metadataList, handleRemoveCondition, selectedDatasetsLoaded])
 
   return (
     <PortalToFollowElem

web/app/components/workflow/nodes/knowledge-retrieval/panel.tsx

@@ -37,6 +37,7 @@ const Panel: FC<NodePanelProps<KnowledgeRetrievalNodeType>> = ({
     handleRetrievalModeChange,
     handleMultipleRetrievalConfigChange,
     selectedDatasets,
+    selectedDatasetsLoaded,
     handleOnDatasetsChange,
     isShowSingleRun,
     hideSingleRun,
@@ -130,6 +131,7 @@ const Panel: FC<NodePanelProps<KnowledgeRetrievalNodeType>> = ({
       <div className='mb-2 py-2'>
         <MetadataFilter
           metadataList={metadataList}
+          selectedDatasetsLoaded={selectedDatasetsLoaded}
           metadataFilterMode={inputs.metadata_filtering_mode}
           metadataFilteringConditions={inputs.metadata_filtering_conditions}
           handleAddCondition={handleAddCondition}

web/app/components/workflow/nodes/knowledge-retrieval/types.ts

@@ -81,6 +81,7 @@ export enum MetadataFilteringVariableType {
 }
 
 export type MetadataFilteringCondition = {
+  id: string
   name: string
   comparison_operator: ComparisonOperator
   value?: string | number
@@ -104,12 +105,13 @@ export type KnowledgeRetrievalNodeType = CommonNodeType & {
 }
 
 export type HandleAddCondition = (metadataItem: MetadataInDoc) => void
-export type HandleRemoveCondition = (name: string) => void
-export type HandleUpdateCondition = (name: string, newCondition: MetadataFilteringCondition) => void
+export type HandleRemoveCondition = (id: string) => void
+export type HandleUpdateCondition = (id: string, newCondition: MetadataFilteringCondition) => void
 export type HandleToggleConditionLogicalOperator = () => void
 
 export type MetadataShape = {
   metadataList?: MetadataInDoc[]
+  selectedDatasetsLoaded?: boolean
   metadataFilteringConditions?: MetadataFilteringConditions
   handleAddCondition: HandleAddCondition
   handleRemoveCondition: HandleRemoveCondition

web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts

@@ -6,6 +6,7 @@ import {
 } from 'react'
 import produce from 'immer'
 import { isEqual } from 'lodash-es'
+import { v4 as uuid4 } from 'uuid'
 import type { ValueSelector, Var } from '../../types'
 import { BlockEnum, VarType } from '../../types'
 import {
@@ -211,6 +212,7 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
     setInputs(newInputs)
   }, [inputs, setInputs, selectedDatasets, currentRerankModel, currentRerankProvider])
 
+  const [selectedDatasetsLoaded, setSelectedDatasetsLoaded] = useState(false)
   // datasets
   useEffect(() => {
     (async () => {
@@ -225,6 +227,7 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
         draft._datasets = selectedDatasets
       })
       setInputs(newInputs)
+      setSelectedDatasetsLoaded(true)
     })()
   // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [])
@@ -315,6 +318,7 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
       operator = ComparisonOperator.equal
 
     const newCondition = {
+      id: uuid4(),
       name,
       comparison_operator: operator,
     }
@@ -333,9 +337,9 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
     setInputs(newInputs)
   }, [setInputs])
 
-  const handleRemoveCondition = useCallback<HandleRemoveCondition>((name) => {
+  const handleRemoveCondition = useCallback<HandleRemoveCondition>((id) => {
     const conditions = inputRef.current.metadata_filtering_conditions?.conditions || []
-    const index = conditions.findIndex(c => c.name === name)
+    const index = conditions.findIndex(c => c.id === id)
     const newInputs = produce(inputRef.current, (draft) => {
       if (index > -1)
         draft.metadata_filtering_conditions?.conditions.splice(index, 1)
@@ -343,9 +347,9 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
     setInputs(newInputs)
   }, [setInputs])
 
-  const handleUpdateCondition = useCallback<HandleUpdateCondition>((name, newCondition) => {
+  const handleUpdateCondition = useCallback<HandleUpdateCondition>((id, newCondition) => {
     const conditions = inputRef.current.metadata_filtering_conditions?.conditions || []
-    const index = conditions.findIndex(c => c.name === name)
+    const index = conditions.findIndex(c => c.id === id)
     const newInputs = produce(inputRef.current, (draft) => {
       if (index > -1)
         draft.metadata_filtering_conditions!.conditions[index] = newCondition
@@ -418,6 +422,7 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
     handleModelChanged,
     handleCompletionParamsChange,
     selectedDatasets: selectedDatasets.filter(d => d.name),
+    selectedDatasetsLoaded,
     handleOnDatasetsChange,
     isShowSingleRun,
     hideSingleRun,

web/app/components/workflow/nodes/knowledge-retrieval/utils.ts

@@ -154,7 +154,8 @@ export const getMultipleRetrievalConfig = (
     result.reranking_mode = RerankingModeEnum.RerankingModel
     if (!result.reranking_model?.provider || !result.reranking_model?.model) {
       if (rerankModelIsValid) {
-        result.reranking_enable = true
+        result.reranking_enable = reranking_enable !== false
+
         result.reranking_model = {
           provider: validRerankModel?.provider || '',
           model: validRerankModel?.model || '',
@@ -168,7 +169,7 @@ export const getMultipleRetrievalConfig = (
       }
     }
     else {
-      result.reranking_enable = true
+      result.reranking_enable = reranking_enable !== false
     }
   }
 
@@ -176,7 +177,8 @@ export const getMultipleRetrievalConfig = (
     if (!reranking_mode) {
       if (validRerankModel?.provider && validRerankModel?.model) {
         result.reranking_mode = RerankingModeEnum.RerankingModel
-        result.reranking_enable = true
+        result.reranking_enable = reranking_enable !== false
+
         result.reranking_model = {
           provider: validRerankModel.provider,
           model: validRerankModel.model,
@@ -194,7 +196,8 @@ export const getMultipleRetrievalConfig = (
     if (reranking_mode === RerankingModeEnum.WeightedScore && weights && shouldSetWeightDefaultValue) {
       if (rerankModelIsValid) {
         result.reranking_mode = RerankingModeEnum.RerankingModel
-        result.reranking_enable = true
+        result.reranking_enable = reranking_enable !== false
+
         result.reranking_model = {
           provider: validRerankModel.provider || '',
           model: validRerankModel.model || '',

web/i18n/ja-JP/dataset-creation.ts

@@ -122,7 +122,7 @@ const translation = {
     removeUrlEmails: 'すべてのURLとメールアドレスを削除する',
     removeStopwords: '「a」「an」「the」などのストップワードを削除する',
     preview: 'プレビュー',
-    previewChunk: 'チュンクをプレビュー',
+    previewChunk: 'チャンクをプレビュー',
     reset: 'リセット',
     indexMode: 'インデックス方法',
     qualified: '高品質',
@@ -168,7 +168,7 @@ const translation = {
     datasetSettingLink: 'ナレッジ設定',
     separatorTip: '区切り文字は、テキストを区切るために使用される文字です。\\n\\n と \\n は、段落と行を区切るために一般的に使用される区切り記号です。カンマ (\\n\\n,\\n) と組み合わせると、最大チャンク長を超えると、段落は行で区切られます。自分で定義した特別な区切り文字を使用することもできます(例:***)。',
     maxLengthCheck: 'チャンクの最大長は {{limit}} 未満にする必要があります',
-    previewChunkTip: 'プレビューを読み込むには、左側の \'チュンクをプレビュー\' ボタンをクリックしてください',
+    previewChunkTip: 'プレビューを読み込むには、左側の \'チャンクをプレビュー\' ボタンをクリックしてください',
     previewChunkCount: '推定チャンク数: {{count}}',
     switch: '切り替え',
     qaSwitchHighQualityTipTitle: 'Q&A形式には高品質なインデックスが必要です',

web/service/log.ts

@@ -17,7 +17,6 @@ import type {
   LogMessageAnnotationsResponse,
   LogMessageFeedbacksRequest,
   LogMessageFeedbacksResponse,
-  WorkflowLogsRequest,
   WorkflowLogsResponse,
   WorkflowRunDetailResponse,
 } from '@/models/log'
@@ -64,7 +63,7 @@ export const fetchAnnotationsCount: Fetcher<AnnotationsCountResponse, { url: str
   return get<AnnotationsCountResponse>(url)
 }
 
-export const fetchWorkflowLogs: Fetcher<WorkflowLogsResponse, { url: string; params?: WorkflowLogsRequest }> = ({ url, params }) => {
+export const fetchWorkflowLogs: Fetcher<WorkflowLogsResponse, { url: string; params: Record<string, any> }> = ({ url, params }) => {
   return get<WorkflowLogsResponse>(url, { params })
 }
 

web/utils/index.ts

@@ -69,10 +69,13 @@ export const correctModelProvider = (provider: string) => {
   return `langgenius/${provider}/${provider}`
 }
 
-export const correctToolProvider = (provider: string) => {
+export const correctToolProvider = (provider: string, toolInCollectionList?: boolean) => {
   if (!provider)
     return ''
 
+  if (toolInCollectionList)
+    return provider
+
   if (provider.includes('/'))
     return provider