feat: server multi models support (#799)

api/models/dataset.py

@@ -9,6 +9,7 @@ from extensions.ext_database import db
 from models.account import Account
 from models.model import App, UploadFile
 
+
 class Dataset(db.Model):
     __tablename__ = 'datasets'
     __table_args__ = (
@@ -268,7 +269,7 @@ class Document(db.Model):
     @property
     def average_segment_length(self):
         if self.word_count and self.word_count != 0 and self.segment_count and self.segment_count != 0:
-            return self.word_count//self.segment_count
+            return self.word_count // self.segment_count
         return 0
 
     @property
@@ -346,16 +347,6 @@ class DocumentSegment(db.Model):
     def document(self):
         return db.session.query(Document).filter(Document.id == self.document_id).first()
 
-    @property
-    def embedding(self):
-        embedding = db.session.query(Embedding).filter(Embedding.hash == self.index_node_hash).first() \
-            if self.index_node_hash else None
-
-        if embedding:
-            return embedding.embedding
-
-        return None
-
     @property
     def previous_segment(self):
         return db.session.query(DocumentSegment).filter(
@@ -436,10 +427,12 @@ class Embedding(db.Model):
     __tablename__ = 'embeddings'
     __table_args__ = (
         db.PrimaryKeyConstraint('id', name='embedding_pkey'),
-        db.UniqueConstraint('hash', name='embedding_hash_idx')
+        db.UniqueConstraint('model_name', 'hash', name='embedding_hash_idx')
     )
 
     id = db.Column(UUID, primary_key=True, server_default=db.text('uuid_generate_v4()'))
+    model_name = db.Column(db.String(40), nullable=False,
+                           server_default=db.text("'text-embedding-ada-002'::character varying"))
     hash = db.Column(db.String(64), nullable=False)
     embedding = db.Column(db.LargeBinary, nullable=False)
     created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
@@ -448,4 +441,4 @@ class Embedding(db.Model):
         self.embedding = pickle.dumps(embedding_data, protocol=pickle.HIGHEST_PROTOCOL)
 
     def get_embedding(self) -> list[float]:
-        return pickle.loads(self.embedding)
+        return pickle.loads(self.embedding)

api/models/provider.py

@@ -9,25 +9,30 @@ class ProviderType(Enum):
     CUSTOM = 'custom'
     SYSTEM = 'system'
 
-
-class ProviderName(Enum):
-    OPENAI = 'openai'
-    AZURE_OPENAI = 'azure_openai'
-    ANTHROPIC = 'anthropic'
-    COHERE = 'cohere'
-    HUGGINGFACEHUB = 'huggingfacehub'
-
     @staticmethod
     def value_of(value):
-        for member in ProviderName:
+        for member in ProviderType:
             if member.value == value:
                 return member
         raise ValueError(f"No matching enum found for value '{value}'")
 
 
 class ProviderQuotaType(Enum):
-    MONTHLY = 'monthly'
+    PAID = 'paid'
+    """hosted paid quota"""
+
+    FREE = 'free'
+    """third-party free quota"""
+
     TRIAL = 'trial'
+    """hosted trial quota"""
+
+    @staticmethod
+    def value_of(value):
+        for member in ProviderQuotaType:
+            if member.value == value:
+                return member
+        raise ValueError(f"No matching enum found for value '{value}'")
 
 
 class Provider(db.Model):
@@ -50,8 +55,8 @@ class Provider(db.Model):
     last_used = db.Column(db.DateTime, nullable=True)
 
     quota_type = db.Column(db.String(40), nullable=True, server_default=db.text("''::character varying"))
-    quota_limit = db.Column(db.Integer, nullable=True)
-    quota_used = db.Column(db.Integer, default=0)
+    quota_limit = db.Column(db.BigInteger, nullable=True)
+    quota_used = db.Column(db.BigInteger, default=0)
 
     created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
     updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
@@ -75,3 +80,96 @@ class Provider(db.Model):
             return self.is_valid
         else:
             return self.is_valid and self.token_is_set
+
+
+class ProviderModel(db.Model):
+    """
+    Provider model representing the API provider_models and their configurations.
+    """
+    __tablename__ = 'provider_models'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='provider_model_pkey'),
+        db.Index('provider_model_tenant_id_provider_idx', 'tenant_id', 'provider_name'),
+        db.UniqueConstraint('tenant_id', 'provider_name', 'model_name', 'model_type', name='unique_provider_model_name')
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    provider_name = db.Column(db.String(40), nullable=False)
+    model_name = db.Column(db.String(40), nullable=False)
+    model_type = db.Column(db.String(40), nullable=False)
+    encrypted_config = db.Column(db.Text, nullable=True)
+    is_valid = db.Column(db.Boolean, nullable=False, server_default=db.text('false'))
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+
+class TenantDefaultModel(db.Model):
+    __tablename__ = 'tenant_default_models'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='tenant_default_model_pkey'),
+        db.Index('tenant_default_model_tenant_id_provider_type_idx', 'tenant_id', 'provider_name', 'model_type'),
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    provider_name = db.Column(db.String(40), nullable=False)
+    model_name = db.Column(db.String(40), nullable=False)
+    model_type = db.Column(db.String(40), nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+
+class TenantPreferredModelProvider(db.Model):
+    __tablename__ = 'tenant_preferred_model_providers'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='tenant_preferred_model_provider_pkey'),
+        db.Index('tenant_preferred_model_provider_tenant_provider_idx', 'tenant_id', 'provider_name'),
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    provider_name = db.Column(db.String(40), nullable=False)
+    preferred_provider_type = db.Column(db.String(40), nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+
+
+class ProviderOrderPaymentStatus(Enum):
+    WAIT_PAY = 'wait_pay'
+    PAID = 'paid'
+    PAY_FAILED = 'pay_failed'
+    REFUNDED = 'refunded'
+
+    @staticmethod
+    def value_of(value):
+        for member in ProviderOrderPaymentStatus:
+            if member.value == value:
+                return member
+        raise ValueError(f"No matching enum found for value '{value}'")
+
+
+
+class ProviderOrder(db.Model):
+    __tablename__ = 'provider_orders'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='provider_order_pkey'),
+        db.Index('provider_order_tenant_provider_idx', 'tenant_id', 'provider_name'),
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    provider_name = db.Column(db.String(40), nullable=False)
+    account_id = db.Column(UUID, nullable=False)
+    payment_product_id = db.Column(db.String(191), nullable=False)
+    payment_id = db.Column(db.String(191))
+    transaction_id = db.Column(db.String(191))
+    quantity = db.Column(db.Integer, nullable=False, server_default=db.text('1'))
+    currency = db.Column(db.String(40))
+    total_amount = db.Column(db.Integer)
+    payment_status = db.Column(db.String(40), nullable=False, server_default=db.text("'wait_pay'::character varying"))
+    paid_at = db.Column(db.DateTime)
+    pay_failed_at = db.Column(db.DateTime)
+    refunded_at = db.Column(db.DateTime)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
+    updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))