feat: Add summary index for knowledge. (#31625)

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com> Co-authored-by: zxhlyh <jasonapring2015@outlook.com> Co-authored-by: Yansong Zhang <916125788@qq.com> Co-authored-by: hj24 <mambahj24@gmail.com> Co-authored-by: CodingOnStar <hanxujiang@dify.ai> Co-authored-by: CodingOnStar <hanxujiang@dify.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
2026-04-05 10:12:43 +08:00 · 2026-01-29 13:47:35 +08:00
parent 5ce3a04a2c
commit c2473d85dc
51 changed files with 3797 additions and 60 deletions
--- a/api/migrations/versions/2026_01_27_1815-788d3099ae3a_add_summary_index_feature.py
+++ b/api/migrations/versions/2026_01_27_1815-788d3099ae3a_add_summary_index_feature.py
@@ -0,0 +1,107 @@
+"""add summary index feature
+
+Revision ID: 788d3099ae3a
+Revises: 9d77545f524e
+Create Date: 2026-01-27 18:15:45.277928
+
+"""
+from alembic import op
+import models as models
+import sqlalchemy as sa
+
+def _is_pg(conn):
+    return conn.dialect.name == "postgresql"
+
+# revision identifiers, used by Alembic.
+revision = '788d3099ae3a'
+down_revision = '9d77545f524e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    conn = op.get_bind()
+    if _is_pg(conn):
+        op.create_table('document_segment_summaries',
+        sa.Column('id', models.types.StringUUID(), nullable=False),
+        sa.Column('dataset_id', models.types.StringUUID(), nullable=False),
+        sa.Column('document_id', models.types.StringUUID(), nullable=False),
+        sa.Column('chunk_id', models.types.StringUUID(), nullable=False),
+        sa.Column('summary_content', models.types.LongText(), nullable=True),
+        sa.Column('summary_index_node_id', sa.String(length=255), nullable=True),
+        sa.Column('summary_index_node_hash', sa.String(length=255), nullable=True),
+        sa.Column('tokens', sa.Integer(), nullable=True),
+        sa.Column('status', sa.String(length=32), server_default=sa.text("'generating'"), nullable=False),
+        sa.Column('error', models.types.LongText(), nullable=True),
+        sa.Column('enabled', sa.Boolean(), server_default=sa.text('true'), nullable=False),
+        sa.Column('disabled_at', sa.DateTime(), nullable=True),
+        sa.Column('disabled_by', models.types.StringUUID(), nullable=True),
+        sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+        sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+        sa.PrimaryKeyConstraint('id', name='document_segment_summaries_pkey')
+        )
+        with op.batch_alter_table('document_segment_summaries', schema=None) as batch_op:
+            batch_op.create_index('document_segment_summaries_chunk_id_idx', ['chunk_id'], unique=False)
+            batch_op.create_index('document_segment_summaries_dataset_id_idx', ['dataset_id'], unique=False)
+            batch_op.create_index('document_segment_summaries_document_id_idx', ['document_id'], unique=False)
+            batch_op.create_index('document_segment_summaries_status_idx', ['status'], unique=False)
+
+        with op.batch_alter_table('datasets', schema=None) as batch_op:
+            batch_op.add_column(sa.Column('summary_index_setting', models.types.AdjustedJSON(), nullable=True))
+
+        with op.batch_alter_table('documents', schema=None) as batch_op:
+            batch_op.add_column(sa.Column('need_summary', sa.Boolean(), server_default=sa.text('false'), nullable=True))
+    else:
+        # MySQL: Use compatible syntax
+        op.create_table(
+            'document_segment_summaries',
+            sa.Column('id', models.types.StringUUID(), nullable=False),
+            sa.Column('dataset_id', models.types.StringUUID(), nullable=False),
+            sa.Column('document_id', models.types.StringUUID(), nullable=False),
+            sa.Column('chunk_id', models.types.StringUUID(), nullable=False),
+            sa.Column('summary_content', models.types.LongText(), nullable=True),
+            sa.Column('summary_index_node_id', sa.String(length=255), nullable=True),
+            sa.Column('summary_index_node_hash', sa.String(length=255), nullable=True),
+            sa.Column('tokens', sa.Integer(), nullable=True),
+            sa.Column('status', sa.String(length=32), server_default=sa.text("'generating'"), nullable=False),
+            sa.Column('error', models.types.LongText(), nullable=True),
+            sa.Column('enabled', sa.Boolean(), server_default=sa.text('true'), nullable=False),
+            sa.Column('disabled_at', sa.DateTime(), nullable=True),
+            sa.Column('disabled_by', models.types.StringUUID(), nullable=True),
+            sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+            sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+            sa.PrimaryKeyConstraint('id', name='document_segment_summaries_pkey'),
+        )
+        with op.batch_alter_table('document_segment_summaries', schema=None) as batch_op:
+            batch_op.create_index('document_segment_summaries_chunk_id_idx', ['chunk_id'], unique=False)
+            batch_op.create_index('document_segment_summaries_dataset_id_idx', ['dataset_id'], unique=False)
+            batch_op.create_index('document_segment_summaries_document_id_idx', ['document_id'], unique=False)
+            batch_op.create_index('document_segment_summaries_status_idx', ['status'], unique=False)
+
+        with op.batch_alter_table('datasets', schema=None) as batch_op:
+            batch_op.add_column(sa.Column('summary_index_setting', models.types.AdjustedJSON(), nullable=True))
+
+        with op.batch_alter_table('documents', schema=None) as batch_op:
+            batch_op.add_column(sa.Column('need_summary', sa.Boolean(), server_default=sa.text('false'), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    with op.batch_alter_table('documents', schema=None) as batch_op:
+        batch_op.drop_column('need_summary')
+
+    with op.batch_alter_table('datasets', schema=None) as batch_op:
+        batch_op.drop_column('summary_index_setting')
+
+    with op.batch_alter_table('document_segment_summaries', schema=None) as batch_op:
+        batch_op.drop_index('document_segment_summaries_status_idx')
+        batch_op.drop_index('document_segment_summaries_document_id_idx')
+        batch_op.drop_index('document_segment_summaries_dataset_id_idx')
+        batch_op.drop_index('document_segment_summaries_chunk_id_idx')
+
+    op.drop_table('document_segment_summaries')
+    # ### end Alembic commands ###