Feat/dataset notion import (#392)

Co-authored-by: StyleZhang <jasonapring2015@outlook.com>
Co-authored-by: JzoNg <jzongcode@gmail.com>

web/app/components/datasets/create/step-two/index.module.css

@@ -14,9 +14,26 @@
 }
 
 .fixed {
+  padding-top: 12px;
+  font-size: 12px;
+  line-height: 18px;
   background: rgba(255, 255, 255, 0.9);
   border-bottom: 0.5px solid #EAECF0;
   backdrop-filter: blur(4px);
+  animation: fix 0.5s;
+}
+
+@keyframes fix {
+  from {
+    padding-top: 42px;
+    font-size: 18px;
+    line-height: 28px;
+  }
+  to {
+    padding-top: 12px;
+    font-size: 12px;
+    line-height: 18px;
+  }
 }
 
 .form {
@@ -273,11 +290,11 @@
   @apply bg-gray-100 caret-primary-600 hover:bg-gray-100 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none focus:bg-white placeholder:text-gray-400;
 }
 
-.file {
+.source {
   @apply flex justify-between items-center mt-8 px-6 py-4 rounded-xl bg-gray-50;
 }
 
-.file .divider {
+.source .divider {
   @apply shrink-0 mx-4 w-px bg-gray-200;
   height: 42px;
 }
@@ -318,9 +335,19 @@
 .fileIcon.json {
   background-image: url(../assets/json.svg);
 }
-
-.fileContent {
-  flex: 1 1 50%;
+.sourceContent {
+  flex: 1 1 auto;
+}
+.sourceCount {
+  @apply shrink-0 ml-1;
+  font-weight: 500;
+  font-size: 13px;
+  line-height: 18px;
+  color: #667085;
+}
+.segmentCount {
+  flex: 1 1 30%;
+  max-width: 120px;
 }
 
 .divider {

web/app/components/datasets/create/step-two/index.tsx

@@ -6,9 +6,10 @@ import { useBoolean } from 'ahooks'
 import { XMarkIcon } from '@heroicons/react/20/solid'
 import cn from 'classnames'
 import Link from 'next/link'
+import { groupBy } from 'lodash-es'
 import PreviewItem from './preview-item'
 import s from './index.module.css'
-import type { CreateDocumentReq, File, FullDocumentDetail, FileIndexingEstimateResponse as IndexingEstimateResponse, PreProcessingRule, Rules, createDocumentResponse } from '@/models/datasets'
+import type { CreateDocumentReq, File, FullDocumentDetail, FileIndexingEstimateResponse as IndexingEstimateResponse, NotionInfo, PreProcessingRule, Rules, createDocumentResponse } from '@/models/datasets'
 import {
   createDocument,
   createFirstDocument,
@@ -20,6 +21,11 @@ import Loading from '@/app/components/base/loading'
 
 import Toast from '@/app/components/base/toast'
 import { formatNumber } from '@/utils/format'
+import type { DataSourceNotionPage } from '@/models/common'
+import { DataSourceType } from '@/models/datasets'
+import NotionIcon from '@/app/components/base/notion-icon'
+
+type Page = DataSourceNotionPage & { workspace_id: string }
 
 type StepTwoProps = {
   isSetting?: boolean
@@ -28,7 +34,9 @@ type StepTwoProps = {
   onSetting: () => void
   datasetId?: string
   indexingType?: string
+  dataSourceType: DataSourceType
   file?: File
+  notionPages?: Page[]
   onStepChange?: (delta: number) => void
   updateIndexingTypeCache?: (type: string) => void
   updateResultCache?: (res: createDocumentResponse) => void
@@ -52,7 +60,9 @@ const StepTwo = ({
   onSetting,
   datasetId,
   indexingType,
+  dataSourceType,
   file,
+  notionPages = [],
   onStepChange,
   updateIndexingTypeCache,
   updateResultCache,
@@ -169,12 +179,54 @@ const StepTwo = ({
     return processRule
   }
 
+  const getNotionInfo = () => {
+    const workspacesMap = groupBy(notionPages, 'workspace_id')
+    const workspaces = Object.keys(workspacesMap).map((workspaceId) => {
+      return {
+        workspaceId,
+        pages: workspacesMap[workspaceId],
+      }
+    })
+    return workspaces.map((workspace) => {
+      return {
+        workspace_id: workspace.workspaceId,
+        pages: workspace.pages.map((page) => {
+          const { page_id, page_name, page_icon, type } = page
+          return {
+            page_id,
+            page_name,
+            page_icon,
+            type,
+          }
+        }),
+      }
+    }) as NotionInfo[]
+  }
+
   const getFileIndexingEstimateParams = () => {
-    const params = {
-      file_id: file?.id,
-      dataset_id: datasetId,
-      indexing_technique: getIndexing_technique(),
-      process_rule: getProcessRule(),
+    let params
+    if (dataSourceType === DataSourceType.FILE) {
+      params = {
+        info_list: {
+          data_source_type: dataSourceType,
+          file_info_list: {
+            // TODO multi files
+            file_ids: [file?.id || ''],
+          },
+        },
+        indexing_technique: getIndexing_technique(),
+        process_rule: getProcessRule(),
+      }
+    }
+    if (dataSourceType === DataSourceType.NOTION) {
+      params = {
+        info_list: {
+          data_source_type: dataSourceType,
+          notion_info_list: getNotionInfo(),
+        },
+        indexing_technique: getIndexing_technique(),
+        process_rule: getProcessRule(),
+      }
     }
     return params
   }
@@ -190,13 +242,22 @@ const StepTwo = ({
     else {
       params = {
         data_source: {
-          type: 'upload_file',
-          info: file?.id,
-          name: file?.name,
+          type: dataSourceType,
+          info_list: {
+            data_source_type: dataSourceType,
+          },
         },
         indexing_technique: getIndexing_technique(),
         process_rule: getProcessRule(),
       } as CreateDocumentReq
+      if (dataSourceType === DataSourceType.FILE) {
+        params.data_source.info_list.file_info_list = {
+          // TODO multi files
+          file_ids: [file?.id || ''],
+        }
+      }
+      if (dataSourceType === DataSourceType.NOTION)
+        params.data_source.info_list.notion_info_list = getNotionInfo()
     }
     return params
   }
@@ -249,9 +310,7 @@ const StepTwo = ({
           body: params,
         })
         updateIndexingTypeCache && updateIndexingTypeCache(indexType)
-        updateResultCache && updateResultCache({
-          document: res,
-        })
+        updateResultCache && updateResultCache(res)
       }
       onStepChange && onStepChange(+1)
       isSetting && onSave && onSave()
@@ -319,7 +378,6 @@ const StepTwo = ({
         <div className={cn(s.form)}>
           <div className={s.label}>{t('datasetCreation.stepTwo.segmentation')}</div>
           <div className='max-w-[640px]'>
-
             <div
               className={cn(
                 s.radioItem,
@@ -467,16 +525,41 @@ const StepTwo = ({
                 <Link className='text-[#155EEF]' href={`/datasets/${datasetId}/settings`}>{t('datasetCreation.stepTwo.datasetSettingLink')}</Link>
               </div>
             )}
-            <div className={s.file}>
-              <div className={s.fileContent}>
-                <div className='mb-2 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.fileName')}</div>
-                <div className='flex items-center text-sm leading-6 font-medium text-gray-800'>
-                  <span className={cn(s.fileIcon, file && s[file.extension])} />
-                  {getFileName(file?.name || '')}
-                </div>
+            {/* TODO multi files */}
+            <div className={s.source}>
+              <div className={s.sourceContent}>
+                {dataSourceType === DataSourceType.FILE && (
+                  <>
+                    <div className='mb-2 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.fileSource')}</div>
+                    <div className='flex items-center text-sm leading-6 font-medium text-gray-800'>
+                      <span className={cn(s.fileIcon, file && s[file.extension])} />
+                      {getFileName(file?.name || '')}
+                    </div>
+                  </>
+                )}
+                {dataSourceType === DataSourceType.NOTION && (
+                  <>
+                    <div className='mb-2 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.notionSource')}</div>
+                    <div className='flex items-center text-sm leading-6 font-medium text-gray-800'>
+                      <NotionIcon
+                        className='shrink-0 mr-1'
+                        type='page'
+                        src={notionPages[0]?.page_icon}
+                      />
+                      {notionPages[0]?.page_name}
+                      {notionPages.length > 1 && (
+                        <span className={s.sourceCount}>
+                          <span>{t('datasetCreation.stepTwo.other')}</span>
+                          <span>{notionPages.length - 1}</span>
+                          <span>{t('datasetCreation.stepTwo.notionUnit')}</span>
+                        </span>
+                      )}
+                    </div>
+                  </>
+                )}
               </div>
               <div className={s.divider} />
-              <div className={s.fileContent}>
+              <div className={s.segmentCount}>
                 <div className='mb-2 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.emstimateSegment')}</div>
                 <div className='flex items-center text-sm leading-6 font-medium text-gray-800'>
                   {