GO-3861 prioritize ft indexation and reindex by space lastOpenedDate

core/block/editor/smartblock/smarttest/smarttest.go

@@ -81,10 +81,11 @@ func (st *SmartTest) SetSpace(space smartblock.Space) {
 }
 
 type stubSpace struct {
+	id string
 }
 
 func (s *stubSpace) Id() string {
-	return ""
+	return s.id
 }
 
 func (s *stubSpace) TreeBuilder() objecttreebuilder.TreeBuilder {
@@ -127,7 +128,8 @@ func (st *SmartTest) Space() smartblock.Space {
 	if st.space != nil {
 		return st.space
 	}
-	return &stubSpace{}
+
+	return &stubSpace{id: st.spaceId}
 }
 
 func (st *SmartTest) EnabledRelationAsDependentObjects() {

core/indexer/fulltext.go

@@ -25,7 +25,7 @@ import (
 var (
 	ftIndexInterval         = 1 * time.Second
 	ftIndexForceMinInterval = time.Second * 10
-	ftBatchLimit            = 1000
+	ftBatchLimit            = 50
 	ftBlockMaxSize          = 1024 * 1024
 )
 
@@ -40,37 +40,28 @@ func (i *indexer) ForceFTIndex() {
 // MUST NOT be called more than once
 func (i *indexer) ftLoopRoutine() {
 	ticker := time.NewTicker(ftIndexInterval)
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go func() {
-		select {
-		case <-i.quit:
-			cancel()
-		case <-ctx.Done():
-		}
-	}()
 
-	i.runFullTextIndexer(ctx)
+	i.runFullTextIndexer(i.componentCtx, i.spacesPriorityGet())
 	defer close(i.ftQueueFinished)
 	var lastForceIndex time.Time
 	for {
 		select {
-		case <-ctx.Done():
+		case <-i.componentCtx.Done():
 			return
 		case <-ticker.C:
-			i.runFullTextIndexer(ctx)
+			i.runFullTextIndexer(i.componentCtx, i.spacesPriorityGet())
 		case <-i.forceFt:
 			if time.Since(lastForceIndex) > ftIndexForceMinInterval {
-				i.runFullTextIndexer(ctx)
+				i.runFullTextIndexer(i.componentCtx, i.spacesPriorityGet())
 				lastForceIndex = time.Now()
 			}
 		}
 	}
 }
 
-func (i *indexer) runFullTextIndexer(ctx context.Context) {
+func (i *indexer) runFullTextIndexer(ctx context.Context, spaceIdsPriority []string) {
 	batcher := i.ftsearch.NewAutoBatcher(ftsearch.AutoBatcherRecommendedMaxDocs, ftsearch.AutoBatcherRecommendedMaxSize)
-	err := i.store.BatchProcessFullTextQueue(ctx, ftBatchLimit, func(objectIds []string) error {
+	err := i.store.BatchProcessFullTextQueue(ctx, spaceIdsPriority, ftBatchLimit, func(objectIds []string) error {
 		for _, objectId := range objectIds {
 			objDocs, err := i.prepareSearchDocument(ctx, objectId)
 			if err != nil {
@@ -224,14 +215,26 @@ func (i *indexer) ftInit() error {
 			return err
 		}
 		if docCount == 0 {
-			ids, err := i.store.ListIds()
+			spaceIds, err := i.storageService.AllSpaceIds()
 			if err != nil {
 				return err
 			}
-			for _, id := range ids {
-				if err := i.store.AddToIndexQueue(id); err != nil {
+			var fullIds []domain.FullID
+			for _, spaceId := range spaceIds {
+				ids, err := i.store.ListIdsBySpace(spaceId)
+				if err != nil {
 					return err
 				}
+				for _, id := range ids {
+					fullIds = append(fullIds, domain.FullID{
+						ObjectID: id,
+						SpaceID:  spaceId,
+					})
+				}
+			}
+			err = i.store.AddToIndexQueue(fullIds...)
+			if err != nil {
+				return err
 			}
 		}
 	}

core/indexer/fulltext_test.go

@@ -17,6 +17,7 @@
 	"github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest"
 	"github.com/anyproto/anytype-heart/core/block/editor/state"
 	"github.com/anyproto/anytype-heart/core/block/source/mock_source"
+	"github.com/anyproto/anytype-heart/core/domain"
 	"github.com/anyproto/anytype-heart/core/indexer/mock_indexer"
 	"github.com/anyproto/anytype-heart/core/wallet"
 	"github.com/anyproto/anytype-heart/core/wallet/mock_wallet"
@@ -32,6 +33,7 @@
 	"github.com/anyproto/anytype-heart/tests/blockbuilder"
 	"github.com/anyproto/anytype-heart/tests/testutil"
 	"github.com/anyproto/anytype-heart/util/pbtypes"
+	"github.com/anyproto/anytype-heart/util/taskmanager"
 )
 
 type IndexerFixture struct {
@@ -64,6 +66,7 @@
 	testApp.Register(objectStore.FTSearch())
 
 	indxr := &indexer{}
+	indxr.spaceReindexQueue = taskmanager.NewTasksManager(1, indxr.taskPrioritySorter)
 
 	indexerFx := &IndexerFixture{
 		indexer:     indxr,
@@ -85,7 +88,7 @@
 	indexerFx.ftsearch = indxr.ftsearch
 	indexerFx.pickerFx = mock_cache.NewMockObjectGetter(t)
 	indxr.picker = indexerFx.pickerFx
-	indxr.quit = make(chan struct{})
+	indxr.componentCtx, indxr.componentCancel = context.WithCancel(context.Background())
 	indxr.forceFt = make(chan struct{})
 	indxr.config = &config.Config{NetworkMode: pb.RpcAccount_LocalOnly}
 
@@ -326,11 +329,14 @@
 					blockbuilder.ID("blockId1"),
 				),
 			)))
-		indexerFx.store.AddToIndexQueue("objectId" + strconv.Itoa(i))
+		indexerFx.store.AddToIndexQueue(domain.FullID{
+			ObjectID: "objectId" + strconv.Itoa(i),
+			SpaceID:  "space1",
+		})
 		indexerFx.pickerFx.EXPECT().GetObject(mock.Anything, "objectId"+strconv.Itoa(i)).Return(smartTest, nil).Once()
 	}
 
-	indexerFx.runFullTextIndexer(context.Background())
+	indexerFx.runFullTextIndexer(context.Background(), []string{"space1"})
 
 	count, _ := indexerFx.ftsearch.DocCount()
 	assert.Equal(t, 10, int(count))
@@ -352,11 +358,14 @@
 				),
 			)))
 		indexerFx.pickerFx.EXPECT().GetObject(mock.Anything, "objectId"+strconv.Itoa(i)).Return(smartTest, nil).Once()
-		indexerFx.store.AddToIndexQueue("objectId" + strconv.Itoa(i))
+		indexerFx.store.AddToIndexQueue(domain.FullID{
+			ObjectID: "objectId" + strconv.Itoa(i),
+			SpaceID:  "space1",
+		})
 
 	}
 
-	indexerFx.runFullTextIndexer(context.Background())
+	indexerFx.runFullTextIndexer(context.Background(), []string{"space1"})
 
 	count, _ = indexerFx.ftsearch.DocCount()
 	assert.Equal(t, 10, int(count))
@@ -381,9 +390,12 @@
 				blockbuilder.ID("blockId1"),
 			),
 		)))
-	indexerFx.store.AddToIndexQueue("objectId1")
+	indexerFx.store.AddToIndexQueue(domain.FullID{
+		ObjectID: "objectId1",
+		SpaceID:  "space1",
+	})
 	indexerFx.pickerFx.EXPECT().GetObject(mock.Anything, mock.Anything).Return(smartTest, nil)
-	indexerFx.runFullTextIndexer(context.Background())
+	indexerFx.runFullTextIndexer(context.Background(), []string{"space1"})
 
 	count, _ = indexerFx.ftsearch.DocCount()
 	assert.Equal(t, uint64(1), count)