Fix pagination in memory db

* Cleanup code
* add pagination test for memory db

Cleanup test

cmd/yorkie/server.go

@@ -228,7 +228,7 @@ func init() {
 		"candidates limit per project for a single housekeeping run",
 	)
 	cmd.Flags().IntVar(
-		&conf.Housekeeping.HousekeepingProjectFetchSize,
+		&conf.Housekeeping.ProjectFetchSize,
 		"housekeeping-project-fetch-size",
 		server.DefaultHousekeepingProjectFetchSize,
 		"housekeeping project fetch size for a single housekeeping run",

server/backend/database/memory/database.go

@@ -21,6 +21,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"sort"
 	gotime "time"
 
 	"github.com/hashicorp/go-memdb"
@@ -242,36 +243,51 @@ func (d *DB) listProjectInfos(
 		return nil, fmt.Errorf("fetch all projects: %w", err)
 	}
 
-	lastIDBytes, err := housekeepingLastProjectID.Bytes()
-	if err != nil {
-		return nil, fmt.Errorf("decode last project id: %w", err)
+	var infos []*database.ProjectInfo
+
+	for raw := iter.Next(); raw != nil; raw = iter.Next() {
+		info := raw.(*database.ProjectInfo).DeepCopy()
+		infos = append(infos, info)
 	}
 
-	var infos []*database.ProjectInfo
-	for i := 0; i < pageSize; {
-		raw := iter.Next()
-		if raw == nil {
-			*housekeepingLastProjectID = database.DefaultProjectID
-			break
+	sort.SliceStable(infos, func(i, j int) bool {
+		firstIDBytes, err := infos[i].ID.Bytes()
+		if err != nil {
+			return true
+		}
+		secondIDBytes, err := infos[j].ID.Bytes()
+		if err != nil {
+			return true
 		}
 
-		info := raw.(*database.ProjectInfo).DeepCopy()
+		return bytes.Compare(firstIDBytes, secondIDBytes) < 0
+	})
 
-		idBytes, err := info.ID.Bytes()
+	beginIndex := 0
+	for i := 0; i < len(infos); i++ {
+		proejctIDBytes, err := infos[i].ID.Bytes()
 		if err != nil {
 			return nil, fmt.Errorf("decode project id: %w", err)
 		}
-
-		if bytes.Compare(idBytes, lastIDBytes) > 0 {
-			infos = append(infos, info)
-			i++
+		lastProjectIDBytes, err := housekeepingLastProjectID.Bytes()
+		if err != nil {
+			return nil, fmt.Errorf("decode last project id: %w", err)
 		}
-
-		if i == pageSize {
-			*housekeepingLastProjectID = infos[len(infos)-1].ID
+		if bytes.Compare(proejctIDBytes, lastProjectIDBytes) > 0 {
+			beginIndex = i
+			break
 		}
 	}
-	return infos, nil
+
+	lastIndex := beginIndex + pageSize - 1
+	if lastIndex >= len(infos) {
+		lastIndex = len(infos) - 1
+		*housekeepingLastProjectID = database.DefaultProjectID
+	} else {
+		*housekeepingLastProjectID = infos[lastIndex].ID
+	}
+
+	return infos[beginIndex : lastIndex+1], nil
 }
 
 // ListProjectInfos returns all project infos owned by owner.

server/backend/database/memory/housekeeping_test.go

@@ -76,4 +76,60 @@ func TestHousekeeping(t *testing.T) {
 		assert.Contains(t, candidates, clientB)
 		assert.NotContains(t, candidates, clientC)
 	})
+
+	t.Run("housekeeping pagination test", func(t *testing.T) {
+		ctx := context.Background()
+		memdb, projects := createDBandProjects(t)
+
+		housekeepingLastProjectID := database.DefaultProjectID
+		fetchSize := 4
+		_, err := memdb.FindDeactivateCandidates(
+			ctx,
+			0,
+			fetchSize,
+			&housekeepingLastProjectID,
+		)
+		assert.NoError(t, err)
+		assert.Equal(t, projects[fetchSize-1].ID, housekeepingLastProjectID)
+
+		_, err = memdb.FindDeactivateCandidates(
+			ctx,
+			0,
+			fetchSize,
+			&housekeepingLastProjectID,
+		)
+		assert.NoError(t, err)
+		assert.Equal(t, projects[fetchSize*2-1].ID, housekeepingLastProjectID)
+
+		_, err = memdb.FindDeactivateCandidates(
+			ctx,
+			0,
+			4,
+			&housekeepingLastProjectID,
+		)
+		assert.NoError(t, err)
+		assert.Equal(t, database.DefaultProjectID, housekeepingLastProjectID)
+	})
+}
+
+func createDBandProjects(t *testing.T) (*memory.DB, []*database.ProjectInfo) {
+	t.Helper()
+
+	ctx := context.Background()
+	memdb, err := memory.New()
+	assert.NoError(t, err)
+
+	clientDeactivateThreshold := "23h"
+	userInfo, err := memdb.CreateUserInfo(ctx, "test", "test")
+	assert.NoError(t, err)
+
+	projects := make([]*database.ProjectInfo, 0)
+	for i := 0; i < 10; i++ {
+		p, err := memdb.CreateProjectInfo(ctx, fmt.Sprintf("%d project", i), userInfo.ID, clientDeactivateThreshold)
+		assert.NoError(t, err)
+
+		projects = append(projects, p)
+	}
+
+	return memdb, projects
 }

server/backend/housekeeping/config.go

@@ -29,8 +29,8 @@ type Config struct {
 	// CandidatesLimitPerProject is the maximum number of candidates to be returned per project.
 	CandidatesLimitPerProject int `yaml:"CandidatesLimitPerProject"`
 
-	// HousekeepingProjectFetchSize is the maximum number of projects to be returned to deactivate candidates.
-	HousekeepingProjectFetchSize int `yaml:"HousekeepingProjectFetchSize"`
+	// ProjectFetchSize is the maximum number of projects to be returned to deactivate candidates.
+	ProjectFetchSize int `yaml:"HousekeepingProjectFetchSize"`
 }
 
 // Validate validates the configuration.
@@ -46,14 +46,14 @@ func (c *Config) Validate() error {
 	if c.CandidatesLimitPerProject <= 0 {
 		return fmt.Errorf(
 			`invalid argument %d for "--housekeeping-candidates-limit-per-project" flag`,
-			c.HousekeepingProjectFetchSize,
+			c.ProjectFetchSize,
 		)
 	}
 
-	if c.HousekeepingProjectFetchSize <= 0 {
+	if c.ProjectFetchSize <= 0 {
 		return fmt.Errorf(
 			`invalid argument %d for "--housekeeping-project-fetc-size" flag`,
-			c.HousekeepingProjectFetchSize,
+			c.ProjectFetchSize,
 		)
 	}
 

server/backend/housekeeping/config_test.go

@@ -27,9 +27,9 @@ import (
 func TestConfig(t *testing.T) {
 	t.Run("validate test", func(t *testing.T) {
 		validConf := housekeeping.Config{
-			Interval:                     "1m",
-			CandidatesLimitPerProject:    100,
-			HousekeepingProjectFetchSize: 100,
+			Interval:                  "1m",
+			CandidatesLimitPerProject: 100,
+			ProjectFetchSize:          100,
 		}
 		assert.NoError(t, validConf.Validate())
 
@@ -42,7 +42,7 @@ func TestConfig(t *testing.T) {
 		assert.Error(t, conf2.Validate())
 
 		conf3 := validConf
-		conf3.HousekeepingProjectFetchSize = -1
+		conf3.ProjectFetchSize = -1
 		assert.Error(t, conf3.Validate())
 	})
 }

server/backend/housekeeping/housekeeping.go

@@ -86,7 +86,7 @@ func New(
 
 		interval:                  interval,
 		candidatesLimitPerProject: conf.CandidatesLimitPerProject,
-		projectFetchSize:          conf.HousekeepingProjectFetchSize,
+		projectFetchSize:          conf.ProjectFetchSize,
 
 		ctx:        ctx,
 		cancelFunc: cancelFunc,

server/config.go

@@ -230,9 +230,9 @@ func newConfig(port int, profilingPort int) *Config {
 			Port: profilingPort,
 		},
 		Housekeeping: &housekeeping.Config{
-			Interval:                     DefaultHousekeepingInterval.String(),
-			CandidatesLimitPerProject:    DefaultHousekeepingCandidatesLimitPerProject,
-			HousekeepingProjectFetchSize: DefaultHousekeepingProjectFetchSize,
+			Interval:                  DefaultHousekeepingInterval.String(),
+			CandidatesLimitPerProject: DefaultHousekeepingCandidatesLimitPerProject,
+			ProjectFetchSize:          DefaultHousekeepingProjectFetchSize,
 		},
 		Backend: &backend.Config{
 			ClientDeactivateThreshold:  DefaultClientDeactivateThreshold,

server/config.sample.yml

@@ -36,8 +36,8 @@ Housekeeping:
   # CandidatesLimitPerProject is the maximum number of candidates to be returned per project (default: 100).
   CandidatesLimitPerProject: 100
 
-  # HousekeepingProjectFetchSize is the maximum number of projects to be returned to deactivate candidates. (default: 100).
-  HousekeepingProjectFetchSize: 100
+  # ProjectFetchSize is the maximum number of projects to be returned to deactivate candidates. (default: 100).
+  ProjectFetchSize: 100
 
 # Backend is the configuration for the backend of Yorkie.
 Backend:

server/rpc/server_test.go

@@ -83,9 +83,9 @@ func TestMain(m *testing.M) {
 		ConnectionTimeout: helper.MongoConnectionTimeout,
 		PingTimeout:       helper.MongoPingTimeout,
 	}, &housekeeping.Config{
-		Interval:                     helper.HousekeepingInterval.String(),
-		CandidatesLimitPerProject:    helper.HousekeepingCandidatesLimitPerProject,
-		HousekeepingProjectFetchSize: helper.HousekeepingProjectFetchSize,
+		Interval:                  helper.HousekeepingInterval.String(),
+		CandidatesLimitPerProject: helper.HousekeepingCandidatesLimitPerProject,
+		ProjectFetchSize:          helper.HousekeepingProjectFetchSize,
 	}, met)
 	if err != nil {
 		log.Fatal(err)

test/helper/helper.go

@@ -222,9 +222,9 @@ func TestConfig() *server.Config {
 			Port: ProfilingPort + portOffset,
 		},
 		Housekeeping: &housekeeping.Config{
-			Interval:                     HousekeepingInterval.String(),
-			CandidatesLimitPerProject:    HousekeepingCandidatesLimitPerProject,
-			HousekeepingProjectFetchSize: HousekeepingProjectFetchSize,
+			Interval:                  HousekeepingInterval.String(),
+			CandidatesLimitPerProject: HousekeepingCandidatesLimitPerProject,
+			ProjectFetchSize:          HousekeepingProjectFetchSize,
 		},
 		Backend: &backend.Config{
 			AdminUser:                  server.DefaultAdminUser,