fix: Do not initialise local querier, when their are no local blocks

This builds on top of #4151 and #4144, as previously initilised heads
already created a folder on all ingesters. Hence the check is not
detecting that there are actually no local blocks to be considered.

Author: simonswine

pkg/ingester/ingester.go

@@ -195,6 +195,22 @@ func (i *Ingester) getOrCreateInstance(tenantID string) (*instance, error) {
 	return inst, nil
 }
 
+func (i *Ingester) hasLocalBlocks(tenantID string) (bool, error) {
+	entries, err := os.ReadDir(filepath.Join(i.dbConfig.DataPath, tenantID, "local"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	for _, entry := range entries {
+		if entry.IsDir() {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 func (i *Ingester) getOrOpenInstance(tenantID string) (*instance, error) {
 	inst, ok := i.getInstanceByID(tenantID)
 	if ok {
@@ -208,14 +224,14 @@ func (i *Ingester) getOrOpenInstance(tenantID string) (*instance, error) {
 		return inst, nil
 	}
 
-	if _, err := os.Stat(filepath.Join(i.dbConfig.DataPath, tenantID)); err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil
-		}
+	hasLocalBlocks, err := i.hasLocalBlocks(tenantID)
+	if err != nil {
 		return nil, err
 	}
+	if !hasLocalBlocks {
+		return nil, nil
+	}
 
-	var err error
 	limiter := NewLimiter(tenantID, i.limits, i.lifecycler, i.cfg.LifecyclerConfig.RingConfig.ReplicationFactor)
 	inst, err = newInstance(i.phlarectx, i.dbConfig, tenantID, i.localBucket, i.storageBucket, limiter)
 	if err != nil {

pkg/ingester/ingester_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"os"
+	"path/filepath"
 	"runtime/pprof"
 	"testing"
 	"time"
@@ -132,11 +133,19 @@ func Test_Query_TenantNotFound(t *testing.T) {
 		},
 	}
 
+	// set the localPath
+	localPath := t.TempDir()
+
+	// foo has an empty local dir
+	fooLocalPath := filepath.Join(localPath, "foo", "local")
+	require.NoError(t, os.MkdirAll(fooLocalPath, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(fooLocalPath, "shipper.json"), []byte(`{"version":1,"uploaded":null}`), 0o755))
+
 	fs, err := client.NewBucket(ctx, cfg, "storage")
 	require.NoError(t, err)
 
 	ing, err := New(ctx, defaultIngesterTestConfig(t), phlaredb.Config{
-		DataPath:         dbPath,
+		DataPath:         localPath,
 		MaxBlockDuration: 30 * time.Hour,
 	}, fs, &fakeLimits{}, 0)
 	require.NoError(t, err)
@@ -150,6 +159,11 @@ func Test_Query_TenantNotFound(t *testing.T) {
 	require.NoError(t, err)
 	require.Empty(t, labelsNames.Msg.Names)
 
+	// check that no tenant are initialized
+	ing.instancesMtx.RLock()
+	require.Len(t, ing.instances, 0)
+	ing.instancesMtx.RUnlock()
+
 	require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing))
 }