Migrate most flags to the new config (#2095)

* Migrate most flags to new config

cfg/config_util.go

@@ -18,6 +18,6 @@ package cfg
 func OverrideWithLoggingFlags(mountConfig *Config, debugFuse bool,
 	debugGCS bool, debugMutex bool) {
 	if debugFuse || debugGCS || debugMutex {
-		mountConfig.Logging.Severity = LogSeverity("TRACE")
+		mountConfig.Logging.Severity = "TRACE"
 	}
 }

cmd/flags.go

@@ -410,15 +410,20 @@ type flagStorage struct {
 	Uid int64
 
 	// Deprecated: Use the param from cfg/config.go
-	Gid          int64
+	Gid int64
+
+	// Deprecated: Use the param from cfg/config.go
 	ImplicitDirs bool
-	OnlyDir      string
+
+	// Deprecated: Use the param from cfg/config.go
+	OnlyDir string
 
 	// Deprecated: Use the param from cfg/config.go
 	RenameDirLimit   int64
 	IgnoreInterrupts bool
 
 	// GCS
+	// Deprecated: Use the param from cfg/config.go
 	CustomEndpoint *url.URL
 
 	// Deprecated: Use the param from cfg/config.go
@@ -442,18 +447,35 @@ type flagStorage struct {
 	AnonymousAccess      bool
 
 	// Tuning
-	MaxRetrySleep             time.Duration
-	StatCacheCapacity         int
-	StatCacheTTL              time.Duration
+	// Deprecated: Use the param from cfg/config.go
+	MaxRetrySleep time.Duration
+
+	// Deprecated: Use the param from cfg/config.go
+	StatCacheCapacity int
+
+	// Deprecated: Use the param from cfg/config.go
+	StatCacheTTL time.Duration
+
+	// Deprecated: Use the param from cfg/config.go
 	TypeCacheTTL              time.Duration
 	KernelListCacheTtlSeconds int64
-	HttpClientTimeout         time.Duration
-	RetryMultiplier           float64
 
 	// Deprecated: Use the param from cfg/config.go
-	TempDir             string
-	ClientProtocol      mountpkg.ClientProtocol
-	MaxConnsPerHost     int
+	HttpClientTimeout time.Duration
+
+	// Deprecated: Use the param from cfg/config.go
+	RetryMultiplier float64
+
+	// Deprecated: Use the param from cfg/config.go
+	TempDir string
+
+	// Deprecated: Use the param from cfg/config.go
+	ClientProtocol mountpkg.ClientProtocol
+
+	// Deprecated: Use the param from cfg/config.go
+	MaxConnsPerHost int
+
+	// Deprecated: Use the param from cfg/config.go
 	MaxIdleConnsPerHost int
 
 	// Deprecated: Use the param from cfg/config.go
@@ -478,10 +500,16 @@ type flagStorage struct {
 	// Debugging
 
 	// Deprecated: Use the param from cfg/config.go
-	DebugFuse       bool
-	DebugGCS        bool
+	DebugFuse bool
+
+	// Deprecated: Use the param from cfg/config.go
+	DebugGCS bool
+
+	// Deprecated: Use the param from cfg/config.go
 	DebugInvariants bool
-	DebugMutex      bool
+
+	// Deprecated: Use the param from cfg/config.go
+	DebugMutex bool
 
 	// Post-mount actions
 
@@ -644,10 +672,6 @@ func validateFlags(flags *flagStorage) (err error) {
 		return fmt.Errorf("SequentialReadSizeMb should be less than %d", maxSequentialReadSizeMb)
 	}
 
-	if !flags.ClientProtocol.IsValid() {
-		return fmt.Errorf("client protocol: %s is not valid", flags.ClientProtocol)
-	}
-
 	if err = validateExperimentalMetadataPrefetchOnMount(flags.ExperimentalMetadataPrefetchOnMount); err != nil {
 		return fmt.Errorf("%s: is not valid; error = %w", ExperimentalMetadataPrefetchOnMountFlag, err)
 	}

cmd/flags_test.go

@@ -299,22 +299,9 @@ func (t *FlagsTest) TestResolvePathForTheFlagsInContext() {
 	assert.Equal(t.T(), nil, err)
 }
 
-func (t *FlagsTest) TestValidateFlagsForValidSequentialReadSizeAndHTTP1ClientProtocol() {
-	flags := &flagStorage{
-		SequentialReadSizeMb:                10,
-		ClientProtocol:                      mountpkg.ClientProtocol("http1"),
-		ExperimentalMetadataPrefetchOnMount: config.DefaultExperimentalMetadataPrefetchOnMount,
-	}
-
-	err := validateFlags(flags)
-
-	assert.Equal(t.T(), nil, err)
-}
-
-func (t *FlagsTest) TestValidateFlagsForZeroSequentialReadSizeAndValidClientProtocol() {
+func (t *FlagsTest) TestValidateFlagsForZeroSequentialReadSize() {
 	flags := &flagStorage{
 		SequentialReadSizeMb:                0,
-		ClientProtocol:                      mountpkg.ClientProtocol("http2"),
 		ExperimentalMetadataPrefetchOnMount: config.DefaultExperimentalMetadataPrefetchOnMount,
 	}
 
@@ -324,10 +311,9 @@ func (t *FlagsTest) TestValidateFlagsForZeroSequentialReadSizeAndValidClientProt
 	assert.Equal(t.T(), "SequentialReadSizeMb should be less than 1024", err.Error())
 }
 
-func (t *FlagsTest) TestValidateFlagsForSequentialReadSizeGreaterThan1024AndValidClientProtocol() {
+func (t *FlagsTest) TestValidateFlagsForSequentialReadSizeGreaterThan1024() {
 	flags := &flagStorage{
 		SequentialReadSizeMb:                2048,
-		ClientProtocol:                      mountpkg.ClientProtocol("http1"),
 		ExperimentalMetadataPrefetchOnMount: config.DefaultExperimentalMetadataPrefetchOnMount,
 	}
 
@@ -337,22 +323,9 @@ func (t *FlagsTest) TestValidateFlagsForSequentialReadSizeGreaterThan1024AndVali
 	assert.Equal(t.T(), "SequentialReadSizeMb should be less than 1024", err.Error())
 }
 
-func (t *FlagsTest) TestValidateFlagsForValidSequentialReadSizeAndInValidClientProtocol() {
-	flags := &flagStorage{
-		SequentialReadSizeMb:                10,
-		ClientProtocol:                      mountpkg.ClientProtocol("http4"),
-		ExperimentalMetadataPrefetchOnMount: config.DefaultExperimentalMetadataPrefetchOnMount,
-	}
-
-	err := validateFlags(flags)
-
-	assert.Equal(t.T(), "client protocol: http4 is not valid", err.Error())
-}
-
-func (t *FlagsTest) TestValidateFlagsForValidSequentialReadSizeAndHTTP2ClientProtocol() {
+func (t *FlagsTest) TestValidateFlagsForValidSequentialReadSize() {
 	flags := &flagStorage{
 		SequentialReadSizeMb:                10,
-		ClientProtocol:                      mountpkg.ClientProtocol("http2"),
 		ExperimentalMetadataPrefetchOnMount: config.DefaultExperimentalMetadataPrefetchOnMount,
 	}
 
@@ -368,7 +341,6 @@ func (t *FlagsTest) TestValidateFlagsForSupportedExperimentalMetadataPrefetchOnM
 		flags := &flagStorage{
 			// Unrelated fields, not being tested here, so set to sane values.
 			SequentialReadSizeMb: 200,
-			ClientProtocol:       mountpkg.ClientProtocol("http2"),
 			// The flag being tested.
 			ExperimentalMetadataPrefetchOnMount: input,
 		}
@@ -386,7 +358,6 @@ func (t *FlagsTest) TestValidateFlagsForUnsupportedExperimentalMetadataPrefetchO
 		flags := &flagStorage{
 			// Unrelated fields, not being tested here, so set to sane values.
 			SequentialReadSizeMb: 200,
-			ClientProtocol:       mountpkg.ClientProtocol("http2"),
 			// The flag being tested.
 			ExperimentalMetadataPrefetchOnMount: input,
 		}

cmd/legacy_main.go

@@ -101,16 +101,16 @@ func getConfigForUserAgent(mountConfig *config.MountConfig) string {
 	}
 	return fmt.Sprintf("%s:%s", isFileCacheEnabled, isFileCacheForRangeReadEnabled)
 }
-func createStorageHandle(newConfig *cfg.Config, flags *flagStorage, mountConfig *config.MountConfig, userAgent string) (storageHandle storage.StorageHandle, err error) {
+func createStorageHandle(newConfig *cfg.Config, mountConfig *config.MountConfig, userAgent string) (storageHandle storage.StorageHandle, err error) {
 	storageClientConfig := storageutil.StorageClientConfig{
 		ClientProtocol:             newConfig.GcsConnection.ClientProtocol,
-		MaxConnsPerHost:            flags.MaxConnsPerHost,
-		MaxIdleConnsPerHost:        flags.MaxIdleConnsPerHost,
-		HttpClientTimeout:          flags.HttpClientTimeout,
-		MaxRetrySleep:              flags.MaxRetrySleep,
-		RetryMultiplier:            flags.RetryMultiplier,
+		MaxConnsPerHost:            int(newConfig.GcsConnection.MaxConnsPerHost),
+		MaxIdleConnsPerHost:        int(newConfig.GcsConnection.MaxIdleConnsPerHost),
+		HttpClientTimeout:          newConfig.GcsConnection.HttpClientTimeout,
+		MaxRetrySleep:              newConfig.GcsRetries.MaxRetrySleep,
+		RetryMultiplier:            newConfig.GcsRetries.Multiplier,
 		UserAgent:                  userAgent,
-		CustomEndpoint:             flags.CustomEndpoint,
+		CustomEndpoint:             newConfig.GcsConnection.CustomEndpoint,
 		KeyFile:                    string(newConfig.GcsAuth.KeyFile),
 		AnonymousAccess:            mountConfig.GCSAuth.AnonymousAccess,
 		TokenUrl:                   newConfig.GcsAuth.TokenUrl,
@@ -136,10 +136,10 @@ func mountWithArgs(
 	mountConfig *config.MountConfig,
 	newConfig *cfg.Config) (mfs *fuse.MountedFileSystem, err error) {
 	// Enable invariant checking if requested.
-	if flags.DebugInvariants {
+	if newConfig.Debug.ExitOnInvariantViolation {
 		locker.EnableInvariantsCheck()
 	}
-	if flags.DebugMutex {
+	if newConfig.Debug.LogMutex {
 		locker.EnableDebugMessages()
 	}
 
@@ -151,7 +151,7 @@ func mountWithArgs(
 	if bucketName != canned.FakeBucketName {
 		userAgent := getUserAgent(newConfig.AppName, getConfigForUserAgent(mountConfig))
 		logger.Info("Creating Storage handle...")
-		storageHandle, err = createStorageHandle(newConfig, flags, mountConfig, userAgent)
+		storageHandle, err = createStorageHandle(newConfig, mountConfig, userAgent)
 		if err != nil {
 			err = fmt.Errorf("failed to create storage handle using createStorageHandle: %w", err)
 			return
@@ -313,12 +313,12 @@ func runCLIApp(c *cli.Context) (err error) {
 	}
 
 	// The following will not warn if the user explicitly passed the default value for StatCacheCapacity.
-	if flags.StatCacheCapacity != mount.DefaultStatCacheCapacity {
+	if newConfig.MetadataCache.DeprecatedStatCacheCapacity != mount.DefaultStatCacheCapacity {
 		logger.Warnf("Deprecated flag stat-cache-capacity used! Please switch to config parameter 'metadata-cache: stat-cache-max-size-mb'.")
 	}
 
 	// The following will not warn if the user explicitly passed the default value for StatCacheTTL or TypeCacheTTL.
-	if flags.StatCacheTTL != mount.DefaultStatOrTypeCacheTTL || flags.TypeCacheTTL != mount.DefaultStatOrTypeCacheTTL {
+	if newConfig.MetadataCache.DeprecatedStatCacheTtl != mount.DefaultStatOrTypeCacheTTL || newConfig.MetadataCache.DeprecatedTypeCacheTtl != mount.DefaultStatOrTypeCacheTTL {
 		logger.Warnf("Deprecated flag stat-cache-ttl and/or type-cache-ttl used! Please switch to config parameter 'metadata-cache: ttl-secs' .")
 	}
 

cmd/legacy_main_test.go

@@ -42,43 +42,25 @@ type MainTest struct {
 }
 
 func (t *MainTest) TestCreateStorageHandle() {
-	flags := &flagStorage{
-		MaxConnsPerHost:     5,
-		MaxIdleConnsPerHost: 100,
-		HttpClientTimeout:   5,
-		MaxRetrySleep:       7,
-		RetryMultiplier:     2,
-	}
-	mountConfig := &config.MountConfig{}
 	newConfig := &cfg.Config{
 		GcsConnection: cfg.GcsConnectionConfig{ClientProtocol: cfg.HTTP1},
 		GcsAuth:       cfg.GcsAuthConfig{KeyFile: "testdata/test_creds.json"}}
 
 	userAgent := "AppName"
-	storageHandle, err := createStorageHandle(newConfig, flags, mountConfig, userAgent)
+	storageHandle, err := createStorageHandle(newConfig, &config.MountConfig{}, userAgent)
 
 	assert.Equal(t.T(), nil, err)
 	assert.NotEqual(t.T(), nil, storageHandle)
 }
 
 func (t *MainTest) TestCreateStorageHandle_WithClientProtocolAsGRPC() {
-	flags := &flagStorage{
-		MaxConnsPerHost:     5,
-		MaxIdleConnsPerHost: 100,
-		HttpClientTimeout:   5,
-		MaxRetrySleep:       7,
-		RetryMultiplier:     2,
-	}
-	mountConfig := &config.MountConfig{
-		GCSConnection: config.GCSConnection{GRPCConnPoolSize: 1},
-	}
 	newConfig := &cfg.Config{
 		GcsConnection: cfg.GcsConnectionConfig{ClientProtocol: cfg.GRPC},
 		GcsAuth:       cfg.GcsAuthConfig{KeyFile: "testdata/test_creds.json"},
 	}
 
 	userAgent := "AppName"
-	storageHandle, err := createStorageHandle(newConfig, flags, mountConfig, userAgent)
+	storageHandle, err := createStorageHandle(newConfig, &config.MountConfig{}, userAgent)
 
 	assert.Equal(t.T(), nil, err)
 	assert.NotEqual(t.T(), nil, storageHandle)

cmd/legacy_param_mapper_test.go

@@ -451,3 +451,25 @@ func TestCustomEndpointResolutionFromFlags(t *testing.T) {
 		assert.Equal(t, *resolvedConfig.GcsConnection.CustomEndpoint, *u)
 	}
 }
+
+func TestValidClientProtocol(t *testing.T) {
+	flags := &flagStorage{
+		ClientProtocol: mountpkg.ClientProtocol("http1"),
+	}
+
+	v, err := PopulateNewConfigFromLegacyFlagsAndConfig(&mockCLIContext{}, flags, &config.MountConfig{})
+
+	if assert.Nil(t, err) {
+		assert.Equal(t, v.GcsConnection.ClientProtocol, cfg.Protocol("http1"))
+	}
+}
+
+func TestInvalidClientProtocol(t *testing.T) {
+	flags := &flagStorage{
+		ClientProtocol: mountpkg.ClientProtocol("http3"),
+	}
+
+	_, err := PopulateNewConfigFromLegacyFlagsAndConfig(&mockCLIContext{}, flags, &config.MountConfig{})
+
+	assert.NotNil(t, err)
+}

cmd/mount.go

@@ -86,10 +86,11 @@ be interacting with the file system.`)
 		gid = uint32(newConfig.FileSystem.Gid)
 	}
 
-	metadataCacheTTL := mount.ResolveMetadataCacheTTL(flags.StatCacheTTL, flags.TypeCacheTTL, mountConfig.MetadataCacheConfig.TtlInSeconds)
-	statCacheMaxSizeMB, err := mount.ResolveStatCacheMaxSizeMB(mountConfig.StatCacheMaxSizeMB, flags.StatCacheCapacity)
+	metadataCacheTTL := mount.ResolveMetadataCacheTTL(newConfig.MetadataCache.DeprecatedStatCacheTtl, newConfig.MetadataCache.DeprecatedTypeCacheTtl, mountConfig.MetadataCacheConfig.TtlInSeconds)
+	statCacheMaxSizeMB, err := mount.ResolveStatCacheMaxSizeMB(mountConfig.StatCacheMaxSizeMB, int(newConfig.MetadataCache.DeprecatedStatCacheCapacity))
 	if err != nil {
-		return nil, fmt.Errorf("failed to calculate StatCacheMaxSizeMB from stat-cache-ttl=%v, metadata-cache:stat-cache-max-size-mb=%v: %w", flags.StatCacheCapacity, mountConfig.StatCacheMaxSizeMB, err)
+		return nil, fmt.Errorf("failed to calculate StatCacheMaxSizeMB from stat-cache-capacity=%v, metadata-cache:stat-cache-max-size-mb=%v: %w",
+			newConfig.MetadataCache.DeprecatedStatCacheCapacity, mountConfig.StatCacheMaxSizeMB, err)
 	}
 
 	bucketCfg := gcsx.BucketConfig{
@@ -102,7 +103,7 @@ be interacting with the file system.`)
 		EnableMonitoring:                   newConfig.Metrics.StackdriverExportInterval > 0,
 		AppendThreshold:                    1 << 21, // 2 MiB, a total guess.
 		TmpObjectPrefix:                    ".gcsfuse_tmp/",
-		DebugGCS:                           flags.DebugGCS,
+		DebugGCS:                           newConfig.Debug.Gcs,
 	}
 	bm := gcsx.NewBucketManager(bucketCfg, storageHandle)
 
@@ -113,7 +114,7 @@ be interacting with the file system.`)
 		BucketName:                 bucketName,
 		LocalFileCache:             false,
 		TempDir:                    string(newConfig.FileSystem.TempDir),
-		ImplicitDirectories:        flags.ImplicitDirs,
+		ImplicitDirectories:        newConfig.ImplicitDirs,
 		InodeAttributeCacheTTL:     metadataCacheTTL,
 		DirTypeCacheTTL:            metadataCacheTTL,
 		Uid:                        uid,