Increase metadata fetch timeout to 30s (#803)

Signed-off-by: Wayne Zhang <qiwzhang@google.com>
cloudendpoints · Jun 17, 2020 · 221f624 · 221f624
1 parent 816a204
commit 221f624
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 11 deletions.
diff --git a/src/api_manager/fetch_metadata.cc b/src/api_manager/fetch_metadata.cc
@@ -42,9 +42,9 @@ const char kMetadataInstanceIdentityToken[] =
 const int kInstanceIdentityTokenExpiration = 3500;
 // Time window (in seconds) of failure status after a failed fetch.
 const int kFailureStatusWindow = 5;
-// GKE metadata server may take >2s to fetch access token.
-// Initial metadata fetch timeout (5s)
-const int kMetadataFetchTimeout = 5000;
+// GKE metadata server may take >20s to fetch access token (b/148454048).
+// Initial metadata fetch timeout (30s)
+const int kMetadataFetchTimeout = 30000;
 // Maximum number of retries to fetch token from metadata
 const int kMetadataTokenFetchRetries = 3;
 // External status message for failure to fetch service account token

diff --git a/src/nginx/t/ApiManager.pm b/src/nginx/t/ApiManager.pm
@@ -580,7 +580,7 @@ sub http($;$;%) {
   my $s = http_start($port, $request, %extra);
 
   return $s if $extra{start} or !defined $s;
-  return http_end($s);
+  return http_end($s, %extra);
 }
 
 sub http_start($;$;%) {
@@ -590,7 +590,9 @@ sub http_start($;$;%) {
   eval {
     local $SIG{ALRM} = sub { die "timeout\n" };
     local $SIG{PIPE} = sub { die "sigpipe\n" };
-    alarm(8);
+    my $timeout = 8;
+    $timeout = $extra{timeout} if $extra{timeout};
+    alarm($timeout);
 
     $s = $extra{socket} || IO::Socket::INET->new(
       Proto => 'tcp',
@@ -621,13 +623,15 @@ sub http_start($;$;%) {
 }
 
 sub http_end($;%) {
-  my ($s) = @_;
+  my ($s, %extra) = @_;
   my $reply;
 
   eval {
     local $SIG{ALRM} = sub { die "timeout\n" };
     local $SIG{PIPE} = sub { die "sigpipe\n" };
-    alarm(8);
+    my $timeout = 8;
+    $timeout = $extra{timeout} if $extra{timeout};
+    alarm($timeout);
 
     local $/;
     $reply = $s->getline();

diff --git a/src/nginx/t/metadata_timeout.t b/src/nginx/t/metadata_timeout.t
@@ -96,7 +96,7 @@ $t->run();
 
 ################################################################################
 
-my $shelves = ApiManager::http_get($NginxPort,'/shelves');
+my $shelves = ApiManager::http_get($NginxPort,'/shelves', (timeout => 40));
 my $books = ApiManager::http_get($NginxPort,'/shelves/Musicals/books');
 
 is($t->waitforfile("$t->{_testdir}/${report_done}"), 1, 'Report body file ready.');
@@ -207,8 +207,8 @@ sub metadata {
 
     $request_count++;
     if ($request_count == 1) {
-        # Trigger a timeout for the first request.
-        sleep 7;
+        # Trigger a timeout for the first request. timeout is 30s.
+        sleep 32;
     }
 
     print $client <<'EOF';

diff --git a/start_esp/fetch_service_config.py b/start_esp/fetch_service_config.py
@@ -60,7 +60,8 @@ def fetch_metadata(metadata, attr_path, required):
     headers = {"Metadata-Flavor": "Google"}
     client = urllib3.PoolManager(ca_certs=certifi.where())
     if required:
-      timeout = 1.0
+      # GKE metadata server may take up to 30s to get access token.
+      timeout = 30.0
       retries = True
     else:
       timeout = 0.1