New vcl_backend_refresh method

bin/varnishd/builtin.vcl

@@ -209,6 +209,47 @@ sub vcl_backend_response {
 	return (deliver);
 }
 
+sub vcl_backend_refresh {
+        call vcl_builtin_backend_refresh;
+        return (merge);
+}
+
+sub vcl_builtin_backend_refresh {
+        call vcl_refresh_valid;
+        call vcl_refresh_conditions;
+        call vcl_refresh_status;
+}
+
+sub vcl_refresh_valid {
+        if (obj_stale.retried) { # read-only, analogous to bereq.retries, but BOOL
+                return (error);
+        }
+        if (!obj_stale.is_valid) {
+                call vcl_refresh_retry;
+        }
+}
+
+sub vcl_refresh_status {
+        if (obj_stale.status != 200) {
+                call vcl_refresh_retry;
+        }
+}
+
+sub vcl_refresh_conditions {
+        if (!bereq.http.if-modified-since &&
+            !bereq.http.if-none-match) {
+                return (error);
+        }
+}
+
+sub vcl_refresh_retry {
+        unset bereq.http.if-modified-since;
+        unset bereq.http.if-none-match;
+        # Same transition as return (fetch) from vcl_backend_fetch,
+        # but turns into an error if obj_stale.retried already true.
+        return (retry(fetch));
+}
+
 sub vcl_builtin_backend_response {
 	call vcl_beresp_range;
 	if (bereq.uncacheable) {

bin/varnishd/cache/cache.h

@@ -388,6 +388,7 @@ struct busyobj {
 	 * is recycled.
 	 */
 	unsigned		retries;
+	unsigned		retried_stale;
 	struct req		*req;
 	struct sess		*sp;
 	struct worker		*wrk;

bin/varnishd/cache/cache_fetch.c

@@ -350,37 +350,24 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo)
  * 304 setup logic
  */
 
-static int
+static void
 vbf_304_logic(struct busyobj *bo)
 {
-	if (bo->stale_oc != NULL &&
-	    ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) {
-		AZ(bo->stale_oc->flags & (OC_F_HFM|OC_F_PRIVATE));
-		if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGCE)) {
-			/*
-			 * If a VFP changed C-E in the stored
-			 * object, then don't overwrite C-E from
-			 * the IMS fetch, and we must weaken any
-			 * new ETag we get.
-			 */
-			RFC2616_Weaken_Etag(bo->beresp);
-		}
-		http_Unset(bo->beresp, H_Content_Encoding);
-		http_Unset(bo->beresp, H_Content_Length);
-		HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp);
-		assert(http_IsStatus(bo->beresp, 200));
-		bo->was_304 = 1;
-	} else if (!bo->uncacheable) {
+
+	AZ(bo->stale_oc->flags & (OC_F_HFM|OC_F_PRIVATE));
+	if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGCE)) {
 		/*
-		 * Backend sent unallowed 304
+		 * If a VFP changed C-E in the stored
+		 * object, then don't overwrite C-E from
+		 * the IMS fetch, and we must weaken any
+		 * new ETag we get.
 		 */
-		VSLb(bo->vsl, SLT_Error,
-		    "304 response but not conditional fetch");
-		bo->htc->doclose = SC_RX_BAD;
-		vbf_cleanup(bo);
-		return (-1);
+		RFC2616_Weaken_Etag(bo->beresp);
 	}
-	return (1);
+	http_Unset(bo->beresp, H_Content_Encoding);
+	http_Unset(bo->beresp, H_Content_Length);
+	HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp);
+	assert(http_IsStatus(bo->beresp, 200));
 }
 
 /*--------------------------------------------------------------------
@@ -392,7 +379,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 {
 	int i;
 	vtim_real now;
-	unsigned handling;
+	unsigned handling, retried_stale, skip_vbr = 0;
 	struct objcore *oc;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -407,8 +394,10 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 		http_Unset(bo->bereq, "\012X-Varnish:");
 
 	http_PrintfHeader(bo->bereq, "X-Varnish: %ju", VXID(bo->vsl->wid));
-
-	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL);
+	if (!bo->retried_stale)
+		VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL);
+	else
+		wrk->vpi->handling = VCL_RET_FETCH;
 
 	if (wrk->vpi->handling == VCL_RET_ABANDON ||
 	    wrk->vpi->handling == VCL_RET_FAIL)
@@ -485,14 +474,49 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	AZ(bo->do_esi);
 	AZ(bo->was_304);
 
-	if (http_IsStatus(bo->beresp, 304) && vbf_304_logic(bo) < 0)
-		return (F_STP_ERROR);
+	if (http_IsStatus(bo->beresp, 304)) {
+		if (bo->stale_oc != NULL) {
+			retried_stale = bo->retried_stale;
+			VCL_backend_refresh_method(bo->vcl, wrk, NULL, bo, NULL);
+			bo->was_304 = 1;
+			switch (wrk->vpi->handling) {
+					case VCL_RET_MERGE:
+						vbf_304_logic(bo);
+						break;
+					case VCL_RET_BERESP:
+						break;
+					case VCL_RET_OBJ_STALE:
+						HTTP_Decode(bo->beresp, ObjGetAttr(bo->wrk, bo->stale_oc, OA_HEADERS, NULL));
+						break;
+					case VCL_RET_RETRY:
+						if (retried_stale == 1) {
+							VSLb(bo->vsl, SLT_VCL_Error,
+							    "Conditional fetch already retried, delivering 503");
+							return (F_STP_ERROR);
+						}
+						/* FALLTHROUGH */
+					case VCL_RET_ERROR:
+					case VCL_RET_ABANDON:
+					case VCL_RET_FAIL:
+						skip_vbr = 1;
+						break;
+					default:
+						WRONG("Illegal return from vcl_backend_refresh{}");
+			}
+		} else if (!bo->uncacheable){
+			VSLb(bo->vsl, SLT_Error,
+			    "304 response but not conditional fetch");
+			bo->htc->doclose = SC_RX_BAD;
+			vbf_cleanup(bo);
+			return (F_STP_ERROR);
+		}
+	}
 
 	if (bo->htc != NULL && bo->htc->doclose == SC_NULL &&
 	    http_GetHdrField(bo->bereq, H_Connection, "close", NULL))
 		bo->htc->doclose = SC_REQ_CLOSE;
-
-	VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL);
+	if (!skip_vbr)
+		VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL);
 
 	if (bo->htc != NULL && bo->htc->doclose == SC_NULL &&
 	    http_GetHdrField(bo->beresp, H_Connection, "close", NULL))

bin/varnishd/cache/cache_vrt.c

@@ -114,6 +114,15 @@ VRT_synth(VRT_CTX, VCL_INT code, VCL_STRING reason)
 	    : http_Status2Reason(ctx->req->err_code % 1000, NULL);
 }
 
+VCL_VOID
+VRT_retry_fetch(VRT_CTX)
+{
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
+
+	ctx->bo->retried_stale = 1;
+}
+
 /*--------------------------------------------------------------------*/
 
 void
@@ -257,6 +266,7 @@ VRT_selecthttp(VRT_CTX, enum gethdr_e where)
 	case HDR_RESP:
 		hp = ctx->http_resp;
 		break;
+	case HDR_OBJ_STALE:
 	case HDR_OBJ:
 		hp = NULL;
 		break;
@@ -281,6 +291,13 @@ VRT_GetHdr(VRT_CTX, VCL_HEADER hs)
 		return (HTTP_GetHdrPack(ctx->req->wrk, ctx->req->objcore,
 		    hs->what));
 	}
+
+	if (hs->where == HDR_OBJ_STALE) {
+		CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
+		CHECK_OBJ_NOTNULL(ctx->bo->stale_oc, OBJCORE_MAGIC);
+		return (HTTP_GetHdrPack(ctx->bo->wrk, ctx->bo->stale_oc,
+		    hs->what));
+	}
 	hp = VRT_selecthttp(ctx, hs->where);
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 	if (!http_GetHdr(hp, hs->what, &p))