From 59788086503a22ab4bcaff9d5aac496942869637 Mon Sep 17 00:00:00 2001 From: Walid Boudebouda Date: Mon, 2 Oct 2023 14:56:47 +0200 Subject: [PATCH 1/6] VCL: introduce sub vcl_backend_refresh This commit introduces a new sub vcl_backend_refresh method intended to allow allow a more flexible approach on object revalidation. --- bin/varnishd/builtin.vcl | 4 ++ doc/sphinx/reference/vcl_step.rst | 38 ++++++++++++++++ doc/sphinx/reference/vcl_var.rst | 76 +++++++++++++++---------------- 3 files changed, 80 insertions(+), 38 deletions(-) diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl index ed459ce137b..74dde825970 100644 --- a/bin/varnishd/builtin.vcl +++ b/bin/varnishd/builtin.vcl @@ -209,6 +209,10 @@ sub vcl_backend_response { return (deliver); } +sub vcl_backend_refresh { + return (merge); +} + sub vcl_builtin_backend_response { call vcl_beresp_range; if (bereq.uncacheable) { diff --git a/doc/sphinx/reference/vcl_step.rst b/doc/sphinx/reference/vcl_step.rst index ee07f616923..780c4e721eb 100644 --- a/doc/sphinx/reference/vcl_step.rst +++ b/doc/sphinx/reference/vcl_step.rst @@ -329,6 +329,44 @@ hashing the request method and/or request body. HEAD request can be satisfied from cached GET responses. +.. _vcl_backend_refresh: + +vcl_backend_refresh +~~~~~~~~~~~~~~~~~~~ + +This subroutine is called after a legitimate 304 response is received from +the backend so that you can chose how to handle the object revalidation, the default +is to apply the RFC logic which is to merge the new headers with the stale ones. + + | + | ``merge`` + | Merge the headers we got from the backend response with + | the ones we had in the cached object. + | + | ``obj_stale`` + | Return the cached object without updating it with the new + | response headers. + | + | ``beresp`` + | Return the beresp object as it is at the end of this subroutine + | without executing the RFC merge logic. + | + | ``fail`` + | see :ref:`fail` section above + | + | ``abandon`` + | see :ref:`abandon` section above + | + | ``retry`` + | Retry the backend transaction. Increases the `retries` counter. + | If the number of retries is higher than *max_retries*, + | control will be passed to :ref:`vcl_backend_error`. + | + | ``error(status code, reason)`` + | Transition to :ref:`vcl_backend_error` with ``beresp.status`` and + | ``beresp.reason`` being preset to the arguments of ``error()`` if + | arguments are provided. + .. _vcl_backend_response: vcl_backend_response diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index e895753b2d1..7d26cb4b2bf 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -938,7 +938,7 @@ beresp Type: HTTP - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh The entire backend response HTTP data structure, useful as argument to VMOD functions. @@ -949,7 +949,7 @@ beresp.age Type: DURATION - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh Default: Age header, or zero. @@ -962,7 +962,7 @@ beresp.backend Type: BACKEND - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh This is the backend we fetched from. If bereq.backend was set to a director, this will be the backend selected @@ -985,7 +985,7 @@ beresp.backend.name Type: STRING - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh Name of the backend this response was fetched from. Same as beresp.backend. @@ -1008,9 +1008,9 @@ beresp.do_esi Type: BOOL - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh - Writable from: vcl_backend_response, vcl_backend_error + Writable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh Default: ``false``. @@ -1029,9 +1029,9 @@ beresp.do_gunzip Type: BOOL - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh - Writable from: vcl_backend_response, vcl_backend_error + Writable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh Default: ``false``. @@ -1050,9 +1050,9 @@ beresp.do_gzip Type: BOOL - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh - Writable from: vcl_backend_response, vcl_backend_error + Writable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh Default: ``false``. @@ -1070,9 +1070,9 @@ beresp.do_stream Type: BOOL - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh - Writable from: vcl_backend_response, vcl_backend_error + Writable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh Default: ``true``. @@ -1155,9 +1155,9 @@ beresp.grace Type: DURATION - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh - Writable from: vcl_backend_response, vcl_backend_error + Writable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh Default: Cache-Control ``stale-while-revalidate`` directive, or ``default_grace`` parameter. @@ -1171,11 +1171,11 @@ beresp.http.* Type: HEADER - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh - Writable from: vcl_backend_response, vcl_backend_error + Writable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh - Unsettable from: vcl_backend_response, vcl_backend_error + Unsettable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh The HTTP headers returned from the server. @@ -1186,7 +1186,7 @@ beresp.http.content-length Type: HEADER - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh The content-length header field is protected, see protected_headers_. @@ -1195,7 +1195,7 @@ beresp.http.transfer-encoding Type: HEADER - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh The transfer-encoding header field is protected, see protected_headers_. @@ -1206,9 +1206,9 @@ beresp.keep Type: DURATION - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh - Writable from: vcl_backend_response, vcl_backend_error + Writable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh Default: ``default_keep`` parameter. @@ -1225,9 +1225,9 @@ beresp.proto ``VCL <= 4.0`` Type: STRING - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh - Writable from: vcl_backend_response, vcl_backend_error + Writable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh The HTTP protocol version the backend replied with. @@ -1238,7 +1238,7 @@ beresp.proto ``VCL >= 4.1`` Type: STRING - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh The HTTP protocol version the backend replied with. @@ -1249,9 +1249,9 @@ beresp.reason Type: STRING - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh - Writable from: vcl_backend_response, vcl_backend_error + Writable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh The HTTP status message returned by the server. @@ -1262,9 +1262,9 @@ beresp.status Type: INT - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh - Writable from: vcl_backend_response, vcl_backend_error + Writable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh The HTTP status code returned by the server. @@ -1277,9 +1277,9 @@ beresp.storage Type: STEVEDORE - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh - Writable from: vcl_backend_response, vcl_backend_error + Writable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh The storage backend to use to save this object. @@ -1288,9 +1288,9 @@ beresp.storage_hint ``VCL <= 4.0`` Type: STRING - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh - Writable from: vcl_backend_response, vcl_backend_error + Writable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh Deprecated since varnish 5.1 and discontinued since VCL @@ -1306,7 +1306,7 @@ beresp.time Type: TIME - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh When the backend headers were fully received just before ``vcl_backend_response {}`` was entered, or when @@ -1336,9 +1336,9 @@ beresp.ttl Type: DURATION - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh - Writable from: vcl_backend_response, vcl_backend_error + Writable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh Default: Cache-Control ``s-maxage`` or ``max-age`` directives, or a value computed from the Expires header's deadline, or the @@ -1353,9 +1353,9 @@ beresp.uncacheable Type: BOOL - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh - Writable from: vcl_backend_response, vcl_backend_error + Writable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh Inherited from bereq.uncacheable, see there. @@ -1373,7 +1373,7 @@ beresp.was_304 Type: BOOL - Readable from: vcl_backend_response, vcl_backend_error + Readable from: vcl_backend_response, vcl_backend_error, vcl_backend_refresh When ``true`` this indicates that we got a 304 response From b0c3ccbdd304d300e1958b40e259a5fc063d04a8 Mon Sep 17 00:00:00 2001 From: Walid Boudebouda Date: Sat, 14 Oct 2023 21:24:07 +0200 Subject: [PATCH 2/6] vrt_var: Generalize obj.* variables bindings --- bin/varnishd/cache/cache_vrt_var.c | 131 ++++++++++++++--------------- 1 file changed, 65 insertions(+), 66 deletions(-) diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 60409beca1d..2ee23753e42 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -147,42 +147,6 @@ VRT_HDR_LR(beresp, reason, HTTP_HDR_REASON) VRT_STATUS_L(beresp) VRT_STATUS_R(beresp) -/*-------------------------------------------------------------------- - * Pulling things out of the packed object->http - */ - -VCL_INT -VRT_r_obj_status(VRT_CTX) -{ - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); - - return (HTTP_GetStatusPack(ctx->req->wrk, ctx->req->objcore)); -} - -VCL_STRING -VRT_r_obj_proto(VRT_CTX) -{ - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); - - return (HTTP_GetHdrPack(ctx->req->wrk, ctx->req->objcore, - H__Proto)); -} - -VCL_STRING -VRT_r_obj_reason(VRT_CTX) -{ - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); - - return (HTTP_GetHdrPack(ctx->req->wrk, ctx->req->objcore, - H__Reason)); -} - /*-------------------------------------------------------------------- * beresp bool-fields */ @@ -530,28 +494,73 @@ VRT_l_beresp_storage_hint(VRT_CTX, const char *str, VCL_STRANDS s) /*--------------------------------------------------------------------*/ -VCL_STEVEDORE -VRT_r_obj_storage(VRT_CTX) -{ - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); - AN(ctx->req->objcore->stobj); - CHECK_OBJ_NOTNULL(ctx->req->objcore->stobj->stevedore, - STEVEDORE_MAGIC); - return (ctx->req->objcore->stobj->stevedore); +#define VRT_OC_VAR_R(which, obj, obj_magic, field) \ +VCL_STEVEDORE \ +VRT_r_##which##_storage(VRT_CTX) \ +{ \ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ + CHECK_OBJ_NOTNULL(ctx->obj, obj_magic); \ + CHECK_OBJ_NOTNULL(ctx->obj->field, OBJCORE_MAGIC); \ + AN(ctx->obj->field->stobj); \ + CHECK_OBJ_NOTNULL(ctx->obj->field->stobj->stevedore, \ + STEVEDORE_MAGIC); \ + return (ctx->obj->field->stobj->stevedore); \ +} \ + \ +VCL_BOOL \ +VRT_r_##which##_can_esi(VRT_CTX) \ +{ \ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ + CHECK_OBJ_NOTNULL(ctx->obj, obj_magic); \ + CHECK_OBJ_NOTNULL(ctx->obj->field, OBJCORE_MAGIC); \ + return (ObjHasAttr(ctx->obj->wrk, ctx->obj->field, \ + OA_ESIDATA)); \ +} \ + \ +VCL_BOOL \ +VRT_r_##which##_uncacheable(VRT_CTX) \ +{ \ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ + CHECK_OBJ_NOTNULL(ctx->obj, obj_magic); \ + CHECK_OBJ_NOTNULL(ctx->obj->field, OBJCORE_MAGIC); \ + \ + return (ctx->obj->field->flags & OC_F_HFM ? 1 : 0); \ +} \ + \ +VCL_INT \ +VRT_r_##which##_status(VRT_CTX) \ +{ \ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ + CHECK_OBJ_NOTNULL(ctx->obj, obj_magic); \ + CHECK_OBJ_NOTNULL(ctx->obj->field, OBJCORE_MAGIC); \ + \ + return (HTTP_GetStatusPack(ctx->obj->wrk, \ + ctx->obj->field)); \ +} \ + \ +VCL_STRING \ +VRT_r_##which##_proto(VRT_CTX) \ +{ \ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ + CHECK_OBJ_NOTNULL(ctx->obj, obj_magic); \ + CHECK_OBJ_NOTNULL(ctx->obj->field, OBJCORE_MAGIC); \ + \ + return (HTTP_GetHdrPack(ctx->obj->wrk, ctx->obj->field, \ + H__Proto)); \ +} \ + \ +VCL_STRING \ +VRT_r_##which##_reason(VRT_CTX) \ +{ \ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ + CHECK_OBJ_NOTNULL(ctx->obj, obj_magic); \ + CHECK_OBJ_NOTNULL(ctx->obj->field, OBJCORE_MAGIC); \ + \ + return (HTTP_GetHdrPack(ctx->obj->wrk, ctx->obj->field, \ + H__Reason)); \ } -/*--------------------------------------------------------------------*/ - -VCL_BOOL -VRT_r_obj_can_esi(VRT_CTX) -{ - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); - return (ObjHasAttr(ctx->req->wrk, ctx->req->objcore, OA_ESIDATA)); -} +VRT_OC_VAR_R(obj, req, REQ_MAGIC, objcore); /*--------------------------------------------------------------------*/ @@ -989,16 +998,6 @@ VRT_r_obj_hits(VRT_CTX) return (ctx->req->is_hit ? ctx->req->objcore->hits : 0); } -VCL_BOOL -VRT_r_obj_uncacheable(VRT_CTX) -{ - - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); - return (ctx->req->objcore->flags & OC_F_HFM ? 1 : 0); -} - /*--------------------------------------------------------------------*/ VCL_BOOL From b4b91fadedb0104c8bbc5c0397962f61b6f6ec44 Mon Sep 17 00:00:00 2001 From: Walid Boudebouda Date: Sat, 14 Oct 2023 20:07:44 +0200 Subject: [PATCH 3/6] VCL: Introduce new obj_stale variable obj_stale variable gives access to the stale object we had in cache when doing a 304 revalidation. It is only readable from vcl_backend_refresh subroutine. --- bin/varnishd/cache/cache.h | 1 + bin/varnishd/cache/cache_vrt.c | 8 ++ bin/varnishd/cache/cache_vrt_var.c | 39 +++++++ doc/sphinx/reference/vcl_var.rst | 180 +++++++++++++++++++++++++++++ include/vrt.h | 16 ++- 5 files changed, 243 insertions(+), 1 deletion(-) diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 10fab7e8d53..9f489cbf0f8 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -388,6 +388,7 @@ struct busyobj { * is recycled. */ unsigned retries; + unsigned retried_stale; struct req *req; struct sess *sp; struct worker *wrk; diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 7f4378fb98f..b171b5945c5 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -257,6 +257,7 @@ VRT_selecthttp(VRT_CTX, enum gethdr_e where) case HDR_RESP: hp = ctx->http_resp; break; + case HDR_OBJ_STALE: case HDR_OBJ: hp = NULL; break; @@ -281,6 +282,13 @@ VRT_GetHdr(VRT_CTX, VCL_HEADER hs) return (HTTP_GetHdrPack(ctx->req->wrk, ctx->req->objcore, hs->what)); } + + if (hs->where == HDR_OBJ_STALE) { + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo->stale_oc, OBJCORE_MAGIC); + return (HTTP_GetHdrPack(ctx->bo->wrk, ctx->bo->stale_oc, + hs->what)); + } hp = VRT_selecthttp(ctx, hs->where); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); if (!http_GetHdr(hp, hs->what, &p)) diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 2ee23753e42..e203ce43756 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -561,6 +561,7 @@ VRT_r_##which##_reason(VRT_CTX) \ } VRT_OC_VAR_R(obj, req, REQ_MAGIC, objcore); +VRT_OC_VAR_R(obj_stale, bo, BUSYOBJ_MAGIC, stale_oc); /*--------------------------------------------------------------------*/ @@ -789,9 +790,13 @@ VRT_r_##which##_##fld(VRT_CTX) \ /*lint -save -e835 */ // Zero right hand arg to '-' +VRT_DO_EXP_R(obj_stale, ctx->bo->stale_oc, ttl, + ttl_now(ctx) - ctx->bo->stale_oc->t_origin) VRT_DO_EXP_R(obj, ctx->req->objcore, ttl, ttl_now(ctx) - ctx->req->objcore->t_origin) +VRT_DO_EXP_R(obj_stale, ctx->bo->stale_oc, grace, 0) VRT_DO_EXP_R(obj, ctx->req->objcore, grace, 0) +VRT_DO_EXP_R(obj_stale, ctx->bo->stale_oc, keep, 0) VRT_DO_EXP_R(obj, ctx->req->objcore, keep, 0) VRT_DO_EXP_L(beresp, ctx->bo->fetch_objcore, ttl, ttl_now(ctx) - ctx->bo->fetch_objcore->t_origin) @@ -823,6 +828,7 @@ VRT_DO_TIME_R(resp, req, t_resp) VRT_DO_TIME_R(bereq, bo, t_first) VRT_DO_TIME_R(beresp, bo, t_resp) VRT_DO_TIME_R(obj, req->objcore, t_origin) +VRT_DO_TIME_R(obj_stale, bo->stale_oc, t_origin) /*-------------------------------------------------------------------- */ @@ -837,6 +843,7 @@ VRT_r_##which##_##age(VRT_CTX) \ return (ttl_now(ctx) - oc->t_origin); \ } +VRT_DO_AGE_R(obj_stale, ctx->bo->stale_oc) VRT_DO_AGE_R(obj, ctx->req->objcore) VRT_DO_AGE_R(beresp, ctx->bo->fetch_objcore) @@ -1000,6 +1007,38 @@ VRT_r_obj_hits(VRT_CTX) /*--------------------------------------------------------------------*/ +VCL_INT +VRT_r_obj_stale_hits(VRT_CTX) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo->stale_oc, OBJCORE_MAGIC); + + return (ctx->bo->stale_oc->hits); +} + +VCL_BOOL +VRT_r_obj_stale_is_valid(VRT_CTX) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo->stale_oc, OBJCORE_MAGIC); + + return (!(ctx->bo->stale_oc->flags & OC_F_DYING)); +} + +VCL_BOOL +VRT_r_obj_stale_retried(VRT_CTX) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo->stale_oc, OBJCORE_MAGIC); + + return (ctx->bo->retried_stale); +} + +/*--------------------------------------------------------------------*/ + VCL_BOOL VRT_r_resp_is_streaming(VRT_CTX) { diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index 7d26cb4b2bf..7fe28e35d80 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -1380,6 +1380,186 @@ beresp.was_304 to our conditional fetch from the backend and turned that into ``beresp.status = 200`` +obj_stale +--------- + +This is the stale object we had in cache. It cannot be modified. + +.. _obj_stale.age: + +obj_stale.age + + Type: DURATION + + Readable from: vcl_backend_refresh + + The age of the stale object. + + +.. _obj_stale.can_esi: + +obj_stale.can_esi + + Type: BOOL + + Readable from: vcl_backend_refresh + + If the stale object can be ESI processed, that is if setting + ``resp.do_esi`` or adding ``esi`` to ``resp.filters`` in + ``vcl_deliver {}`` would cause the response body to be ESI + processed. + + +.. _obj_stale.grace: + +obj_stale.grace + + Type: DURATION + + Readable from: vcl_backend_refresh + + The stale object's grace period in seconds. + + +.. _obj_stale.hits: + +obj_stale.hits + + Type: INT + + Readable from: vcl_backend_refresh + + The count of cache-hits on this stale object. + + In `vcl_deliver` a value of 0 indicates a cache miss. + + +.. _obj_stale.http: + +obj_stale.http.* + + Type: HEADER + + Readable from: vcl_backend_refresh + + The HTTP headers stored in the stale object. + + See req.http_ for general notes. + + +.. _obj_stale.keep: + +obj_stale.keep + + Type: DURATION + + Readable from: vcl_backend_refresh + + The stale object's keep period in seconds. + + +.. _obj_stale.proto: + +obj_stale.proto + + Type: STRING + + Readable from: vcl_backend_refresh + + The HTTP protocol version stored in the stale object. + + +.. _obj_stale.reason: + +obj_stale.reason + + Type: STRING + + Readable from: vcl_backend_refresh + + + The HTTP reason phrase stored in the stale object. + + +.. _obj_stale.status: + +obj_stale.status + + Type: INT + + Readable from: vcl_backend_refresh + + + The HTTP status code stored in the stale object. + + More information in the `HTTP response status`_ section. + + +.. _obj_stale.storage: + +obj_stale.storage + + Type: STEVEDORE + + Readable from: vcl_backend_refresh + + The storage backend where this stale object is stored. + + +.. _obj_stale.time: + +obj_stale.time + + Type: TIME + + Readable from: vcl_backend_refresh + + The time the stale object was created from the perspective of the + server which generated it. This will roughly be equivalent to + ``now`` - ``obj.age``. + + +.. _obj_stale.ttl: + +obj_stale.ttl + + Type: DURATION + + Readable from: vcl_backend_refresh + + The stale object's time to live, in seconds. + + +.. _obj_stale.uncacheable: + +obj_stale.uncacheable + + Type: BOOL + + Readable from: vcl_backend_refresh + + Whether the stale object is uncacheable (pass, hit-for-pass or + hit-for-miss). + +.. _obj_stale.is_valid: + +obj_stale.is_valid + + Type: BOOL + + Readable from: vcl_backend_refresh + + Whether the stale object is still valid. + +.. _obj_stale.retried: + +obj_stale.retried + + Type: BOOL + + Readable from: vcl_backend_refresh + + Whether a fetch of this stale object was already retried. obj --- diff --git a/include/vrt.h b/include/vrt.h index 7d573f3e2f7..300e5ff41aa 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -58,6 +58,19 @@ * binary/load-time compatible, increment MAJOR version * * NEXT (2024-09-15) + * VRT_r_obj_stale_age() added + * VRT_r_obj_stale_can_esi() added + * VRT_r_obj_stale_grace() added + * VRT_r_obj_stale_hits() added + * VRT_r_obj_stale_keep() added + * VRT_r_obj_stale_proto() added + * VRT_r_obj_stale_reason() added + * VRT_r_obj_stale_status() added + * VRT_r_obj_stale_storage() added + * VRT_r_obj_stale_time() added + * VRT_r_obj_stale_ttl() added + * VRT_r_obj_stale_uncacheable() added + * enum gethdr_e has new value HDR_OBJ_STALE * struct vrt_backend.backend_wait_timeout added * struct vrt_backend.backend_wait_limit added * 19.1 (2024-05-27) @@ -675,7 +688,8 @@ enum gethdr_e { HDR_RESP, HDR_OBJ, HDR_BEREQ, - HDR_BERESP + HDR_BERESP, + HDR_OBJ_STALE }; struct gethdr_s { From 1a24fe51fb96ba7cae118f6146956930247dc8f0 Mon Sep 17 00:00:00 2001 From: Walid Boudebouda Date: Fri, 23 Aug 2024 18:23:12 +0200 Subject: [PATCH 4/6] VCL: Introduce "return retry(fetch)" --- bin/varnishd/cache/cache_vrt.c | 9 +++++++++ include/vrt.h | 4 +++- lib/libvcc/vcc_action.c | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index b171b5945c5..abb2bdbc7a5 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -114,6 +114,15 @@ VRT_synth(VRT_CTX, VCL_INT code, VCL_STRING reason) : http_Status2Reason(ctx->req->err_code % 1000, NULL); } +VCL_VOID +VRT_retry_fetch(VRT_CTX) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + + ctx->bo->retried_stale = 1; +} + /*--------------------------------------------------------------------*/ void diff --git a/include/vrt.h b/include/vrt.h index 300e5ff41aa..df6f1c1e574 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -58,7 +58,8 @@ * binary/load-time compatible, increment MAJOR version * * NEXT (2024-09-15) - * VRT_r_obj_stale_age() added + * VRT_retry_fetch() Added + * VRT_r_obj_stale_age() added * VRT_r_obj_stale_can_esi() added * VRT_r_obj_stale_grace() added * VRT_r_obj_stale_hits() added @@ -719,6 +720,7 @@ VCL_BYTES VRT_CacheReqBody(VRT_CTX, VCL_BYTES maxsize); VCL_STRING VRT_ban_string(VRT_CTX, VCL_STRING); VCL_INT VRT_purge(VRT_CTX, VCL_DURATION, VCL_DURATION, VCL_DURATION); VCL_VOID VRT_synth(VRT_CTX, VCL_INT, VCL_STRING); +VCL_VOID VRT_retry_fetch(VRT_CTX); VCL_VOID VRT_hit_for_pass(VRT_CTX, VCL_DURATION); VCL_BOOL VRT_ValidHdr(VRT_CTX, VCL_STRANDS); diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c index f7c2ce29fbb..b545db5ecb9 100644 --- a/lib/libvcc/vcc_action.c +++ b/lib/libvcc/vcc_action.c @@ -263,6 +263,23 @@ vcc_act_return_pass(struct vcc *tl) Fb(tl, 1, ");\n"); tl->indent -= INDENT; } + +/*--------------------------------------------------------------------*/ + +static void +vcc_act_return_retry(struct vcc *tl) +{ + SkipToken(tl, '('); + if (!vcc_IdIs(tl->t, "fetch")) { + VSB_cat(tl->sb, "Expected: return (retry(fetch))\n"); + vcc_ErrWhere(tl, tl->t); + return; + } + Fb(tl, 1, "VRT_retry_fetch(ctx);\n"); + vcc_NextToken(tl); + SkipToken(tl, ')'); +} + /*--------------------------------------------------------------------*/ static void @@ -390,6 +407,8 @@ vcc_act_return(struct vcc *tl, struct token *t, struct symbol *sym) vcc_act_return_pass(tl); else if (hand == VCL_RET_FAIL) vcc_act_return_fail(tl); + else if (hand == VCL_RET_RETRY) + vcc_act_return_retry(tl); else { VSB_cat(tl->sb, "Arguments not allowed.\n"); vcc_ErrWhere(tl, tl->t); From faae012fc5ef1d39c8837444a37623db7e218a2d Mon Sep 17 00:00:00 2001 From: Walid Boudebouda Date: Mon, 26 Aug 2024 14:01:37 +0200 Subject: [PATCH 5/6] VBF: Move 304 logic to builtin VCL --- bin/varnishd/builtin.vcl | 39 +++++++++- bin/varnishd/cache/cache_fetch.c | 90 +++++++++++++--------- bin/varnishtest/tests/b00089.vtc | 123 +++++++++++++++++++++++++++++++ bin/varnishtest/tests/b00090.vtc | 78 ++++++++++++++++++++ bin/varnishtest/tests/r01672.vtc | 6 ++ 5 files changed, 302 insertions(+), 34 deletions(-) create mode 100644 bin/varnishtest/tests/b00089.vtc create mode 100644 bin/varnishtest/tests/b00090.vtc diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl index 74dde825970..41f6dc51f43 100644 --- a/bin/varnishd/builtin.vcl +++ b/bin/varnishd/builtin.vcl @@ -210,7 +210,44 @@ sub vcl_backend_response { } sub vcl_backend_refresh { - return (merge); + call vcl_builtin_backend_refresh; + return (merge); +} + +sub vcl_builtin_backend_refresh { + call vcl_refresh_valid; + call vcl_refresh_conditions; + call vcl_refresh_status; +} + +sub vcl_refresh_valid { + if (obj_stale.retried) { # read-only, analogous to bereq.retries, but BOOL + return (error); + } + if (!obj_stale.is_valid) { + call vcl_refresh_retry; + } +} + +sub vcl_refresh_status { + if (obj_stale.status != 200) { + call vcl_refresh_retry; + } +} + +sub vcl_refresh_conditions { + if (!bereq.http.if-modified-since && + !bereq.http.if-none-match) { + return (error); + } +} + +sub vcl_refresh_retry { + unset bereq.http.if-modified-since; + unset bereq.http.if-none-match; + # Same transition as return (fetch) from vcl_backend_fetch, + # but turns into an error if obj_stale.retried already true. + return (retry(fetch)); } sub vcl_builtin_backend_response { diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 76a819c2c61..3c85770bda9 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -350,37 +350,24 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo) * 304 setup logic */ -static int +static void vbf_304_logic(struct busyobj *bo) { - if (bo->stale_oc != NULL && - ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) { - AZ(bo->stale_oc->flags & (OC_F_HFM|OC_F_PRIVATE)); - if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGCE)) { - /* - * If a VFP changed C-E in the stored - * object, then don't overwrite C-E from - * the IMS fetch, and we must weaken any - * new ETag we get. - */ - RFC2616_Weaken_Etag(bo->beresp); - } - http_Unset(bo->beresp, H_Content_Encoding); - http_Unset(bo->beresp, H_Content_Length); - HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp); - assert(http_IsStatus(bo->beresp, 200)); - bo->was_304 = 1; - } else if (!bo->uncacheable) { + + AZ(bo->stale_oc->flags & (OC_F_HFM|OC_F_PRIVATE)); + if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGCE)) { /* - * Backend sent unallowed 304 + * If a VFP changed C-E in the stored + * object, then don't overwrite C-E from + * the IMS fetch, and we must weaken any + * new ETag we get. */ - VSLb(bo->vsl, SLT_Error, - "304 response but not conditional fetch"); - bo->htc->doclose = SC_RX_BAD; - vbf_cleanup(bo); - return (-1); + RFC2616_Weaken_Etag(bo->beresp); } - return (1); + http_Unset(bo->beresp, H_Content_Encoding); + http_Unset(bo->beresp, H_Content_Length); + HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp); + assert(http_IsStatus(bo->beresp, 200)); } /*-------------------------------------------------------------------- @@ -392,7 +379,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) { int i; vtim_real now; - unsigned handling; + unsigned handling, retried_stale, skip_vbr = 0; struct objcore *oc; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -407,8 +394,10 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) http_Unset(bo->bereq, "\012X-Varnish:"); http_PrintfHeader(bo->bereq, "X-Varnish: %ju", VXID(bo->vsl->wid)); - - VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL); + if (!bo->retried_stale) + VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL); + else + wrk->vpi->handling = VCL_RET_FETCH; if (wrk->vpi->handling == VCL_RET_ABANDON || wrk->vpi->handling == VCL_RET_FAIL) @@ -485,14 +474,49 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) AZ(bo->do_esi); AZ(bo->was_304); - if (http_IsStatus(bo->beresp, 304) && vbf_304_logic(bo) < 0) - return (F_STP_ERROR); + if (http_IsStatus(bo->beresp, 304)) { + if (bo->stale_oc != NULL) { + retried_stale = bo->retried_stale; + VCL_backend_refresh_method(bo->vcl, wrk, NULL, bo, NULL); + bo->was_304 = 1; + switch (wrk->vpi->handling) { + case VCL_RET_MERGE: + vbf_304_logic(bo); + break; + case VCL_RET_BERESP: + break; + case VCL_RET_OBJ_STALE: + HTTP_Decode(bo->beresp, ObjGetAttr(bo->wrk, bo->stale_oc, OA_HEADERS, NULL)); + break; + case VCL_RET_RETRY: + if (retried_stale == 1) { + VSLb(bo->vsl, SLT_VCL_Error, + "Conditional fetch already retried, delivering 503"); + return (F_STP_ERROR); + } + /* FALLTHROUGH */ + case VCL_RET_ERROR: + case VCL_RET_ABANDON: + case VCL_RET_FAIL: + skip_vbr = 1; + break; + default: + WRONG("Illegal return from vcl_backend_refresh{}"); + } + } else if (!bo->uncacheable){ + VSLb(bo->vsl, SLT_Error, + "304 response but not conditional fetch"); + bo->htc->doclose = SC_RX_BAD; + vbf_cleanup(bo); + return (F_STP_ERROR); + } + } if (bo->htc != NULL && bo->htc->doclose == SC_NULL && http_GetHdrField(bo->bereq, H_Connection, "close", NULL)) bo->htc->doclose = SC_REQ_CLOSE; - - VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL); + if (!skip_vbr) + VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL); if (bo->htc != NULL && bo->htc->doclose == SC_NULL && http_GetHdrField(bo->beresp, H_Connection, "close", NULL)) diff --git a/bin/varnishtest/tests/b00089.vtc b/bin/varnishtest/tests/b00089.vtc new file mode 100644 index 00000000000..015e653b1c3 --- /dev/null +++ b/bin/varnishtest/tests/b00089.vtc @@ -0,0 +1,123 @@ +varnishtest "Test vcl_backend_refresh" + +server s1 { + rxreq + txresp -hdr "Etag: abcd" -hdr "from-bo: true" -bodylen 10 + + rxreq + expect req.http.if-none-match == "abcd" + txresp -status 304 -hdr "be304-1: true" + + rxreq + expect req.http.if-none-match == "abcd" + txresp -status 304 -hdr "be304-2: true" + + rxreq + expect req.http.if-none-match == "abcd" + txresp -status 304 -hdr "be304-3: true" +} -start + +varnish v1 -vcl+backend { + + sub vcl_backend_response { + set beresp.http.vbresp = "true"; + set beresp.ttl = 0.01s; + set beresp.grace = 0s; + set beresp.keep = 10m; + set beresp.http.was-304 = beresp.was_304; + } + + sub vcl_backend_refresh { + set beresp.http.vbref = "true"; + } + +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.http.was-304 == false + expect resp.http.vbref == + expect resp.http.vbresp == true + expect resp.http.from-bo == true +} -run + +delay 0.01 + +client c2 { + txreq + rxresp + expect resp.status == 200 + expect resp.http.was-304 == true + expect resp.http.be304-1 == true + expect resp.http.vbref == true + expect resp.http.vbresp == true + expect resp.http.from-bo == true +} -run + +varnish v1 -vcl+backend { + + sub vcl_backend_response { + set beresp.http.vbresp = "true"; + set beresp.ttl = 0.01s; + set beresp.grace = 0s; + set beresp.keep = 10m; + set beresp.http.was-304 = beresp.was_304; + } + + + sub vcl_refresh_status { + set beresp.http.vbref = "true"; + if (obj_stale.status == 200) { + return (obj_stale); + } + } +} + +delay 0.01 + +client c3 { + txreq + rxresp + expect resp.status == 200 + expect resp.http.was-304 == true + expect resp.http.be304-1 == true + expect resp.http.be304-2 == + expect resp.http.vbref == true + expect resp.http.vbresp == true + expect resp.http.from-bo == true +} -run + +varnish v1 -vcl+backend { + + sub vcl_backend_response { + set beresp.http.vbresp = "true"; + set beresp.ttl = 0.01s; + set beresp.grace = 0s; + set beresp.keep = 10m; + set beresp.http.was-304 = beresp.was_304; + } + + sub vcl_refresh_status { + set beresp.http.vbref = "true"; + if (obj_stale.status == 200) { + return (beresp); + } + } +} + +delay 0.01 + +client c4 { + txreq + rxresp + expect resp.status == 304 + expect resp.http.was-304 == true + expect resp.http.be304-1 == + expect resp.http.be304-2 == + expect resp.http.be304-3 == true + expect resp.http.vbref == true + expect resp.http.vbresp == true + expect resp.http.from-bo == +} -run diff --git a/bin/varnishtest/tests/b00090.vtc b/bin/varnishtest/tests/b00090.vtc new file mode 100644 index 00000000000..681da0fc32f --- /dev/null +++ b/bin/varnishtest/tests/b00090.vtc @@ -0,0 +1,78 @@ +varnishtest "Test obj_stale vcl variables" + +server s1 { + rxreq + txresp -hdr "Etag: abcd" -hdr "from-bo: true" -bodylen 10 + rxreq + expect req.http.if-none-match == "abcd" + txresp -status 304 +} -start + +varnish v1 -vcl+backend { + + sub vcl_backend_response { + set beresp.http.vbresp = "true"; + set beresp.ttl = 0.01s; + set beresp.grace = 0s; + set beresp.keep = 10m; + set beresp.http.was-304 = beresp.was_304; + } + + sub vcl_refresh_status { + if (obj_stale.status == 200) { + set beresp.http.vbref = "true"; + + set beresp.http.http = obj_stale.http.from-bo; + set beresp.http.age = obj_stale.age; + set beresp.http.can_esi = obj_stale.can_esi; + set beresp.http.grace = obj_stale.grace; + set beresp.http.hits = obj_stale.hits; + set beresp.http.keep = obj_stale.keep; + set beresp.http.proto = obj_stale.proto; + set beresp.http.reason = obj_stale.reason; + set beresp.http.status = obj_stale.status; + set beresp.http.storage = obj_stale.storage; + set beresp.http.time = obj_stale.time; + set beresp.http.ttl = obj_stale.ttl; + set beresp.http.uncacheable = obj_stale.uncacheable; + } + return (merge); + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + + expect resp.http.was-304 == false + expect resp.http.vbref == + expect resp.http.vbresp == true + expect resp.http.from-bo == true +} -run + +delay 0.01 + +client c2 { + txreq + rxresp + expect resp.status == 200 + expect resp.http.was-304 == true + expect resp.http.vbresp == true + expect resp.http.vbref == true + expect resp.http.from-bo == true + + expect resp.http.http == true + expect resp.http.age == 0 + expect resp.http.can_esi == false + expect resp.http.grace == 0.000 + expect resp.http.hits == 0 + expect resp.http.keep == 600.000 + expect resp.http.proto == HTTP/1.1 + expect resp.http.reason == OK + expect resp.http.status == 200 + expect resp.http.storage == storage.s0 + expect resp.http.time != + expect resp.http.ttl != + expect resp.http.uncacheable == false +} -run diff --git a/bin/varnishtest/tests/r01672.vtc b/bin/varnishtest/tests/r01672.vtc index e59670d7943..077de1a3ca4 100644 --- a/bin/varnishtest/tests/r01672.vtc +++ b/bin/varnishtest/tests/r01672.vtc @@ -17,6 +17,12 @@ varnish v1 -vcl+backend { set beresp.grace = 0s; set beresp.keep = 10s; } + + sub vcl_refresh_status { + if (obj_stale.status != 200) { + return (error); + } + } } -start client c1 { From dbdbf05f852e60ff251d414b450666f97955c2ce Mon Sep 17 00:00:00 2001 From: Walid Boudebouda Date: Tue, 10 Oct 2023 19:03:17 +0200 Subject: [PATCH 6/6] VBF: add vcl_backend_refresh to cache_fetch.dot --- doc/graphviz/cache_fetch.dot | 22 +- doc/graphviz/cache_fetch.svg | 498 +++++++++++++++++++++-------------- 2 files changed, 324 insertions(+), 196 deletions(-) diff --git a/doc/graphviz/cache_fetch.dot b/doc/graphviz/cache_fetch.dot index 96c0967a049..8c093452a16 100644 --- a/doc/graphviz/cache_fetch.dot +++ b/doc/graphviz/cache_fetch.dot @@ -43,9 +43,27 @@ digraph cache_fetch { ] v_b_f:error:s -> v_b_e v_b_f:fetch:s -> v_b_hdrs [style=bold] + RETRY_FETCH [shape=plaintext] + RETRY_FETCH -> v_b_hdrs [style=bold] v_b_hdrs [ label="send bereq,\nread beresp (headers)"] - v_b_hdrs -> v_b_r [style=bold] + v_b_hdrs -> v_b_r [label="\"no stale object\""] [style=bold] v_b_hdrs -> v_b_e + v_b_hdrs -> v_b_refresh [label="\"stale object\""] [style=bold] + + v_b_refresh [ + shape=record + label="{vbf_stp_startfetch:|{vcl_backend_refresh\{\}|{bereq.*|beresp.*}}|{error|fail|{retry|{max?|ok?}}|{retry(fetch)|{max?|ok?}}|abandon|merge|obj_stale|beresp}}" + ] + v_b_refresh:error:s -> v_b_e + v_b_refresh:retry -> v_b_r_retry [color=purple] + v_b_refresh:rf_retry -> v_b_r_retry_fetch [color=purple] + v_b_refresh:rf_max -> v_b_e + v_b_refresh:max -> v_b_e + + v_b_refresh:merge -> v_b_r + v_b_refresh:beresp -> v_b_r + v_b_refresh:stale -> v_b_r + v_b_r [ shape=record label="{vbf_stp_startfetch:|{vcl_backend_response\{\}|{bereq.*|beresp.*}}|{error|fail|{retry|{max?|ok?}}|abandon|{deliver or pass|{304?|other?}}}}" @@ -57,6 +75,8 @@ digraph cache_fetch { v_b_r:non_304:s -> vbf_stp_fetch v_b_r_retry [label="RETRY",shape=plaintext] + + v_b_r_retry_fetch [label="RETRY_FETCH",shape=plaintext] vbf_stp_fetchbody [ shape=record diff --git a/doc/graphviz/cache_fetch.svg b/doc/graphviz/cache_fetch.svg index fa09e056e73..0fad71a0410 100644 --- a/doc/graphviz/cache_fetch.svg +++ b/doc/graphviz/cache_fetch.svg @@ -1,338 +1,446 @@ - - - + + cache_fetch - + cluster_backend - + RETRY -RETRY +RETRY v_b_f - -vbf_stp_startfetch: - -vcl_backend_fetch{} - -bereq.* - -error - -fail - -abandon - -fetch + +vbf_stp_startfetch: + +vcl_backend_fetch{} + +bereq.* + +error + +fail + +abandon + +fetch RETRY->v_b_f - - + + v_b_f_BGFETCH - -BGFETCH + +BGFETCH v_b_f_BGFETCH->v_b_f - - + + v_b_f_FETCH - -FETCH + +FETCH v_b_f_FETCH->v_b_f - - + + v_b_f_FETCH->v_b_f - - + + v_b_e - -vbf_stp_error: - -vcl_backend_error{} - -bereq.* - -beresp.* - -retry - -fail - -max? - -ok? - -abandon - -deliver + +vbf_stp_error: + +vcl_backend_error{} + +bereq.* + +beresp.* + +retry + +fail + +max? + +ok? + +abandon + +deliver v_b_f:s->v_b_e - - + + v_b_hdrs - -send bereq, -read beresp (headers) + +send bereq, +read beresp (headers) v_b_f:s->v_b_hdrs - - + + - + FETCH_DONE - -FETCH_DONE + +FETCH_DONE - + v_b_e:deliver->FETCH_DONE - - -"backend synth" + + +"backend synth" - + FETCH_FAIL - -FETCH_FAIL + +FETCH_FAIL - + v_b_e:s->FETCH_FAIL - - + + - + v_b_e_retry -RETRY +RETRY - + v_b_e:retry->v_b_e_retry - - + + - + v_b_hdrs->v_b_e - - + + - + v_b_r - -vbf_stp_startfetch: - -vcl_backend_response{} - -bereq.* - -beresp.* - -error - -fail - -retry - -max? - -ok? - -abandon - -deliver or pass - -304? - -other? + +vbf_stp_startfetch: + +vcl_backend_response{} + +bereq.* + +beresp.* + +error + +fail + +retry + +max? + +ok? + +abandon + +deliver or pass + +304? + +other? - + v_b_hdrs->v_b_r - - + + +"no stale object" + + + +v_b_refresh + +vbf_stp_startfetch: + +vcl_backend_refresh{} + +bereq.* + +beresp.* + +error + +fail + +retry + +max? + +ok? + +retry(fetch) + +max? + +ok? + +abandon + +merge + +obj_stale + +beresp + + + +v_b_hdrs->v_b_refresh + + +"stale object" + + + +RETRY_FETCH +RETRY_FETCH + + + +RETRY_FETCH->v_b_hdrs + + - + v_b_r:s->v_b_e - - + + - + v_b_r:max->v_b_e - - + + - + v_b_r_retry -RETRY +RETRY - + v_b_r:retry->v_b_r_retry - - + + - + vbf_stp_condfetch - -vbf_stp_condfetch: - -copy obj attr - -steal body - -fetch_fail? - -ok? + +vbf_stp_condfetch: + +copy obj attr + +steal body + +fetch_fail? + +ok? - + v_b_r:s->vbf_stp_condfetch - - + + - + vbf_stp_fetch - -vbf_stp_fetch: - -setup VFPs - -get object - -error? - -body? + +vbf_stp_fetch: + +setup VFPs + +get object + +error? + +body? - + v_b_r:s->vbf_stp_fetch - - + + + + + +v_b_refresh:s->v_b_e + + + + + +v_b_refresh:rf_max->v_b_e + + + + + +v_b_refresh:max->v_b_e + + + + + +v_b_refresh:merge->v_b_r + + + + + +v_b_refresh:beresp->v_b_r + + + + + +v_b_refresh:stale->v_b_r + + + + + +v_b_refresh:retry->v_b_r_retry + + + + + +v_b_r_retry_fetch +RETRY_FETCH + + + +v_b_refresh:rf_retry->v_b_r_retry_fetch + + - + vbf_stp_fetchend - -vbf_stp_fetchend: - -finalize object and director - -done + +vbf_stp_fetchend: + +finalize object and director + +done - + vbf_stp_condfetch:s->vbf_stp_fetchend - - + + - + vbf_stp_fetchbody - -vbf_stp_fetchbody: - -get storage - -read body, run VFPs - -fetch_fail? - -error? - -ok? + +vbf_stp_fetchbody: + +get storage + +read body, run VFPs + +fetch_fail? + +error? + +ok? - + vbf_stp_fetch:s->vbf_stp_fetchbody - - + + - + vbf_stp_fetch:s->vbf_stp_fetchend - - + + - + vbf_stp_fetchbody:s->vbf_stp_fetchend - - + + - + vbf_stp_fetchend:s->FETCH_DONE - - + + - + fail -fail +fail - + fail->FETCH_FAIL - - + + - + abandon -abandon +abandon - + abandon->FETCH_FAIL - - + +