diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000000..947c32ebb92 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: nigoroll diff --git a/README.rst b/README.rst index 20db8b20d1a..e8d5ab44f55 100644 --- a/README.rst +++ b/README.rst @@ -1,12 +1,48 @@ -.. - Copyright (c) 2016-2020 Varnish Software AS - SPDX-License-Identifier: BSD-2-Clause - See LICENSE file for full text of license - Varnish Cache ============= -This is Varnish Cache, the high-performance HTTP accelerator. +This is Varnish Cache, the high-performance HTTP accelerator with +additional features and fixes. + +How this Repository is Organized +-------------------------------- + +The ``unmerged_code`` branch is created by merging feature/bug fix +branches onto `Varnish Cache master`_. These branches are usually for +`Varnish Cache Pull Requests`_. + +.. _Varnish Cache master: https://github.com/varnishcache/varnish-cache/tree/master +.. _Varnish Cache Pull Requests: https://github.com/varnishcache/varnish-cache/pulls +*NOTE* The ``unmerged_code`` branch gets force-pushed to +github. Individual releases of this repository are published as +branches named ``unmerged_code_``\ *
*\ ``_``\ +**. + +* Users wishing to use the lastest code should run:: + + $origin=${remote} + git pull + git reset --hard ${remote}/unmerged_code + +* Alternatively, pull the respective release branch. + +VMOD Compatibility +------------------ + +Changes merged in this branch might break existing interfaces and thus +require changes to VMODs. We make sure that this branch works with a +set of VMODs and create branches where necessary. + +The list of VMODs which we support and the respective repositories and +branches is contained in the file ``VMODS.json``. + +When we create a release branch, we add ``VMODS.commits.json`` to +contain a list of VMOD commit ids which have been successfully built +with the release branch. + + +General Information +------------------- Documentation and additional information about Varnish is available on https://www.varnish-cache.org/ @@ -15,15 +51,3 @@ Technical questions about Varnish and this release should be addressed to . Please see CONTRIBUTING for how to contribute patches and report bugs. - -For questions about commercial support and services related to Varnish -see the `Varnish HTTP Cache Business page -`_ . - -.. |ccibadge| image:: https://circleci.com/gh/varnishcache/varnish-cache/tree/master.svg?style=svg - :target: https://circleci.com/gh/varnishcache/varnish-cache/tree/master -.. _vtest: https://varnish-cache.org/vtest/ - -CircleCI tests: |ccibadge| - -More platforms are tested via vtest_ diff --git a/VMODS.json b/VMODS.json new file mode 100644 index 00000000000..e4b1e1df94f --- /dev/null +++ b/VMODS.json @@ -0,0 +1,122 @@ +[ + { + "vmod": "libvdfp-pipe", + "git": "https://code.uplex.de/uplex-varnish/libvdfp-pipe.git", + "branch": "master" + }, + { + "vmod": "libvdp-pesi", + "git": "https://code.uplex.de/uplex-varnish/libvdp-pesi.git", + "branch": "master" + }, + { + "vmod": "libvfp-brotli", + "git": "https://code.uplex.de/uplex-varnish/libvfp-brotli.git", + "branch": "master" + }, + { + "vmod": "libvmod-all_healthy", + "git": "https://code.uplex.de/uplex-varnish/libvmod-all_healthy.git", + "branch": "master" + }, + { + "vmod": "libvmod-blobdigest", + "git": "https://code.uplex.de/uplex-varnish/libvmod-blobdigest.git", + "branch": "master" + }, + { + "vmod": "libvmod-blobsynth", + "git": "https://code.uplex.de/uplex-varnish/libvmod-blobsynth.git", + "branch": "master" + }, + { + "vmod": "libvmod-cluster", + "git": "https://code.uplex.de/uplex-varnish/libvmod-cluster.git", + "branch": "master" + }, + { + "vmod": "libvmod-crypto", + "git": "https://code.uplex.de/uplex-varnish/libvmod-crypto.git", + "branch": "master" + }, + { + "vmod": "libvmod-dispatch", + "git": "https://code.uplex.de/uplex-varnish/libvmod-dispatch.git", + "branch": "master" + }, + { + "vmod": "libvmod-dns", + "git": "https://github.com/nigoroll/libvmod-dns.git", + "branch": "master" + }, + { + "vmod": "libvmod-dynamic", + "git": "https://github.com/nigoroll/libvmod-dynamic.git", + "branch": "proxy_via_6" + }, + { + "vmod": "libvmod-frozen", + "git": "https://code.uplex.de/uplex-varnish/libvmod-frozen.git", + "branch": "master" + }, + { + "vmod": "libvmod-gcrypt", + "git": "https://code.uplex.de/uplex-varnish/libvmod-gcrypt.git", + "branch": "master" + }, + { + "vmod": "libvmod-geoip2", + "git": "https://github.com/nigoroll/libvmod-geoip2.git", + "branch": "lookup_fields" + }, + { + "vmod": "libvmod-hoailona", + "git": "https://code.uplex.de/uplex-varnish/libvmod-hoailona.git", + "branch": "master" + }, + { + "vmod": "libvmod-querystring", + "git": "https://github.com/nigoroll/libvmod-querystring", + "branch": "master" + }, + { + "vmod": "libvmod-re", + "git": "https://code.uplex.de/uplex-varnish/libvmod-re.git", + "branch": "vre_capture" + }, + { + "vmod": "libvmod-re2", + "git": "https://code.uplex.de/uplex-varnish/libvmod-re2.git", + "branch": "master" + }, + { + "vmod": "libvmod-selector", + "git": "https://code.uplex.de/uplex-varnish/libvmod-selector.git", + "branch": "master" + }, + { + "vmod": "libvmod-tus", + "git": "https://code.uplex.de/uplex-varnish/libvmod-tus.git", + "branch": "master" + }, + { + "vmod": "libvmod-weightadjust", + "git": "https://code.uplex.de/uplex-varnish/libvmod-weightadjust.git", + "branch": "master" + }, + { + "vmod": "libvmod-xcounter", + "git": "https://github.com/xcir/libvmod-xcounter.git", + "branch": "master" + }, + { + "vmod": "varnish-modules", + "git": "https://github.com/nigoroll/varnish-modules.git", + "branch": "master" + }, + { + "vmod": "varnish-objvar", + "git": "https://code.uplex.de/uplex-varnish/varnish-objvar.git", + "branch": "master" + } +] diff --git a/build.py b/build.py new file mode 100755 index 00000000000..ba1279e2168 --- /dev/null +++ b/build.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3.9 + +# - build and test varnish-cache and a number of vmods from a JSON +# config file +# - generate a JSON output file with the git commit hash of each +# of the used repositories +# - leave no trace: The build happens entirely out-of-tree in +# temporary directories +# +# ARGS: [check] + +import json +import os +import sys + +from contextlib import contextmanager +from multiprocessing import cpu_count +from subprocess import run +from tempfile import TemporaryDirectory + + +def r(*args, **kwargs): + r = run(*args, **kwargs) + if r.returncode: + print("FAILED " + " ".join(*args)) + sys.exit(r.returncode) + return r + + +@contextmanager +def pushd(new_dir): + previous_dir = os.getcwd() + os.chdir(new_dir) + try: + yield + finally: + os.chdir(previous_dir) + + +def bootstrap_autogen_configure(srcdir, prefix, **kwargs): + """Run a dumb check if bootstrap looks like it would call configure + Dridi-style. + * If bootstrap calls configure, return the call to be made from the + builddir. + * Otherwise, call bootstrap or autogen and return the configure + call to be made from builddir + """ + + p = "--prefix=" + prefix + a = os.path.join(srcdir, "bootstrap") + + if os.path.isfile(a): + with open(a, "r") as file: + if "/configure" in file.read(): + return [a, p] + else: + a = os.path.join(srcdir, "autogen.sh") + + with pushd(srcdir): + r([a], **kwargs) + + a = os.path.join(srcdir, "configure") + return [a, p] + + +def make(configure, **kwargs): + r(configure, **kwargs) + r(["make", "-j", str(cpu_count() + 2)], **kwargs) + if len(sys.argv) >= 2 and sys.argv[1] == "check": + r(["make", "-j", str(cpu_count() * 15), "check"], **kwargs) + r(["make", "install"], **kwargs) + + +def build(srcdir, prefix, **kwargs): + configure = bootstrap_autogen_configure(srcdir, prefix, **kwargs) + run(["make", "maintainer-clean"]) + run(["make", "distclean"]) + with TemporaryDirectory() as builddir: + with pushd(str(builddir)): + make(configure, **kwargs) + + +def build_vmod(name, prefix): + env = os.environ + # only required by vmod_dispatch + env["VARNISHSRC"] = varnishsrc + env["PKG_CONFIG_PATH"] = os.path.join(prefix, "lib", "pkgconfig") + env["ACLOCAL_PATH"] = os.path.join(prefix, "share", "aclocal") + build(os.getcwd(), prefix, env=env) + + +# XXX how can we avoid special casing? +# XXX does not support out-of-tree build +def build_maxmind(prefix): + with pushd("libmaxminddb"): + configure = bootstrap_autogen_configure(os.getcwd(), prefix) + make(configure) + + +def clone_build_vmod(vmod, prefix): + with TemporaryDirectory() as gitdir: + with pushd(str(gitdir)): + r(["git", "clone", "--recursive", vmod["git"]]) + with pushd(vmod["vmod"]): + r(["git", "checkout", vmod["branch"]]) + sub = run(["git", "describe"], capture_output=True, text=True) + if sub.returncode: + sub = r(["git", "rev-parse", "--short", "HEAD"], + capture_output=True, text=True) + vmod["rev"] = sub.stdout.rstrip() + del vmod["branch"] + if "geoip2" in vmod["vmod"]: + build_maxmind(prefix) + build_vmod(vmod["vmod"], prefix) + + +def build_vmods(vmods, prefix): + for vmod in vmods: + clone_build_vmod(vmod, prefix) + + +varnishsrc = os.getcwd() +vmods = None +with open(os.path.join(varnishsrc, "VMODS.json"), "rb") as file: + vmods = json.loads(file.read()) + +with TemporaryDirectory() as prefix: + build(varnishsrc, str(prefix)) + build_vmods(vmods, str(prefix)) + +with open(os.path.join(varnishsrc, "VMODS_BUILT.json"), "w") as file: + json.dump(vmods, file, indent=4, sort_keys=True) diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index 1e5762be8c9..f0ad661830a 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -1417,6 +1417,17 @@ now When converted to STRING in expressions it returns a formatted timestamp like ``Tue, 20 Feb 2018 09:30:31 GMT`` + ``now`` remains stable for the duration of any built-in VCL + subroutine to make time-based calculations predictable and + avoid edge cases. + + In other words, even if considerable amounts of time are spent + in VCL, ``now`` will always represent the point in time when + the respective built-in VCL subroutine was entered. ``now`` is + thus not suitable for any kind of time measurements. See + :ref:`std.timestamp()`, :ref:`std.now()` and + :ref:`std.timed_call()` in :ref:`vmod_std(3)`. + vcl.trace Type: BOOL diff --git a/update_this_branch.sh b/update_this_branch.sh new file mode 100755 index 00000000000..f320448dde4 --- /dev/null +++ b/update_this_branch.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +# this script is used by @nigoroll to update this branch and the merged branches + +# to fetch +typeset -ra remotes=( + Algunenano + AlveElde + Dridi + asadsa92 + andrewwiik + bsdphk + carlosabalde + daghf + gquintard + hermunn + mbgrydeland + nigoroll + rezan + scn + slimhazard + stevendore +) + +# other people's PRs and own which we do not rebase +typeset -ra branches_norebase=( + +# alternative route by phk +# Dridi/issue_3114 # #3123 +# diverged too much and not semantically relevant +# Dridi/_type # #3158 +# needs rebase +# Dridi/vtc-tunnel # #3375 +# not ready +# Dridi/vsl-stable # #3468 +) + +typeset -ra branches=( + autoconf-2.71 # #3804 + proxy_via_6_vtp-preamble # #3128 + acl_merge # #3563 + v1l_reopen # old PR was turned down + + #vpi_wip # #3515 contained in next + vcl_trace # #3515 follow-up, no PR yet + std_now_timed_call # #3553 + vre_capture_fix # #3725 + + ## TO BE RE-DONE + #ws_highwater # #3285 +) + +typeset -r this_branch="$(git symbolic-ref --short HEAD)" + +typeset -r upstream_remote="origin" +typeset -r my_remote="nigoroll" + + +typeset -r save=/tmp/varnish_${this_brnach}.save.$$ +typeset -ra save_files=( + README.rst + update_this_branch.sh + VMODS.json + build.py + .github/FUNDING.yml +) +set -eux + +for r in "${remotes[@]}" ; do + git fetch $r +done + +for b in "${branches[@]}" ; do + git checkout "${b}" + git pull + if ! git rebase master ; then + echo SUBSHELL TO FIX + bash + fi + git push -f "${my_remote}" +done + +git checkout "${this_branch}" + +echo -n merge? +read y +if [[ "$y" != "y" ]] ; then + exit +fi + +rm -rf "${save}" +mkdir -p "${save}" +find "${save_files[@]}" | cpio -dump "${save}" +git checkout master +#git pull +git reset --hard "${upstream_remote}"/master +git checkout "${this_branch}" +git reset --hard master +git checkout "${this_branch}" +for b in "${branches_norebase[@]}" "${branches[@]}" ; do + if ! git merge --no-ff -m "merge $b" $b ; then + echo SUBSHELL TO FIX + bash + fi +done +( cd "${save}" && find "${save_files[@]}" | cpio -dump "${OLDPWD}" ) +mv VMODS.json VMODS.json. && \ + jq < VMODS.json. >VMODS.json 'sort_by(.vmod)' && \ + rm VMODS.json. +git add "${save_files[@]}" +git commit -am 'rebased and remerged' +./build.py +git add VMODS_BUILT.json +git commit -m 'vmod revisions successfully built' VMODS_BUILT.json +n=$(date +%Y%m%d_%H%M%S) +git checkout -b unmerged_code_"${n}" +git push "${my_remote}" unmerged_code_"${n}" +git checkout unmerged_code +rm -rf "${save}" +set +x +echo +echo DONE. when happy, issue: +echo +echo git push -f "${my_remote}" diff --git a/vmod/tests/std_b00001.vtc b/vmod/tests/std_b00001.vtc index e0bc5602371..6bba1c99f0f 100644 --- a/vmod/tests/std_b00001.vtc +++ b/vmod/tests/std_b00001.vtc @@ -1,4 +1,4 @@ -varnishtest "Test std.random()" +varnishtest "Test std.random(), std.now(), std.timed_call()" server s1 { rxreq @@ -6,14 +6,22 @@ server s1 { } -start varnish v1 -vcl+backend { + import vtc; import std; - sub vcl_deliver { + sub dice { set resp.http.rnd1 = std.random(0, 1); set resp.http.rnd2 = std.random(0, 10); set resp.http.rnd3 = std.random(8, 10); set resp.http.rnd4 = std.random(99, 100); } + + sub vcl_deliver { + set resp.http.t0 = std.integer(time=std.now()); + vtc.sleep(1s); + set resp.http.rolling-us = std.timed_call(dice) * 1000 * 1000; + set resp.http.t1 = std.integer(time=std.now()); + } } -start varnish v1 -cliok "debug.srandom" @@ -25,4 +33,6 @@ client c1 { expect resp.http.rnd2 == 0.390 expect resp.http.rnd3 == 8.585 expect resp.http.rnd4 == 99.636 + expect resp.http.t1 -gt resp.http.t0 + expect resp.http.rolling-us -gt 1 } -run diff --git a/vmod/vmod_std.c b/vmod/vmod_std.c index 8ce144eb9c9..78a288d57b5 100644 --- a/vmod/vmod_std.c +++ b/vmod/vmod_std.c @@ -370,3 +370,22 @@ vmod_ban_error(VRT_CTX) r = ""; return (r); } + +VCL_TIME v_matchproto_(td_std_now) +vmod_now(VRT_CTX) +{ + + (void) ctx; + return (VTIM_real()); +} + +VCL_DURATION v_matchproto_(td_std_timed_call) +vmod_timed_call(VRT_CTX, VCL_SUB sub) +{ + vtim_mono b; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + b = VTIM_mono(); + VRT_call(ctx, sub); + return (VTIM_mono() - b); +} diff --git a/vmod/vmod_std.vcc b/vmod/vmod_std.vcc index acd273c884e..cef9590c41b 100644 --- a/vmod/vmod_std.vcc +++ b/vmod/vmod_std.vcc @@ -659,6 +659,16 @@ Returns a textual error description of the last `std.ban()`_ call from the same task or the empty string if there either was no error or no `std.ban()`_ call. +$Function TIME now() + +Returns the current time. In contrast to the ``now`` built-in +variable, every call returns a new value. + +$Function DURATION timed_call(SUB) + +Call the given SUB and return a high precision measurement of the +execution time. + DEPRECATED functions ====================