From 23dedc6de3b2dfe9e8ea4108ea384b1b57094cb1 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Sun, 14 May 2023 15:31:58 +0300 Subject: [PATCH 001/138] Fix GH link in docs --- docs/assets/docsify-init.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/assets/docsify-init.js b/docs/assets/docsify-init.js index 08a0a968c5..3b01c4e4c2 100644 --- a/docs/assets/docsify-init.js +++ b/docs/assets/docsify-init.js @@ -10,7 +10,7 @@ const documentTitleBase = document.title; const linksMenu = '
' + '' + - '' + + '' + '' + '' + '
'; From bc5ca9a344ee1c04d19664e51fcbc4453a48e329 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Mon, 15 May 2023 20:06:46 +0300 Subject: [PATCH 002/138] Add URL replacements --- CHANGELOG.md | 2 ++ config/config.go | 7 ++++++- config/configurators/configurators.go | 22 +++++++++++++++++++- docs/configuration.md | 11 +++++++++- options/processing_options_test.go | 29 +++++++++++++++++++++++++++ options/url.go | 10 ++++++--- 6 files changed, 75 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index daa0c20fef..d3bbc1957c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased] +### Add +- Add `IMGPROXY_URL_REPLACEMENTS` config. ## [3.17.0] - 2023-05-10 ### Add diff --git a/config/config.go b/config/config.go index b5438bcec4..b60c81e378 100644 --- a/config/config.go +++ b/config/config.go @@ -126,7 +126,8 @@ var ( LastModifiedEnabled bool - BaseURL string + BaseURL string + URLReplacements map[*regexp.Regexp]string Presets []string OnlyPresets bool @@ -317,6 +318,7 @@ func Reset() { LastModifiedEnabled = false BaseURL = "" + URLReplacements = make(map[*regexp.Regexp]string) Presets = make([]string, 0) OnlyPresets = false @@ -518,6 +520,9 @@ func Configure() error { configurators.Bool(&LastModifiedEnabled, "IMGPROXY_USE_LAST_MODIFIED") configurators.String(&BaseURL, "IMGPROXY_BASE_URL") + if err := configurators.Replacements(&URLReplacements, "IMGPROXY_URL_REPLACEMENTS"); err != nil { + return err + } configurators.StringSlice(&Presets, "IMGPROXY_PRESETS") if err := configurators.StringSliceFile(&Presets, presetsPath); err != nil { diff --git a/config/configurators/configurators.go b/config/configurators/configurators.go index 0079b6a4fe..945d21da6f 100644 --- a/config/configurators/configurators.go +++ b/config/configurators/configurators.go @@ -221,6 +221,26 @@ func Patterns(s *[]*regexp.Regexp, name string) { } } +func Replacements(m *map[*regexp.Regexp]string, name string) error { + var sm map[string]string + + if err := StringMap(&sm, name); err != nil { + return err + } + + if len(sm) > 0 { + mm := make(map[*regexp.Regexp]string) + + for k, v := range sm { + mm[RegexpFromPattern(k)] = v + } + + *m = mm + } + + return nil +} + func RegexpFromPattern(pattern string) *regexp.Regexp { var result strings.Builder // Perform prefix matching @@ -228,7 +248,7 @@ func RegexpFromPattern(pattern string) *regexp.Regexp { for i, part := range strings.Split(pattern, "*") { // Add a regexp match all without slashes for each wildcard character if i > 0 { - result.WriteString("[^/]*") + result.WriteString("([^/]*)") } // Quote other parts of the pattern diff --git a/docs/configuration.md b/docs/configuration.md index 4b4ae67658..0d3bcb713f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -406,6 +406,16 @@ imgproxy can process files from OpenStack Object Storage, but this feature is di Check out the [Serving files from OpenStack Object Storage](serving_files_from_openstack_swift.md) guide to learn more. +## Source image URLs + +* `IMGPROXY_BASE_URL`: a base URL prefix that will be added to each source image URL. For example, if the base URL is `http://example.com/images` and `/path/to/image.png` is requested, imgproxy will download the source image from `http://example.com/images/path/to/image.png`. If the image URL already contains the prefix, it won't be added. Default: blank + +* `IMGPROXY_URL_REPLACEMENTS`: a list of `pattern=replacement` pairs, semicolon (`;`) divided. imgproxy will replace source URL prefixes matching the pattern with the corresponding replacement. Wildcards can be included in patterns with `*` to match all characters except `/`. `${N}` in replacement strings will be replaced with wildcard values, where `N` is the number of the wildcard. Examples: + * `mys3://=s3://my_bucket/images/` will replace `mys3://image01.jpg` with `s3://my_bucket/images/image01.jpg` + * `mys3://*/=s3://my_bucket/${1}/images` will replace `mys3://items/image01.jpg` with `s3://my_bucket/items/images/image01.jpg` + +**📝 Note:** Replacements defined in `IMGPROXY_URL_REPLACEMENTS` are applied before `IMGPROXY_BASE_URL` is added. + ## Metrics ### New Relic :id=new-relic-metrics @@ -527,7 +537,6 @@ imgproxy can send logs to syslog, but this feature is disabled by default. To en ## Miscellaneous -* `IMGPROXY_BASE_URL`: a base URL prefix that will be added to each requested image URL. For example, if the base URL is `http://example.com/images` and `/path/to/image.png` is requested, imgproxy will download the source image from `http://example.com/images/path/to/image.png`. If the image URL already contains the prefix, it won't be added. Default: blank * `IMGPROXY_USE_LINEAR_COLORSPACE`: when `true`, imgproxy will process images in linear colorspace. This will slow down processing. Note that images won't be fully processed in linear colorspace while shrink-on-load is enabled (see below). * `IMGPROXY_DISABLE_SHRINK_ON_LOAD`: when `true`, disables shrink-on-load for JPEGs and WebP files. Allows processing the entire image in linear colorspace but dramatically slows down resizing and increases memory usage when working with large images. * `IMGPROXY_STRIP_METADATA`: when `true`, imgproxy will strip all metadata (EXIF, IPTC, etc.) from JPEG and WebP output images. Default: `true` diff --git a/options/processing_options_test.go b/options/processing_options_test.go index 4727c04193..2a48fa5a2a 100644 --- a/options/processing_options_test.go +++ b/options/processing_options_test.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/url" + "regexp" "testing" "github.com/stretchr/testify/require" @@ -54,6 +55,20 @@ func (s *ProcessingOptionsTestSuite) TestParseBase64URLWithBase() { require.Equal(s.T(), imagetype.PNG, po.Format) } +func (s *ProcessingOptionsTestSuite) TestParseBase64URLWithReplacement() { + config.URLReplacements = map[*regexp.Regexp]string{ + regexp.MustCompile("^test://([^/]*)/"): "http://images.dev/${1}/dolor/", + } + + originURL := "test://lorem/ipsum.jpg?param=value" + path := fmt.Sprintf("/size:100:100/%s.png", base64.RawURLEncoding.EncodeToString([]byte(originURL))) + po, imageURL, err := ParsePath(path, make(http.Header)) + + require.Nil(s.T(), err) + require.Equal(s.T(), "http://images.dev/lorem/dolor/ipsum.jpg?param=value", imageURL) + require.Equal(s.T(), imagetype.PNG, po.Format) +} + func (s *ProcessingOptionsTestSuite) TestParsePlainURL() { originURL := "http://images.dev/lorem/ipsum.jpg" path := fmt.Sprintf("/size:100:100/plain/%s@png", originURL) @@ -96,6 +111,20 @@ func (s *ProcessingOptionsTestSuite) TestParsePlainURLWithBase() { require.Equal(s.T(), imagetype.PNG, po.Format) } +func (s *ProcessingOptionsTestSuite) TestParsePlainURLWithReplacement() { + config.URLReplacements = map[*regexp.Regexp]string{ + regexp.MustCompile("^test://([^/]*)/"): "http://images.dev/${1}/dolor/", + } + + originURL := "test://lorem/ipsum.jpg" + path := fmt.Sprintf("/size:100:100/plain/%s@png", originURL) + po, imageURL, err := ParsePath(path, make(http.Header)) + + require.Nil(s.T(), err) + require.Equal(s.T(), "http://images.dev/lorem/dolor/ipsum.jpg", imageURL) + require.Equal(s.T(), imagetype.PNG, po.Format) +} + func (s *ProcessingOptionsTestSuite) TestParsePlainURLEscapedWithBase() { config.BaseURL = "http://images.dev/" diff --git a/options/url.go b/options/url.go index 850b796736..b4e8e5cc95 100644 --- a/options/url.go +++ b/options/url.go @@ -12,7 +12,11 @@ import ( const urlTokenPlain = "plain" -func addBaseURL(u string) string { +func preprocessURL(u string) string { + for re, repl := range config.URLReplacements { + u = re.ReplaceAllString(u, repl) + } + if len(config.BaseURL) == 0 || strings.HasPrefix(u, config.BaseURL) { return u } @@ -43,7 +47,7 @@ func decodeBase64URL(parts []string) (string, string, error) { return "", "", fmt.Errorf("Invalid url encoding: %s", encoded) } - return addBaseURL(string(imageURL)), format, nil + return preprocessURL(string(imageURL)), format, nil } func decodePlainURL(parts []string) (string, string, error) { @@ -69,7 +73,7 @@ func decodePlainURL(parts []string) (string, string, error) { return "", "", fmt.Errorf("Invalid url encoding: %s", encoded) } - return addBaseURL(unescaped), format, nil + return preprocessURL(unescaped), format, nil } func DecodeURL(parts []string) (string, string, error) { From 0ea06c88c2534c798ebc62e96fe86b825f23b973 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Wed, 17 May 2023 13:41:30 +0300 Subject: [PATCH 003/138] Update base Docker image --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index fb8d1c64c6..95aeca064f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE_VERSION="v3.3.5" +ARG BASE_IMAGE_VERSION="v3.4.0" FROM darthsim/imgproxy-base:${BASE_IMAGE_VERSION} From aad0186b0a28bb5c79242cc4e3ed43dc3166a648 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Wed, 17 May 2023 13:42:30 +0300 Subject: [PATCH 004/138] Enable vector optimizations in vips --- vips/vips.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vips/vips.go b/vips/vips.go index 39949e336c..2dee4b683f 100644 --- a/vips/vips.go +++ b/vips/vips.go @@ -62,9 +62,7 @@ func Init() error { C.vips_concurrency_set(1) - // Vector calculations cause SIGSEGV sometimes when working with JPEG. - // It's better to disable it since profit it quite small - C.vips_vector_set_enabled(0) + C.vips_vector_set_enabled(1) if len(os.Getenv("IMGPROXY_VIPS_LEAK_CHECK")) > 0 { C.vips_leak_set(C.gboolean(1)) From aaaf1c706d755c5352b46fe0fd32691fa79740a7 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Wed, 17 May 2023 17:47:20 +0300 Subject: [PATCH 005/138] Normalize IMGPROXY_PATH_PREFIX and IMGPROXY_HEALTH_CHECK_PATH --- config/config.go | 4 ++-- config/configurators/configurators.go | 19 +++++++++++++++++++ healthcheck.go | 2 +- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/config/config.go b/config/config.go index b60c81e378..05ded60deb 100644 --- a/config/config.go +++ b/config/config.go @@ -406,7 +406,7 @@ func Configure() error { configurators.Bool(&SoReuseport, "IMGPROXY_SO_REUSEPORT") - configurators.String(&PathPrefix, "IMGPROXY_PATH_PREFIX") + configurators.URLPath(&PathPrefix, "IMGPROXY_PATH_PREFIX") configurators.MegaInt(&MaxSrcResolution, "IMGPROXY_MAX_SRC_RESOLUTION") configurators.Int(&MaxSrcFileSize, "IMGPROXY_MAX_SRC_FILE_SIZE") @@ -449,7 +449,7 @@ func Configure() error { configurators.Bool(&EnforceAvif, "IMGPROXY_ENFORCE_AVIF") configurators.Bool(&EnableClientHints, "IMGPROXY_ENABLE_CLIENT_HINTS") - configurators.String(&HealthCheckPath, "IMGPROXY_HEALTH_CHECK_PATH") + configurators.URLPath(&HealthCheckPath, "IMGPROXY_HEALTH_CHECK_PATH") if err := configurators.ImageTypes(&PreferredFormats, "IMGPROXY_PREFERRED_FORMATS"); err != nil { return err diff --git a/config/configurators/configurators.go b/config/configurators/configurators.go index 945d21da6f..de3052a1d2 100644 --- a/config/configurators/configurators.go +++ b/config/configurators/configurators.go @@ -102,6 +102,25 @@ func Bool(b *bool, name string) { } } +func URLPath(s *string, name string) { + if env := os.Getenv(name); len(env) > 0 { + if i := strings.IndexByte(env, '?'); i >= 0 { + env = env[:i] + } + if i := strings.IndexByte(env, '#'); i >= 0 { + env = env[:i] + } + if len(env) > 0 && env[len(env)-1] == '/' { + env = env[:len(env)-1] + } + if len(env) > 0 && env[0] != '/' { + env = "/" + env + } + + *s = env + } +} + func ImageTypes(it *[]imagetype.Type, name string) error { if env := os.Getenv(name); len(env) > 0 { parts := strings.Split(env, ",") diff --git a/healthcheck.go b/healthcheck.go index df69fd4acb..9843795e11 100644 --- a/healthcheck.go +++ b/healthcheck.go @@ -19,7 +19,7 @@ func healthcheck() int { configurators.String(&network, "IMGPROXY_NETWORK") configurators.String(&bind, "IMGPROXY_BIND") - configurators.String(&pathprefix, "IMGPROXY_PATH_PREFIX") + configurators.URLPath(&pathprefix, "IMGPROXY_PATH_PREFIX") httpc := http.Client{ Transport: &http.Transport{ From 219d37ee5ba1b4fb447ecb86e68694f6f0df1180 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Wed, 17 May 2023 20:25:45 +0300 Subject: [PATCH 006/138] Update base Docker image --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 95aeca064f..60b9d26076 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE_VERSION="v3.4.0" +ARG BASE_IMAGE_VERSION="v3.5.0" FROM darthsim/imgproxy-base:${BASE_IMAGE_VERSION} From ce24671ca0659fe8dd16b02615f2d9089e74b7f3 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Wed, 17 May 2023 20:26:20 +0300 Subject: [PATCH 007/138] Update NOTICE --- NOTICE | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/NOTICE b/NOTICE index 5f334ac22c..4bc6a2ebfb 100644 --- a/NOTICE +++ b/NOTICE @@ -5101,6 +5101,121 @@ zlib ================================================================================ +zlib-ng + + (C) 1995-2013 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source distribution. + +================================================================================ + +libffi + + libffi - Copyright (c) 1996-2022 Anthony Green, Red Hat, Inc and others. + See source files for details. + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + ``Software''), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================================ + +Orc + + The majority of the source code and the collective work is subject + to the following license: + + Copyright 2002 - 2009 David A. Schleef + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + + The source code implementing the Mersenne Twister algorithm is + subject to the following license: + + Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura, + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. The names of its contributors may not be used to endorse or promote + products derived from this software without specific prior written + permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================================ + libtiff Acknowledgments and Other Issues From 3048c30e1e1d14d471795837335d9f9d2171c5a1 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 23 May 2023 18:26:06 +0300 Subject: [PATCH 008/138] Cast to origin pixel format after premultiplication --- vips/vips.c | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/vips/vips.c b/vips/vips.c index a56eea6eb9..4178ff29b2 100644 --- a/vips/vips.c +++ b/vips/vips.c @@ -200,13 +200,14 @@ vips_resize_go(VipsImage *in, VipsImage **out, double wscale, double hscale) { VipsBandFormat format = vips_band_format(in); VipsImage *base = vips_image_new(); - VipsImage **t = (VipsImage **) vips_object_local_array(VIPS_OBJECT(base), 3); + VipsImage **t = (VipsImage **) vips_object_local_array(VIPS_OBJECT(base), 4); int res = vips_premultiply(in, &t[0], NULL) || - vips_resize(t[0], &t[1], wscale, "vscale", hscale, NULL) || - vips_unpremultiply(t[1], &t[2], NULL) || - vips_cast(t[2], out, format, NULL); + vips_cast(t[0], &t[1], format, NULL) || + vips_resize(t[1], &t[2], wscale, "vscale", hscale, NULL) || + vips_unpremultiply(t[2], &t[3], NULL) || + vips_cast(t[3], out, format, NULL); clear_image(&base); @@ -306,14 +307,17 @@ vips_apply_filters(VipsImage *in, VipsImage **out, double blur_sigma, double sharp_sigma, int pixelate_pixels) { VipsImage *base = vips_image_new(); - VipsImage **t = (VipsImage **) vips_object_local_array(VIPS_OBJECT(base), 9); + VipsImage **t = (VipsImage **) vips_object_local_array(VIPS_OBJECT(base), 10); VipsInterpretation interpretation = in->Type; VipsBandFormat format = in->BandFmt; gboolean premultiplied = FALSE; if ((blur_sigma > 0 || sharp_sigma > 0) && vips_image_hasalpha(in)) { - if (vips_premultiply(in, &t[0], NULL)) { + if ( + vips_premultiply(in, &t[0], NULL) || + vips_cast(t[0], &t[1], format, NULL) + ) { clear_image(&base); return 1; } @@ -323,21 +327,21 @@ vips_apply_filters(VipsImage *in, VipsImage **out, double blur_sigma, } if (blur_sigma > 0.0) { - if (vips_gaussblur(in, &t[1], blur_sigma, NULL)) { + if (vips_gaussblur(in, &t[2], blur_sigma, NULL)) { clear_image(&base); return 1; } - in = t[1]; + in = t[2]; } if (sharp_sigma > 0.0) { - if (vips_sharpen(in, &t[2], "sigma", sharp_sigma, NULL)) { + if (vips_sharpen(in, &t[3], "sigma", sharp_sigma, NULL)) { clear_image(&base); return 1; } - in = t[2]; + in = t[3]; } pixelate_pixels = VIPS_MIN(pixelate_pixels, VIPS_MAX(in->Xsize, in->Ysize)); @@ -352,46 +356,46 @@ vips_apply_filters(VipsImage *in, VipsImage **out, double blur_sigma, th = (int)(VIPS_CEIL((double)h / pixelate_pixels)) * pixelate_pixels; if (tw > w || th > h) { - if (vips_embed(in, &t[3], 0, 0, tw, th, "extend", VIPS_EXTEND_MIRROR, NULL)) { + if (vips_embed(in, &t[4], 0, 0, tw, th, "extend", VIPS_EXTEND_MIRROR, NULL)) { clear_image(&base); return 1; } - in = t[3]; + in = t[4]; } if ( - vips_shrink(in, &t[4], pixelate_pixels, pixelate_pixels, NULL) || - vips_zoom(t[4], &t[5], pixelate_pixels, pixelate_pixels, NULL) + vips_shrink(in, &t[5], pixelate_pixels, pixelate_pixels, NULL) || + vips_zoom(t[5], &t[6], pixelate_pixels, pixelate_pixels, NULL) ) { clear_image(&base); return 1; } - in = t[5]; + in = t[6]; if (tw > w || th > h) { - if (vips_extract_area(in, &t[6], 0, 0, w, h, NULL)) { + if (vips_extract_area(in, &t[7], 0, 0, w, h, NULL)) { clear_image(&base); return 1; } - in = t[6]; + in = t[7]; } } if (premultiplied) { - if (vips_unpremultiply(in, &t[7], NULL)) { + if (vips_unpremultiply(in, &t[8], NULL)) { clear_image(&base); return 1; } - in = t[7]; + in = t[8]; } int res = - vips_colourspace(in, &t[8], interpretation, NULL) || - vips_cast(t[8], out, format, NULL); + vips_colourspace(in, &t[9], interpretation, NULL) || + vips_cast(t[9], out, format, NULL); clear_image(&base); From 157843ccb32ac4c8d365e57c2081bdde278a2b15 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 23 May 2023 19:08:35 +0300 Subject: [PATCH 009/138] Use VIPS_META_BITS_PER_SAMPLE image header instead of palette-bit-depth when available --- processing/watermark.go | 2 +- vips/vips.c | 25 +++++++++++++++++-------- vips/vips.go | 4 ++-- vips/vips.h | 2 ++ 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/processing/watermark.go b/processing/watermark.go index db5ba8410e..23845a53de 100644 --- a/processing/watermark.go +++ b/processing/watermark.go @@ -69,7 +69,7 @@ func prepareWatermark(wm *vips.Image, wmData *imagedata.ImageData, opts *options } } - wm.RemoveHeader("palette-bit-depth") + wm.RemoveBitsPerSampleHeader() return nil } diff --git a/vips/vips.c b/vips/vips.c index 4178ff29b2..e6cce86ad4 100644 --- a/vips/vips.c +++ b/vips/vips.c @@ -15,6 +15,10 @@ #define VIPS_GIF_RESOLUTION_LIMITED \ (VIPS_MAJOR_VERSION == 8 && VIPS_MINOR_VERSION <= 12) +#ifndef VIPS_META_BITS_PER_SAMPLE +#define VIPS_META_BITS_PER_SAMPLE "palette-bit-depth" +#endif + int vips_initialize() { return vips_init("imgproxy"); @@ -134,17 +138,22 @@ vips_get_orientation(VipsImage *image) { } int -vips_get_palette_bit_depth(VipsImage *image) { - int palette_bit_depth; +vips_get_bits_per_sample(VipsImage *image) { + int bits_per_sample; if ( - vips_image_get_typeof(image, "palette-bit-depth") == G_TYPE_INT && - vips_image_get_int(image, "palette-bit-depth", &palette_bit_depth) == 0 - ) return palette_bit_depth; + vips_image_get_typeof(image, VIPS_META_BITS_PER_SAMPLE) == G_TYPE_INT && + vips_image_get_int(image, VIPS_META_BITS_PER_SAMPLE, &bits_per_sample) == 0 + ) return bits_per_sample; return 0; } +void +vips_remove_bits_per_sample(VipsImage *image) { + vips_image_remove(image, VIPS_META_BITS_PER_SAMPLE); +} + VipsBandFormat vips_band_format(VipsImage *in) { return in->BandFmt; @@ -610,7 +619,7 @@ vips_strip(VipsImage *in, VipsImage **out, int keep_exif_copyright) { if ( (strcmp(name, VIPS_META_ICC_NAME) == 0) || - (strcmp(name, "palette-bit-depth") == 0) || + (strcmp(name, VIPS_META_BITS_PER_SAMPLE) == 0) || (strcmp(name, "width") == 0) || (strcmp(name, "height") == 0) || (strcmp(name, "bands") == 0) || @@ -664,8 +673,8 @@ vips_pngsave_go(VipsImage *in, void **buf, size_t *len, int interlace, int quant else if (colors > 4) bitdepth = 4; else if (colors > 2) bitdepth = 2; } else { - bitdepth = vips_get_palette_bit_depth(in); - if (bitdepth) { + bitdepth = vips_get_bits_per_sample(in); + if (bitdepth && bitdepth <= 8) { if (bitdepth > 4) bitdepth = 8; else if (bitdepth > 2) bitdepth = 4; quantize = 1; diff --git a/vips/vips.go b/vips/vips.go index 2dee4b683f..954e0abcdf 100644 --- a/vips/vips.go +++ b/vips/vips.go @@ -482,8 +482,8 @@ func (img *Image) SetBlob(name string, value []byte) { C.vips_image_set_blob_copy(img.VipsImage, cachedCString(name), unsafe.Pointer(&value[0]), C.size_t(len(value))) } -func (img *Image) RemoveHeader(name string) { - C.vips_image_remove(img.VipsImage, cachedCString(name)) +func (img *Image) RemoveBitsPerSampleHeader() { + C.vips_remove_bits_per_sample(img.VipsImage) } func (img *Image) CastUchar() error { diff --git a/vips/vips.h b/vips/vips.h index 7d84539a8c..c4071442c0 100644 --- a/vips/vips.h +++ b/vips/vips.h @@ -28,6 +28,8 @@ void vips_strip_meta(VipsImage *image); VipsBandFormat vips_band_format(VipsImage *in); +void vips_remove_bits_per_sample(VipsImage * image); + gboolean vips_is_animated(VipsImage * in); int vips_image_get_array_int_go(VipsImage *image, const char *name, int **out, int *n); From 39c436e5273a6d2e0d5fe6627fb9602fb0e53ba2 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 23 May 2023 19:09:30 +0300 Subject: [PATCH 010/138] Preserve GIF bit-per-sample --- CHANGELOG.md | 3 +++ vips/vips.c | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3bbc1957c..137a61d284 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ ### Add - Add `IMGPROXY_URL_REPLACEMENTS` config. +### Change +- Preserve GIF's bit-per-sample. + ## [3.17.0] - 2023-05-10 ### Add - Add `process_resident_memory_bytes`, `process_virtual_memory_bytes`, `go_memstats_sys_bytes`, `go_memstats_heap_idle_bytes`, `go_memstats_heap_inuse_bytes`, `go_goroutines`, `go_threads`, `buffer_default_size_bytes`, `buffer_max_size_bytes`, and `buffer_size_bytes` metrics to OpenTelemetry. diff --git a/vips/vips.c b/vips/vips.c index e6cce86ad4..2a4cfb0a25 100644 --- a/vips/vips.c +++ b/vips/vips.c @@ -712,7 +712,9 @@ vips_webpsave_go(VipsImage *in, void **buf, size_t *len, int quality) { int vips_gifsave_go(VipsImage *in, void **buf, size_t *len) { #if VIPS_SUPPORT_GIFSAVE - return vips_gifsave_buffer(in, buf, len, NULL); + int bitdepth = vips_get_bits_per_sample(in); + if (bitdepth <= 0 || bitdepth > 8 ) bitdepth = 8; + return vips_gifsave_buffer(in, buf, len, "bitdepth", bitdepth, NULL); #else vips_error("vips_gifsave_go", "Saving GIF is not supported (libvips 8.12+ reuired)"); return 1; From 5a7b6068c79569366080d5966492a7f6e790de2b Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 23 May 2023 19:10:05 +0300 Subject: [PATCH 011/138] Update base Docker image --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 60b9d26076..4662f164f1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE_VERSION="v3.5.0" +ARG BASE_IMAGE_VERSION="v3.5.1" FROM darthsim/imgproxy-base:${BASE_IMAGE_VERSION} From 524434b63fa10d60cb748b6627f3e29a4a29dfcb Mon Sep 17 00:00:00 2001 From: Niklas Mollenhauer Date: Wed, 24 May 2023 15:30:00 +0200 Subject: [PATCH 012/138] Drop dependency + use built-in `base64url` (#1156) * Drop dependency + use built-in `base64url` [`create-hmac`](https://github.com/browserify/createHmac) is meant for compat between node and browser compat and was last updated in 2018. Computing the HMAC signature on the client (browser) does not make any sense in 99% of the use-cases, as the secrets would be needed on the client side. This means that we can drop browser support and just use the native node module, which is also exported by the `create-hmac` when running on node. `Buffer.toString()` also accepts "base64url" as an encoding, so we can drop the `urlSafeBase64` in favor of that. * Use encoder from hmac instance --- examples/signature.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/examples/signature.js b/examples/signature.js index ecbbce759c..06eb08e96c 100644 --- a/examples/signature.js +++ b/examples/signature.js @@ -1,19 +1,16 @@ -const createHmac = require('create-hmac') +import { createHmac } from 'node:crypto'; const KEY = '943b421c9eb07c830af81030552c86009268de4e532ba2ee2eab8247c6da0881' const SALT = '520f986b998545b4785e0defbc4f3c1203f22de2374a3d53cb7a7fe9fea309c5' -const urlSafeBase64 = (string) => { - return Buffer.from(string).toString('base64').replace(/=/g, '').replace(/\+/g, '-').replace(/\//g, '_') -} - const hexDecode = (hex) => Buffer.from(hex, 'hex') const sign = (salt, target, secret) => { const hmac = createHmac('sha256', hexDecode(secret)) hmac.update(hexDecode(salt)) hmac.update(target) - return urlSafeBase64(hmac.digest()) + + return hmac.digest('base64url') } const path = "/rs:fit:300:300/plain/http://img.example.com/pretty/image.jpg" From b3905c0cd373adc37dc3fb7d6707cca5e9d5d451 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Sun, 28 May 2023 19:35:29 +0300 Subject: [PATCH 013/138] Respond with 422 on error during image loading --- CHANGELOG.md | 1 + vips/vips.go | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 137a61d284..85e8fe68b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ### Change - Preserve GIF's bit-per-sample. +- Respond with 422 on error during image loading. ## [3.17.0] - 2023-05-10 ### Add diff --git a/vips/vips.go b/vips/vips.go index 954e0abcdf..126692a863 100644 --- a/vips/vips.go +++ b/vips/vips.go @@ -11,6 +11,7 @@ import ( "math" "os" "runtime" + "strings" "sync" "unsafe" @@ -152,7 +153,14 @@ func Cleanup() { func Error() error { defer C.vips_error_clear() - return ierrors.NewUnexpected(C.GoString(C.vips_error_buffer()), 1) + + errstr := strings.TrimSpace(C.GoString(C.vips_error_buffer())) + + if strings.Contains(errstr, "load_buffer: ") { + return ierrors.New(422, errstr, "Broken or unsupported image") + } + + return ierrors.NewUnexpected(errstr, 1) } func hasOperation(name string) bool { From a246eda065aa7894afe516c956f0e570da3424e7 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Sun, 28 May 2023 19:57:50 +0300 Subject: [PATCH 014/138] Ensure URL replacements are executed in the specified order --- config/config.go | 6 ++++-- config/configurators/configurators.go | 30 +++++++++++++++++---------- options/processing_options_test.go | 10 +++++---- options/url.go | 4 ++-- 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/config/config.go b/config/config.go index 05ded60deb..073d186c1f 100644 --- a/config/config.go +++ b/config/config.go @@ -16,6 +16,8 @@ import ( "github.com/imgproxy/imgproxy/v3/version" ) +type URLReplacement = configurators.URLReplacement + var ( Network string Bind string @@ -127,7 +129,7 @@ var ( LastModifiedEnabled bool BaseURL string - URLReplacements map[*regexp.Regexp]string + URLReplacements []URLReplacement Presets []string OnlyPresets bool @@ -318,7 +320,7 @@ func Reset() { LastModifiedEnabled = false BaseURL = "" - URLReplacements = make(map[*regexp.Regexp]string) + URLReplacements = make([]URLReplacement, 0) Presets = make([]string, 0) OnlyPresets = false diff --git a/config/configurators/configurators.go b/config/configurators/configurators.go index de3052a1d2..5ea47c3835 100644 --- a/config/configurators/configurators.go +++ b/config/configurators/configurators.go @@ -12,6 +12,11 @@ import ( "github.com/imgproxy/imgproxy/v3/imagetype" ) +type URLReplacement struct { + Regexp *regexp.Regexp + Replacement string +} + func Int(i *int, name string) { if env, err := strconv.Atoi(os.Getenv(name)); err == nil { *i = env @@ -240,21 +245,24 @@ func Patterns(s *[]*regexp.Regexp, name string) { } } -func Replacements(m *map[*regexp.Regexp]string, name string) error { - var sm map[string]string - - if err := StringMap(&sm, name); err != nil { - return err - } +func Replacements(s *[]URLReplacement, name string) error { + if env := os.Getenv(name); len(env) > 0 { + ss := []URLReplacement(nil) - if len(sm) > 0 { - mm := make(map[*regexp.Regexp]string) + keyvalues := strings.Split(env, ";") - for k, v := range sm { - mm[RegexpFromPattern(k)] = v + for _, keyvalue := range keyvalues { + parts := strings.SplitN(keyvalue, "=", 2) + if len(parts) != 2 { + return fmt.Errorf("Invalid key/value: %s", keyvalue) + } + ss = append(ss, URLReplacement{ + Regexp: RegexpFromPattern(parts[0]), + Replacement: parts[1], + }) } - *m = mm + *s = ss } return nil diff --git a/options/processing_options_test.go b/options/processing_options_test.go index 2a48fa5a2a..2c1416a6b9 100644 --- a/options/processing_options_test.go +++ b/options/processing_options_test.go @@ -56,8 +56,9 @@ func (s *ProcessingOptionsTestSuite) TestParseBase64URLWithBase() { } func (s *ProcessingOptionsTestSuite) TestParseBase64URLWithReplacement() { - config.URLReplacements = map[*regexp.Regexp]string{ - regexp.MustCompile("^test://([^/]*)/"): "http://images.dev/${1}/dolor/", + config.URLReplacements = []config.URLReplacement{ + {Regexp: regexp.MustCompile("^test://([^/]*)/"), Replacement: "test2://images.dev/${1}/dolor/"}, + {Regexp: regexp.MustCompile("^test2://"), Replacement: "http://"}, } originURL := "test://lorem/ipsum.jpg?param=value" @@ -112,8 +113,9 @@ func (s *ProcessingOptionsTestSuite) TestParsePlainURLWithBase() { } func (s *ProcessingOptionsTestSuite) TestParsePlainURLWithReplacement() { - config.URLReplacements = map[*regexp.Regexp]string{ - regexp.MustCompile("^test://([^/]*)/"): "http://images.dev/${1}/dolor/", + config.URLReplacements = []config.URLReplacement{ + {Regexp: regexp.MustCompile("^test://([^/]*)/"), Replacement: "test2://images.dev/${1}/dolor/"}, + {Regexp: regexp.MustCompile("^test2://"), Replacement: "http://"}, } originURL := "test://lorem/ipsum.jpg" diff --git a/options/url.go b/options/url.go index b4e8e5cc95..e3ba82f94c 100644 --- a/options/url.go +++ b/options/url.go @@ -13,8 +13,8 @@ import ( const urlTokenPlain = "plain" func preprocessURL(u string) string { - for re, repl := range config.URLReplacements { - u = re.ReplaceAllString(u, repl) + for _, repl := range config.URLReplacements { + u = repl.Regexp.ReplaceAllString(u, repl.Replacement) } if len(config.BaseURL) == 0 || strings.HasPrefix(u, config.BaseURL) { From 2e6a3c6dd110f859cae917721e9ac43e9e0935ec Mon Sep 17 00:00:00 2001 From: DarthSim Date: Mon, 29 May 2023 19:48:39 +0300 Subject: [PATCH 015/138] Separate Photoshop and IPTC metadata parsing/dumping --- imagemeta/iptc/iptc.go | 73 ++-------------------------- imagemeta/photoshop/photoshop.go | 81 ++++++++++++++++++++++++++++++++ processing/strip_metadata.go | 31 ++++++++---- 3 files changed, 107 insertions(+), 78 deletions(-) create mode 100644 imagemeta/photoshop/photoshop.go diff --git a/imagemeta/iptc/iptc.go b/imagemeta/iptc/iptc.go index 88fa1e1898..f8148b82fd 100644 --- a/imagemeta/iptc/iptc.go +++ b/imagemeta/iptc/iptc.go @@ -10,13 +10,9 @@ import ( ) var ( - ps3Header = []byte("Photoshop 3.0\x00") - ps3BlockHeader = []byte("8BIM") - ps3IptcRecourceID = []byte("\x04\x04") - iptcTagHeader = byte(0x1c) + iptcTagHeader = byte(0x1c) - errInvalidPS3Header = errors.New("invalid Photoshop 3.0 header") - errInvalidDataSize = errors.New("invalid IPTC data size") + errInvalidDataSize = errors.New("invalid IPTC data size") ) type IptcMap map[TagKey][]TagValue @@ -69,7 +65,7 @@ func (m IptcMap) MarshalJSON() ([]byte, error) { return json.Marshal(mm) } -func ParseTags(data []byte, m IptcMap) error { +func Parse(data []byte, m IptcMap) error { buf := bytes.NewBuffer(data) // Min tag size is 5 (2 tagHeader) @@ -114,52 +110,7 @@ func ParseTags(data []byte, m IptcMap) error { return nil } -func ParsePS3(data []byte, m IptcMap) error { - buf := bytes.NewBuffer(data) - - if !bytes.Equal(buf.Next(14), ps3Header) { - return errInvalidPS3Header - } - - // Read blocks - // Minimal block size is 12 (4 blockHeader + 2 resoureceID + 2 name + 4 blockSize) - for buf.Len() >= 12 { - if !bytes.Equal(buf.Bytes()[:4], ps3BlockHeader) { - buf.Next(1) - continue - } - - // Skip block header - buf.Next(4) - - resoureceID := buf.Next(2) - - // Skip name - // Name is zero terminated string padded to even - for buf.Len() > 0 && buf.Next(2)[1] != 0 { - } - - if buf.Len() < 4 { - break - } - - blockSize := int(binary.BigEndian.Uint32(buf.Next(4))) - - if buf.Len() < blockSize { - break - } - blockData := buf.Next(blockSize) - - // 1028 is IPTC tags block - if bytes.Equal(resoureceID, ps3IptcRecourceID) { - return ParseTags(blockData, m) - } - } - - return nil -} - -func (m IptcMap) DumpTags() []byte { +func (m IptcMap) Dump() []byte { buf := new(bytes.Buffer) for key, values := range m { @@ -187,19 +138,3 @@ func (m IptcMap) DumpTags() []byte { return buf.Bytes() } - -func (m IptcMap) Dump() []byte { - tagsDump := m.DumpTags() - - buf := new(bytes.Buffer) - buf.Grow(26) - - buf.Write(ps3Header) - buf.Write(ps3BlockHeader) - buf.Write(ps3IptcRecourceID) - buf.Write([]byte{0, 0}) - binary.Write(buf, binary.BigEndian, uint32(len(tagsDump))) - buf.Write(tagsDump) - - return buf.Bytes() -} diff --git a/imagemeta/photoshop/photoshop.go b/imagemeta/photoshop/photoshop.go new file mode 100644 index 0000000000..297e180678 --- /dev/null +++ b/imagemeta/photoshop/photoshop.go @@ -0,0 +1,81 @@ +package photoshop + +import ( + "bytes" + "encoding/binary" + "errors" +) + +var ( + ps3Header = []byte("Photoshop 3.0\x00") + ps3BlockHeader = []byte("8BIM") + + errInvalidPS3Header = errors.New("invalid Photoshop 3.0 header") +) + +const ( + IptcKey = "\x04\x04" + ResolutionKey = "\x03\xed" +) + +type PhotoshopMap map[string][]byte + +func Parse(data []byte, m PhotoshopMap) error { + buf := bytes.NewBuffer(data) + + if !bytes.Equal(buf.Next(14), ps3Header) { + return errInvalidPS3Header + } + + // Read blocks + // Minimal block size is 12 (4 blockHeader + 2 resoureceID + 2 name + 4 blockSize) + for buf.Len() >= 12 { + if !bytes.Equal(buf.Bytes()[:4], ps3BlockHeader) { + buf.Next(1) + continue + } + + // Skip block header + buf.Next(4) + + resoureceID := buf.Next(2) + + // Skip name + // Name is zero terminated string padded to even + for buf.Len() > 0 && buf.Next(2)[1] != 0 { + } + + if buf.Len() < 4 { + break + } + + blockSize := int(binary.BigEndian.Uint32(buf.Next(4))) + + if buf.Len() < blockSize { + break + } + blockData := buf.Next(blockSize) + + m[string(resoureceID)] = blockData + } + + return nil +} + +func (m PhotoshopMap) Dump() []byte { + buf := new(bytes.Buffer) + buf.Grow(26) + + buf.Write(ps3Header) + buf.Write(ps3BlockHeader) + + for id, data := range m { + buf.WriteString(id) + // Write empty name + buf.Write([]byte{0, 0}) + binary.Write(buf, binary.BigEndian, uint32(len(data))) + buf.Write(data) + } + + return buf.Bytes() +} diff --git a/processing/strip_metadata.go b/processing/strip_metadata.go index a30f05415b..d346546d6b 100644 --- a/processing/strip_metadata.go +++ b/processing/strip_metadata.go @@ -7,18 +7,27 @@ import ( "github.com/imgproxy/imgproxy/v3/imagedata" "github.com/imgproxy/imgproxy/v3/imagemeta/iptc" + "github.com/imgproxy/imgproxy/v3/imagemeta/photoshop" "github.com/imgproxy/imgproxy/v3/options" "github.com/imgproxy/imgproxy/v3/vips" ) -func stripIPTC(img *vips.Image) []byte { - iptcData, err := img.GetBlob("iptc-data") - if err != nil || len(iptcData) == 0 { +func stripPS3(img *vips.Image) []byte { + ps3Data, err := img.GetBlob("iptc-data") + if err != nil || len(ps3Data) == 0 { + return nil + } + + ps3Map := make(photoshop.PhotoshopMap) + photoshop.Parse(ps3Data, ps3Map) + + iptcData, found := ps3Map[photoshop.IptcKey] + if !found { return nil } iptcMap := make(iptc.IptcMap) - err = iptc.ParsePS3(iptcData, iptcMap) + err = iptc.Parse(iptcData, iptcMap) if err != nil { return nil } @@ -33,7 +42,11 @@ func stripIPTC(img *vips.Image) []byte { return nil } - return iptcMap.Dump() + ps3Map = photoshop.PhotoshopMap{ + photoshop.IptcKey: iptcMap.Dump(), + } + + return ps3Map.Dump() } func stripXMP(img *vips.Image) []byte { @@ -97,10 +110,10 @@ func stripMetadata(pctx *pipelineContext, img *vips.Image, po *options.Processin return nil } - var iptcData, xmpData []byte + var ps3Data, xmpData []byte if po.KeepCopyright { - iptcData = stripIPTC(img) + ps3Data = stripPS3(img) xmpData = stripXMP(img) } @@ -109,8 +122,8 @@ func stripMetadata(pctx *pipelineContext, img *vips.Image, po *options.Processin } if po.KeepCopyright { - if len(iptcData) > 0 { - img.SetBlob("iptc-data", iptcData) + if len(ps3Data) > 0 { + img.SetBlob("iptc-data", ps3Data) } if len(xmpData) > 0 { From b20b5ff7684f7e63d1a90dc24ec8aa2412c7c128 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Mon, 29 May 2023 21:27:47 +0300 Subject: [PATCH 016/138] Fix Photoshop metadata dump --- imagemeta/photoshop/photoshop.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/imagemeta/photoshop/photoshop.go b/imagemeta/photoshop/photoshop.go index 297e180678..625d61766a 100644 --- a/imagemeta/photoshop/photoshop.go +++ b/imagemeta/photoshop/photoshop.go @@ -67,9 +67,13 @@ func (m PhotoshopMap) Dump() []byte { buf.Grow(26) buf.Write(ps3Header) - buf.Write(ps3BlockHeader) for id, data := range m { + if len(data) == 0 { + continue + } + + buf.Write(ps3BlockHeader) buf.WriteString(id) // Write empty name buf.Write([]byte{0, 0}) From ac87eea3174a67f03b553b7b754af6d674ea9fc7 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 30 May 2023 20:08:04 +0300 Subject: [PATCH 017/138] Update changelog and docs --- CHANGELOG.md | 4 ++++ docs/configuration.md | 1 + docs/generating_the_url.md | 12 ++++++++++++ docs/getting_the_image_info.md | 11 +++++++++++ 4 files changed, 28 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85e8fe68b7..06ae16f553 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ ## [Unreleased] ### Add - Add `IMGPROXY_URL_REPLACEMENTS` config. +- (pro) Add `IMGPROXY_STRIP_METADATA_DPI` config. +- (pro) Add [dpi](https://docs.imgproxy.net/latest/configuration?id=dpi) processing option. +- (pro) Add WebP EXIF and XMP to the `/info` response. +- (pro) Add Photoshop resolution data to the `/info` response. ### Change - Preserve GIF's bit-per-sample. diff --git a/docs/configuration.md b/docs/configuration.md index 0d3bcb713f..9515e5f40d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -541,6 +541,7 @@ imgproxy can send logs to syslog, but this feature is disabled by default. To en * `IMGPROXY_DISABLE_SHRINK_ON_LOAD`: when `true`, disables shrink-on-load for JPEGs and WebP files. Allows processing the entire image in linear colorspace but dramatically slows down resizing and increases memory usage when working with large images. * `IMGPROXY_STRIP_METADATA`: when `true`, imgproxy will strip all metadata (EXIF, IPTC, etc.) from JPEG and WebP output images. Default: `true` * `IMGPROXY_KEEP_COPYRIGHT`: when `true`, imgproxy will not remove copyright info while stripping metadata. Default: `true` +* `IMGPROXY_STRIP_METADATA_DPI`: ![pro](./assets/pro.svg) the DPI metadata value that should be set for the image when its metadata is stripped. Default: `72.0` * `IMGPROXY_STRIP_COLOR_PROFILE`: when `true`, imgproxy will transform the embedded color profile (ICC) to sRGB and remove it from the image. Otherwise, imgproxy will try to keep it as is. Default: `true` * `IMGPROXY_AUTO_ROTATE`: when `true`, imgproxy will automatically rotate images based on the EXIF Orientation parameter (if available in the image meta data). The orientation tag will be removed from the image in all cases. Default: `true` * `IMGPROXY_ENFORCE_THUMBNAIL`: when `true` and the source image has an embedded thumbnail, imgproxy will always use the embedded thumbnail instead of the main image. Currently, only thumbnails embedded in `heic` and `avif` are supported. Default: `false` diff --git a/docs/generating_the_url.md b/docs/generating_the_url.md index 8cb4752cb5..1f9d7a62e3 100644 --- a/docs/generating_the_url.md +++ b/docs/generating_the_url.md @@ -552,6 +552,18 @@ kcr:%keep_copyright When set to `1`, `t` or `true`, imgproxy will not remove copyright info while stripping metadata. This is normally controlled by the [IMGPROXY_KEEP_COPYRIGHT](configuration.md#miscellaneous) configuration but this procesing option allows the configuration to be set for each request. +### DPI![pro](./assets/pro.svg) :id=dpi + +``` +dpi:%dpi +``` + +When set, imgproxy will replace the image's DPI metadata with the provided value. When set to `0`, imgproxy won't change the image's DPI or will reset it to the default value if the image's metadata should be stripped. + +**📝 Note:** This processing option takes effect whether imgproxy should strip the image's metadata or not. + +Default: `0` + ### Strip color profile ``` diff --git a/docs/getting_the_image_info.md b/docs/getting_the_image_info.md index ccabbfa2ee..5d992d7cd0 100644 --- a/docs/getting_the_image_info.md +++ b/docs/getting_the_image_info.md @@ -55,6 +55,7 @@ imgproxy responses with a JSON body and returns the following info: * `exif`: Exif data * `iptc`: IPTC data * `xmp`: XMP data +* `photoshop`: Photoshop metadata (currently, only the resolution data) * `video_meta`: metadata from the video **📝 Note:** There are lots of IPTC tags in the spec, but imgproxy supports only a few of them. If you need some tags to be supported, just contact us. @@ -98,6 +99,16 @@ imgproxy responses with a JSON body and returns the following info: "photoshop": { "DateCreated": "2016-09-11T18:44:50.003" } + }, + "photoshop": { + "resolution": { + "XResolution": 240, + "XResolutionUnit": "inches", + "WidthUnit": "inches", + "YResolution": 240, + "YResolutionUnit": "inches", + "HeightUnit": "inches" + } } } ``` From 597673b6502a447ef368ec85dac95fd1234ef284 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 30 May 2023 20:10:01 +0300 Subject: [PATCH 018/138] Fix changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06ae16f553..80b0f516d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Add - Add `IMGPROXY_URL_REPLACEMENTS` config. - (pro) Add `IMGPROXY_STRIP_METADATA_DPI` config. -- (pro) Add [dpi](https://docs.imgproxy.net/latest/configuration?id=dpi) processing option. +- (pro) Add [dpi](https://docs.imgproxy.net/latest/generating_the_url?id=dpi) processing option. - (pro) Add WebP EXIF and XMP to the `/info` response. - (pro) Add Photoshop resolution data to the `/info` response. From b9e0134f2e9371c7dead35c22b9e257c8f78e61a Mon Sep 17 00:00:00 2001 From: DarthSim Date: Wed, 31 May 2023 22:52:36 +0300 Subject: [PATCH 019/138] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80b0f516d7..29faa1fe37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ - Preserve GIF's bit-per-sample. - Respond with 422 on error during image loading. +### Fix +- (pro) Fix applying the `resizing_algorithm` processing option when resizing images with an alpha channel. + ## [3.17.0] - 2023-05-10 ### Add - Add `process_resident_memory_bytes`, `process_virtual_memory_bytes`, `go_memstats_sys_bytes`, `go_memstats_heap_idle_bytes`, `go_memstats_heap_inuse_bytes`, `go_goroutines`, `go_threads`, `buffer_default_size_bytes`, `buffer_max_size_bytes`, and `buffer_size_bytes` metrics to OpenTelemetry. From 8629c5eca1e422908363f471513bfc887d778a85 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Wed, 31 May 2023 22:55:51 +0300 Subject: [PATCH 020/138] Bump version --- CHANGELOG.md | 2 ++ version/version.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29faa1fe37..9baf37cf5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased] + +## [3.18.0] - 2023-05-31 ### Add - Add `IMGPROXY_URL_REPLACEMENTS` config. - (pro) Add `IMGPROXY_STRIP_METADATA_DPI` config. diff --git a/version/version.go b/version/version.go index 8fe870f3d3..ba9864d0c1 100644 --- a/version/version.go +++ b/version/version.go @@ -1,6 +1,6 @@ package version -const version = "3.17.0" +const version = "3.18.0" func Version() string { return version From 9b6074094a83eef54797fc269beb667932483b76 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 6 Jun 2023 21:09:42 +0300 Subject: [PATCH 021/138] Fix return value from vips_resize_go --- vips/vips.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vips/vips.c b/vips/vips.c index 2a4cfb0a25..353cbcf6e4 100644 --- a/vips/vips.c +++ b/vips/vips.c @@ -220,7 +220,7 @@ vips_resize_go(VipsImage *in, VipsImage **out, double wscale, double hscale) { clear_image(&base); - return 0; + return res; } int From 8227592451b70e96127af526e80868ef1c33902c Mon Sep 17 00:00:00 2001 From: DarthSim Date: Mon, 12 Jun 2023 22:00:55 +0300 Subject: [PATCH 022/138] Optimize bufpool --- bufpool/bufpool.go | 24 ++++++++++++++++++++++-- imagedata/read.go | 4 +--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/bufpool/bufpool.go b/bufpool/bufpool.go index 6eb97c9237..2cd0f054c4 100644 --- a/bufpool/bufpool.go +++ b/bufpool/bufpool.go @@ -106,7 +106,7 @@ func (p *Pool) calibrateAndClean() { for i := 0; i < steps; i++ { callsSum += p.tmpCalls[i] - if callsSum > defSum || defStep < 0 { + if defStep < 0 && callsSum > defSum { defStep = i } @@ -219,7 +219,7 @@ func (p *Pool) Get(size int, grow bool) *bytes.Buffer { growSize := p.defaultSize if grow { - growSize = imath.Max(size, growSize) + growSize = imath.Max(p.normalizeCap(size), growSize) } // Grow the buffer only if we know the requested size and it is smaller than @@ -261,6 +261,26 @@ func (p *Pool) Put(buf *bytes.Buffer) { p.insert(buf) } +// GrowBuffer growth capacity of the buffer to the normalized provided value +func (p *Pool) GrowBuffer(buf *bytes.Buffer, cap int) { + cap = p.normalizeCap(cap) + if buf.Cap() < cap { + buf.Grow(cap - buf.Len()) + } +} + +func (p *Pool) normalizeCap(cap int) int { + // Don't normalize cap if it's larger than maxSize + // since we'll throw this buf out anyway + maxSize := int(atomic.LoadUint64(&p.maxSize)) + if maxSize > 0 && cap > maxSize { + return cap + } + + ind := index(cap) + return imath.Max(cap, minSize< buf.Cap() { - buf.Grow(contentLength - buf.Len()) - } + downloadBufPool.GrowBuffer(buf, contentLength) if err = br.Flush(); err != nil { buf.Reset() From 56b294cbd3bcd18334c55b20e0ecac8531f72467 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Sat, 17 Jun 2023 20:15:11 +0300 Subject: [PATCH 023/138] Update docs --- docs/generating_the_url.md | 2 +- docs/getting_the_image_info.md | 2 +- docs/signing_the_url.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/generating_the_url.md b/docs/generating_the_url.md index 1f9d7a62e3..e89031225b 100644 --- a/docs/generating_the_url.md +++ b/docs/generating_the_url.md @@ -885,7 +885,7 @@ When using an encoded source URL, you can specify the [extension](#extension) af /aHR0cDovL2V4YW1w/bGUuY29tL2ltYWdl/cy9jdXJpb3NpdHku/anBn.png ``` -### Encrypted with AES-CBC +### Encrypted with AES-CBC![pro](./assets/pro.svg) :id=encrypted-with-aes-cbc The source URL can be encrypted with the AES-CBC algorithm, prepended by the `/enc/` segment. The encrypted URL can be split with `/` as desired: diff --git a/docs/getting_the_image_info.md b/docs/getting_the_image_info.md index 5d992d7cd0..4121bc52ce 100644 --- a/docs/getting_the_image_info.md +++ b/docs/getting_the_image_info.md @@ -36,7 +36,7 @@ The source URL can be encoded with URL-safe Base64. The encoded URL can be split /aHR0cDovL2V4YW1w/bGUuY29tL2ltYWdl/cy9jdXJpb3NpdHku/anBn ``` -#### Encrypted with AES-CBC +#### Encrypted with AES-CBC![pro](./assets/pro.svg) :id=encrypted-with-aes-cbc The source URL can be encrypted with the AES-CBC algorithm, prepended by the `/enc/` segment. The encrypted URL can be split with `/` as desired: diff --git a/docs/signing_the_url.md b/docs/signing_the_url.md index fc98b3ac72..1d7f6d4966 100644 --- a/docs/signing_the_url.md +++ b/docs/signing_the_url.md @@ -23,8 +23,8 @@ A signature is a URL-safe Base64-encoded HMAC digest of the rest of the path, in * Take the part of the path after the signature: - * For [processing URLs](generating_the_url.md): `/%processing_options/%encoded_url.%extension` or `/%processing_options/plain/%plain_url@%extension` - * For [info URLs](getting_the_image_info.md): `/%encoded_url` or `/plain/%plain_url` + * For [processing URLs](generating_the_url.md): `/%processing_options/%encoded_url.%extension`, `/%processing_options/plain/%plain_url@%extension`, or `/%processing_options/enc/%encrypted_url.%extension` + * For [info URLs](getting_the_image_info.md): `/%encoded_url`, `/plain/%plain_url`, or `/enc/%encrypted_url` * Add a salt to the beginning. * Calculate the HMAC digest using SHA256. * Encode the result with URL-safe Base64. From 9175c66feeaf25729ef2e8b5ca62763e920a51ac Mon Sep 17 00:00:00 2001 From: DarthSim Date: Thu, 29 Jun 2023 20:05:24 +0300 Subject: [PATCH 024/138] Update maximum and default AVIF speed --- CHANGELOG.md | 2 ++ config/config.go | 6 +++--- docs/configuration.md | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9baf37cf5a..9755de0c6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased] +### Change +- Change maximum and default values of `IMGPROXY_AVIF_SPEED` to `9`. ## [3.18.0] - 2023-05-31 ### Add diff --git a/config/config.go b/config/config.go index 073d186c1f..1397546dd0 100644 --- a/config/config.go +++ b/config/config.go @@ -242,7 +242,7 @@ func Reset() { PngInterlaced = false PngQuantize = false PngQuantizationColors = 256 - AvifSpeed = 8 + AvifSpeed = 9 Quality = 80 FormatQuality = map[imagetype.Type]int{imagetype.AVIF: 65} StripMetadata = true @@ -658,8 +658,8 @@ func Configure() error { if AvifSpeed < 0 { return fmt.Errorf("Avif speed should be greater than 0, now - %d\n", AvifSpeed) - } else if AvifSpeed > 8 { - return fmt.Errorf("Avif speed can't be greater than 8, now - %d\n", AvifSpeed) + } else if AvifSpeed > 9 { + return fmt.Errorf("Avif speed can't be greater than 9, now - %d\n", AvifSpeed) } if Quality <= 0 { diff --git a/docs/configuration.md b/docs/configuration.md index 9515e5f40d..c5c8e3c678 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -176,7 +176,7 @@ When cookie forwarding is activated, by default, imgproxy assumes the scope of t ### Advanced AVIF compression -* `IMGPROXY_AVIF_SPEED`: controls the CPU effort spent improving compression. The lowest speed is at 0 and the fastest is at 8. Default: `8` +* `IMGPROXY_AVIF_SPEED`: controls the CPU effort spent improving compression. The lowest speed is at 0 and the fastest is at 9. Default: `9` ### Autoquality From 9c1a3befd105dc955c482011532300b0afaf49cb Mon Sep 17 00:00:00 2001 From: DarthSim Date: Thu, 29 Jun 2023 20:16:55 +0300 Subject: [PATCH 025/138] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9755de0c6e..08c748990f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## [Unreleased] ### Change - Change maximum and default values of `IMGPROXY_AVIF_SPEED` to `9`. +- (pro) Fix detection of some videos. +- (pro) Better calculation of the image complexity during choosing the best format. ## [3.18.0] - 2023-05-31 ### Add From d094b57a6fe3db9760c6538653019f63e5932137 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Thu, 29 Jun 2023 20:19:30 +0300 Subject: [PATCH 026/138] Update base Docker image --- CHANGELOG.md | 1 + docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08c748990f..3ab9a44205 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Change maximum and default values of `IMGPROXY_AVIF_SPEED` to `9`. - (pro) Fix detection of some videos. - (pro) Better calculation of the image complexity during choosing the best format. +- (docker) Fix freezes and crashes introduced in v3.18.0 by liborc. ## [3.18.0] - 2023-05-31 ### Add diff --git a/docker/Dockerfile b/docker/Dockerfile index 4662f164f1..8167733cad 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE_VERSION="v3.5.1" +ARG BASE_IMAGE_VERSION="v3.6.0" FROM darthsim/imgproxy-base:${BASE_IMAGE_VERSION} From 1dacb3beb13300063b1d5851d56f5cfac3525874 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Thu, 29 Jun 2023 20:20:43 +0300 Subject: [PATCH 027/138] Bump version --- CHANGELOG.md | 2 ++ version/version.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ab9a44205..d07bba78f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased] + +## [3.18.1] - 2023-07-29 ### Change - Change maximum and default values of `IMGPROXY_AVIF_SPEED` to `9`. - (pro) Fix detection of some videos. diff --git a/version/version.go b/version/version.go index ba9864d0c1..aa6a312a97 100644 --- a/version/version.go +++ b/version/version.go @@ -1,6 +1,6 @@ package version -const version = "3.18.0" +const version = "3.18.1" func Version() string { return version From 245974121807971172e80b856f4d3e9f8a2d7f5e Mon Sep 17 00:00:00 2001 From: DarthSim Date: Fri, 30 Jun 2023 17:52:08 +0300 Subject: [PATCH 028/138] Unsharpening => unsharp masking in docs --- docs/configuration.md | 16 ++++++++-------- docs/generating_the_url.md | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index c5c8e3c678..1d8e11a72d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -279,16 +279,16 @@ imgproxy Pro can extract specific video frames to create thumbnails. This featur Read more about watermarks in the [Watermark](watermark.md) guide. -## Unsharpening +## Unsharp masking -imgproxy Pro can apply an unsharpening mask to your images. +imgproxy Pro can apply unsharp masking to your images. -* `IMGPROXY_UNSHARPENING_MODE`: ![pro](./assets/pro.svg) controls when an unsharpenning mask should be applied. The following modes are supported: - * `auto`: _(default)_ apply an unsharpening mask only when an image is downscaled and the `sharpen` option has not been set. - * `none`: the unsharpening mask is not applied. - * `always`: always applies the unsharpening mask. -* `IMGPROXY_UNSHARPENING_WEIGHT`: ![pro](./assets/pro.svg) a floating-point number that defines how neighboring pixels will affect the current pixel. The greater the value, the sharper the image. This value should be greater than zero. Default: `1` -* `IMGPROXY_UNSHARPENING_DIVIDOR`: ![pro](./assets/pro.svg) a floating-point number that defines the unsharpening strength. The lesser the value, the sharper the image. This value be greater than zero. Default: `24` +* `IMGPROXY_UNSHARP_MASKING_MODE`: ![pro](./assets/pro.svg) controls when unsharp masking should be applied. The following modes are supported: + * `auto`: _(default)_ apply unsharp masking only when an image is downscaled and the `sharpen` option has not been set. + * `none`: unsharp masking is not applied. + * `always`: always applies unsharp masking. +* `IMGPROXY_UNSHARP_MASKING_WEIGHT`: ![pro](./assets/pro.svg) a floating-point number that defines how neighboring pixels will affect the current pixel. The greater the value, the sharper the image. This value should be greater than zero. Default: `1` +* `IMGPROXY_UNSHARP_MASKING_DIVIDER`: ![pro](./assets/pro.svg) a floating-point number that defines unsharp masking strength. The lesser the value, the sharper the image. This value be greater than zero. Default: `24` ## Smart crop diff --git a/docs/generating_the_url.md b/docs/generating_the_url.md index e89031225b..a4572601a3 100644 --- a/docs/generating_the_url.md +++ b/docs/generating_the_url.md @@ -397,14 +397,14 @@ When set, imgproxy will apply the pixelate filter to the resulting image. The va Default: disabled -### Unsharpening![pro](./assets/pro.svg) :id=unsharpening +### Unsharp masking![pro](./assets/pro.svg) :id=unsharp-masking ``` -unsharpening:%mode:%weight:%dividor -ush:%mode:%weight:%dividor +unsharp_masking:%mode:%weight:%divider +ush:%mode:%weight:%divider ``` -Allows redefining unsharpening options. All arguments have the same meaning as [Unsharpening](configuration.md#unsharpening) configs. All arguments are optional and can be omitted. +Allows redefining unsharp masking options. All arguments have the same meaning as [Unsharp masking](configuration.md#unsharp-masking) configs. All arguments are optional and can be omitted. ### Blur detections![pro](./assets/pro.svg) :id=blur-detections From 136a5d093fa7e9e6047afac42ad3526a68623686 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Sun, 2 Jul 2023 22:58:32 +0300 Subject: [PATCH 029/138] Fix saving to JPEG when using linear colorspace --- CHANGELOG.md | 2 ++ processing/export_color_profile.go | 6 ++++++ vips/vips.go | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d07bba78f0..e756417e14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased] +### Fix +- Fix saving to JPEG when using linear colorspace. ## [3.18.1] - 2023-07-29 ### Change diff --git a/processing/export_color_profile.go b/processing/export_color_profile.go index 4323a37093..d15d869fc3 100644 --- a/processing/export_color_profile.go +++ b/processing/export_color_profile.go @@ -9,6 +9,12 @@ import ( func exportColorProfile(pctx *pipelineContext, img *vips.Image, po *options.ProcessingOptions, imgdata *imagedata.ImageData) error { keepProfile := !po.StripColorProfile && po.Format.SupportsColourProfile() + if img.IsLinear() { + if err := img.RgbColourspace(); err != nil { + return err + } + } + if pctx.iccImported { if keepProfile { // We imported ICC profile and want to keep it, diff --git a/vips/vips.go b/vips/vips.go index 126692a863..3154ec9626 100644 --- a/vips/vips.go +++ b/vips/vips.go @@ -638,6 +638,10 @@ func (img *Image) IsRGB() bool { format == C.VIPS_INTERPRETATION_RGB16 } +func (img *Image) IsLinear() bool { + return C.vips_image_guess_interpretation(img.VipsImage) == C.VIPS_INTERPRETATION_scRGB +} + func (img *Image) ImportColourProfile() error { var tmp *C.VipsImage From 4e4b9edb53dd0ee9aa5c7ee4a1b389de0cc6b0ac Mon Sep 17 00:00:00 2001 From: DarthSim Date: Sun, 2 Jul 2023 23:01:44 +0300 Subject: [PATCH 030/138] Exclude twitter.com from check-links git hook --- lefthook.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lefthook.yml b/lefthook.yml index 43520d292b..83182617e9 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -10,4 +10,4 @@ pre-push: commands: check-links: tags: docs - run: command -v lychee && lychee docs README.md CHANGELOG.md --exclude localhost --exclude-path docs/index.html + run: command -v lychee && lychee docs README.md CHANGELOG.md --exclude localhost --exclude twitter.com --exclude-path docs/index.html From 4de9b83899ce6c6929e6777c9be616c1e1efa514 Mon Sep 17 00:00:00 2001 From: Andrei Vydrin Date: Thu, 22 Jun 2023 18:22:10 +0700 Subject: [PATCH 031/138] fix(svg/sanitize): preserve headers from origin data --- processing_handler.go | 2 +- processing_handler_test.go | 25 ++++++++++++++++++++++++- svg/svg.go | 7 ++++--- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/processing_handler.go b/processing_handler.go index a913da4ab0..b8aa5ff089 100644 --- a/processing_handler.go +++ b/processing_handler.go @@ -362,7 +362,7 @@ func handleProcessing(reqID string, rw http.ResponseWriter, r *http.Request) { // Don't process SVG if originData.Type == imagetype.SVG { if config.SanitizeSvg { - sanitized, svgErr := svg.Satitize(originData) + sanitized, svgErr := svg.Sanitize(originData) checkErr(ctx, "svg_processing", svgErr) // Since we'll replace origin data, it's better to close it to return diff --git a/processing_handler_test.go b/processing_handler_test.go index abafff4ec8..f5051e9077 100644 --- a/processing_handler_test.go +++ b/processing_handler_test.go @@ -317,13 +317,36 @@ func (s *ProcessingHandlerTestSuite) TestSkipProcessingSVG() { require.Equal(s.T(), 200, res.StatusCode) actual := s.readBody(res) - expected, err := svg.Satitize(&imagedata.ImageData{Data: s.readTestFile("test1.svg")}) + expected, err := svg.Sanitize(&imagedata.ImageData{Data: s.readTestFile("test1.svg")}) require.Nil(s.T(), err) require.True(s.T(), bytes.Equal(expected.Data, actual)) } +func (s *ProcessingHandlerTestSuite) TestPreserveOriginSVGHeaders() { + rw := s.send("/unsafe/rs:fill:4:4/plain/local:///test1.svg") + res := rw.Result() + + require.Equal(s.T(), 200, res.StatusCode) + + actual := s.readBody(res) + originHeaders := map[string]string{ + "Content-Type": "image/svg+xml", + "Cache-Control": "public, max-age=12345", + } + + expected, err := svg.Sanitize(&imagedata.ImageData{ + Data: s.readTestFile("test1.svg"), + Headers: originHeaders, + }) + + require.Nil(s.T(), err) + + require.True(s.T(), bytes.Equal(expected.Data, actual)) + require.Equal(s.T(), originHeaders, expected.Headers) +} + func (s *ProcessingHandlerTestSuite) TestNotSkipProcessingSVGToJPG() { rw := s.send("/unsafe/rs:fill:4:4/plain/local:///test1.svg@jpg") res := rw.Result() diff --git a/svg/svg.go b/svg/svg.go index 8e51d0b83d..7e3d3e053d 100644 --- a/svg/svg.go +++ b/svg/svg.go @@ -27,7 +27,7 @@ const feDropShadowTemplate = ` ` -func Satitize(data *imagedata.ImageData) (*imagedata.ImageData, error) { +func Sanitize(data *imagedata.ImageData) (*imagedata.ImageData, error) { r := bytes.NewReader(data.Data) l := xml.NewLexer(parse.NewInput(r)) @@ -62,8 +62,9 @@ func Satitize(data *imagedata.ImageData) (*imagedata.ImageData, error) { } newData := imagedata.ImageData{ - Data: buf.Bytes(), - Type: data.Type, + Data: buf.Bytes(), + Type: data.Type, + Headers: data.Headers, } newData.SetCancel(cancel) From 5f3d551f25935092c2c5b091342512a57ed8f16b Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 4 Jul 2023 18:30:26 +0300 Subject: [PATCH 032/138] Preserve headers in svg.FixUnsupported --- svg/svg.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/svg/svg.go b/svg/svg.go index 7e3d3e053d..e6ddc37310 100644 --- a/svg/svg.go +++ b/svg/svg.go @@ -194,8 +194,9 @@ func FixUnsupported(data *imagedata.ImageData) (*imagedata.ImageData, bool, erro } newData := imagedata.ImageData{ - Data: buf.Bytes(), - Type: data.Type, + Data: buf.Bytes(), + Type: data.Type, + Headers: data.Headers, } newData.SetCancel(cancel) From 93063787b1fcfe323a413086792bb506bb0f6db0 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 4 Jul 2023 18:31:06 +0300 Subject: [PATCH 033/138] Add svg package tests --- processing_handler_test.go | 23 -------- svg/svg.go | 22 ++++---- svg/svg_test.go | 83 ++++++++++++++++++++++++++++ testdata/test1.drop-shadow.fixed.svg | 17 ++++++ testdata/test1.drop-shadow.svg | 9 +++ testdata/test1.sanitized.svg | 4 ++ testdata/test1.svg | 9 ++- 7 files changed, 131 insertions(+), 36 deletions(-) create mode 100644 svg/svg_test.go create mode 100644 testdata/test1.drop-shadow.fixed.svg create mode 100644 testdata/test1.drop-shadow.svg create mode 100644 testdata/test1.sanitized.svg diff --git a/processing_handler_test.go b/processing_handler_test.go index f5051e9077..449e032009 100644 --- a/processing_handler_test.go +++ b/processing_handler_test.go @@ -324,29 +324,6 @@ func (s *ProcessingHandlerTestSuite) TestSkipProcessingSVG() { require.True(s.T(), bytes.Equal(expected.Data, actual)) } -func (s *ProcessingHandlerTestSuite) TestPreserveOriginSVGHeaders() { - rw := s.send("/unsafe/rs:fill:4:4/plain/local:///test1.svg") - res := rw.Result() - - require.Equal(s.T(), 200, res.StatusCode) - - actual := s.readBody(res) - originHeaders := map[string]string{ - "Content-Type": "image/svg+xml", - "Cache-Control": "public, max-age=12345", - } - - expected, err := svg.Sanitize(&imagedata.ImageData{ - Data: s.readTestFile("test1.svg"), - Headers: originHeaders, - }) - - require.Nil(s.T(), err) - - require.True(s.T(), bytes.Equal(expected.Data, actual)) - require.Equal(s.T(), originHeaders, expected.Headers) -} - func (s *ProcessingHandlerTestSuite) TestNotSkipProcessingSVGToJPG() { rw := s.send("/unsafe/rs:fill:4:4/plain/local:///test1.svg@jpg") res := rw.Result() diff --git a/svg/svg.go b/svg/svg.go index e6ddc37310..6bead71f59 100644 --- a/svg/svg.go +++ b/svg/svg.go @@ -15,17 +15,17 @@ import ( var feDropShadowName = []byte("feDropShadow") -const feDropShadowTemplate = ` - - - - - - - - - -` +var feDropShadowTemplate = strings.TrimSpace(` + + + + + + + + + +`) func Sanitize(data *imagedata.ImageData) (*imagedata.ImageData, error) { r := bytes.NewReader(data.Data) diff --git a/svg/svg_test.go b/svg/svg_test.go new file mode 100644 index 0000000000..8d72c72587 --- /dev/null +++ b/svg/svg_test.go @@ -0,0 +1,83 @@ +package svg + +import ( + "os" + "path/filepath" + "regexp" + "testing" + + "github.com/imgproxy/imgproxy/v3/config" + "github.com/imgproxy/imgproxy/v3/imagedata" + "github.com/imgproxy/imgproxy/v3/imagetype" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type SvgTestSuite struct { + suite.Suite +} + +func (s *SvgTestSuite) SetupSuite() { + config.Reset() + + err := imagedata.Init() + require.Nil(s.T(), err) +} + +func (s *SvgTestSuite) readTestFile(name string) *imagedata.ImageData { + wd, err := os.Getwd() + require.Nil(s.T(), err) + + data, err := os.ReadFile(filepath.Join(wd, "..", "testdata", name)) + require.Nil(s.T(), err) + + return &imagedata.ImageData{ + Type: imagetype.SVG, + Data: data, + Headers: map[string]string{ + "Content-Type": "image/svg+xml", + "Cache-Control": "public, max-age=12345", + }, + } +} + +func (s *SvgTestSuite) TestSanitize() { + origin := s.readTestFile("test1.svg") + expected := s.readTestFile("test1.sanitized.svg") + + actual, err := Sanitize(origin) + + require.Nil(s.T(), err) + require.Equal(s.T(), string(expected.Data), string(actual.Data)) + require.Equal(s.T(), origin.Headers, actual.Headers) +} + +func (s *SvgTestSuite) TestFixUnsupportedDropShadow() { + origin := s.readTestFile("test1.drop-shadow.svg") + expected := s.readTestFile("test1.drop-shadow.fixed.svg") + + actual, changed, err := FixUnsupported(origin) + + // `FixUnsupported` generates random IDs, we need to replace them for the test + re := regexp.MustCompile(`"ds(in|of)-.+?"`) + actualData := re.ReplaceAllString(string(actual.Data), `"ds$1-test"`) + + require.Nil(s.T(), err) + require.True(s.T(), changed) + require.Equal(s.T(), string(expected.Data), actualData) + require.Equal(s.T(), origin.Headers, actual.Headers) +} + +func (s *SvgTestSuite) TestFixUnsupportedNothingChanged() { + origin := s.readTestFile("test1.svg") + + actual, changed, err := FixUnsupported(origin) + + require.Nil(s.T(), err) + require.False(s.T(), changed) + require.Equal(s.T(), origin, actual) +} + +func TestSvg(t *testing.T) { + suite.Run(t, new(SvgTestSuite)) +} diff --git a/testdata/test1.drop-shadow.fixed.svg b/testdata/test1.drop-shadow.fixed.svg new file mode 100644 index 0000000000..ef53c4cdf5 --- /dev/null +++ b/testdata/test1.drop-shadow.fixed.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/testdata/test1.drop-shadow.svg b/testdata/test1.drop-shadow.svg new file mode 100644 index 0000000000..1f5764dbbd --- /dev/null +++ b/testdata/test1.drop-shadow.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/testdata/test1.sanitized.svg b/testdata/test1.sanitized.svg new file mode 100644 index 0000000000..9da5c05389 --- /dev/null +++ b/testdata/test1.sanitized.svg @@ -0,0 +1,4 @@ + + + + diff --git a/testdata/test1.svg b/testdata/test1.svg index 0e203a57cd..e32f2db523 100644 --- a/testdata/test1.svg +++ b/testdata/test1.svg @@ -1,3 +1,8 @@ - - + + + From 523f3a654661ec3a80fa19411cffe2ca55b6189a Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 4 Jul 2023 18:34:11 +0300 Subject: [PATCH 034/138] Clone headers in the svg package --- svg/svg.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/svg/svg.go b/svg/svg.go index 6bead71f59..cfc702618c 100644 --- a/svg/svg.go +++ b/svg/svg.go @@ -27,6 +27,19 @@ var feDropShadowTemplate = strings.TrimSpace(` `) +func cloneHeaders(src map[string]string) map[string]string { + if src == nil { + return nil + } + + dst := make(map[string]string, len(src)) + for k, v := range src { + dst[k] = v + } + + return dst +} + func Sanitize(data *imagedata.ImageData) (*imagedata.ImageData, error) { r := bytes.NewReader(data.Data) l := xml.NewLexer(parse.NewInput(r)) @@ -64,7 +77,7 @@ func Sanitize(data *imagedata.ImageData) (*imagedata.ImageData, error) { newData := imagedata.ImageData{ Data: buf.Bytes(), Type: data.Type, - Headers: data.Headers, + Headers: cloneHeaders(data.Headers), } newData.SetCancel(cancel) @@ -196,7 +209,7 @@ func FixUnsupported(data *imagedata.ImageData) (*imagedata.ImageData, bool, erro newData := imagedata.ImageData{ Data: buf.Bytes(), Type: data.Type, - Headers: data.Headers, + Headers: cloneHeaders(data.Headers), } newData.SetCancel(cancel) From 94c95980378d9634329c586c5c832a2e01866754 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 4 Jul 2023 18:36:51 +0300 Subject: [PATCH 035/138] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e756417e14..2c74fc1ba8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Unreleased] ### Fix - Fix saving to JPEG when using linear colorspace. +- Fix the `Cache-Control` and `Expires` headers passthrough when SVG is sanitized or fixed. ## [3.18.1] - 2023-07-29 ### Change From 271b15381faf9810060fb16dc79b1a4e3ea74eaf Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 4 Jul 2023 18:48:26 +0300 Subject: [PATCH 036/138] Update testdata/test1.svg --- testdata/test1.svg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testdata/test1.svg b/testdata/test1.svg index e32f2db523..7f2dfccd80 100644 --- a/testdata/test1.svg +++ b/testdata/test1.svg @@ -1,8 +1,8 @@ From 3265bfa12c40af9e251ef008921435b89e91a5da Mon Sep 17 00:00:00 2001 From: DarthSim Date: Wed, 12 Jul 2023 12:48:03 +0300 Subject: [PATCH 037/138] Update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c74fc1ba8..91fbac7dc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,9 @@ ### Fix - Fix saving to JPEG when using linear colorspace. - Fix the `Cache-Control` and `Expires` headers passthrough when SVG is sanitized or fixed. +- (pro) Fix complexity calculation for still images. -## [3.18.1] - 2023-07-29 +## [3.18.1] - 2023-06-29 ### Change - Change maximum and default values of `IMGPROXY_AVIF_SPEED` to `9`. - (pro) Fix detection of some videos. From d9ad1aee148ddc4c46a4154f35f5add98d63497f Mon Sep 17 00:00:00 2001 From: DarthSim Date: Thu, 13 Jul 2023 14:59:36 +0300 Subject: [PATCH 038/138] Update base Docker image --- CHANGELOG.md | 1 + docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91fbac7dc8..d10f58457f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Fix saving to JPEG when using linear colorspace. - Fix the `Cache-Control` and `Expires` headers passthrough when SVG is sanitized or fixed. - (pro) Fix complexity calculation for still images. +- (docker) Fix crashes during some resizing cases. ## [3.18.1] - 2023-06-29 ### Change diff --git a/docker/Dockerfile b/docker/Dockerfile index 8167733cad..2be0afe2f1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE_VERSION="v3.6.0" +ARG BASE_IMAGE_VERSION="v3.6.1" FROM darthsim/imgproxy-base:${BASE_IMAGE_VERSION} From d9bc546dc08f30a1188162f2bdfcf0acad722fe4 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Thu, 13 Jul 2023 15:16:14 +0300 Subject: [PATCH 039/138] Bump version --- CHANGELOG.md | 2 ++ version/version.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d10f58457f..7e1b1dea32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased] + +## [3.18.2] - 2023-07-13 ### Fix - Fix saving to JPEG when using linear colorspace. - Fix the `Cache-Control` and `Expires` headers passthrough when SVG is sanitized or fixed. diff --git a/version/version.go b/version/version.go index aa6a312a97..cae0601ffc 100644 --- a/version/version.go +++ b/version/version.go @@ -1,6 +1,6 @@ package version -const version = "3.18.1" +const version = "3.18.2" func Version() string { return version From 94492e812c283b913dbe75c06a789f9f76bee4a6 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Wed, 19 Jul 2023 16:31:00 +0300 Subject: [PATCH 040/138] Don't report `context cancelled` errors to metrics collectors --- CHANGELOG.md | 2 ++ processing_handler.go | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e1b1dea32..543ecdbdac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased] +### Change +- Don't report `The image request is cancelled` errors. ## [3.18.2] - 2023-07-13 ### Fix diff --git a/processing_handler.go b/processing_handler.go index b8aa5ff089..3b479d7b40 100644 --- a/processing_handler.go +++ b/processing_handler.go @@ -167,7 +167,7 @@ func respondWithNotModified(reqID string, r *http.Request, rw http.ResponseWrite ) } -func sendErrAndPanic(ctx context.Context, errType string, err error) { +func sendErr(ctx context.Context, errType string, err error) { send := true if ierr, ok := err.(*ierrors.Error); ok { @@ -183,7 +183,10 @@ func sendErrAndPanic(ctx context.Context, errType string, err error) { if send { metrics.SendError(ctx, errType, err) } +} +func sendErrAndPanic(ctx context.Context, errType string, err error) { + sendErr(ctx, errType, err) panic(err) } @@ -329,7 +332,7 @@ func handleProcessing(reqID string, rw http.ResponseWriter, r *http.Request) { errorreport.Report(err, r) } - metrics.SendError(ctx, "download", err) + sendErr(ctx, "download", err) if imagedata.FallbackImage == nil { panic(err) From fd2566a489ff860678d8759afcdc07737b79fce2 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 25 Jul 2023 18:58:05 +0300 Subject: [PATCH 041/138] Fix IMGPROXY_CACHE_CONTROL_PASSTHROUGH + IMGPROXY_FALLBACK_IMAGE_TTL behavior --- CHANGELOG.md | 4 ++++ processing_handler.go | 16 ++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 543ecdbdac..b60c4db499 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ ### Change - Don't report `The image request is cancelled` errors. +### Fix +- Fix the `Cache-Control` and `Expires` headers behavior when both `IMGPROXY_CACHE_CONTROL_PASSTHROUGH` and `IMGPROXY_FALLBACK_IMAGE_TTL` configs are set. +- (pro) Fix the `IMGPROXY_FALLBACK_IMAGE_TTL` config behavior when the `fallback_image_url` processing option is used. + ## [3.18.2] - 2023-07-13 ### Fix - Fix saving to JPEG when using linear colorspace. diff --git a/processing_handler.go b/processing_handler.go index 3b479d7b40..22b2a6bb24 100644 --- a/processing_handler.go +++ b/processing_handler.go @@ -57,15 +57,20 @@ func initProcessingHandler() { func setCacheControl(rw http.ResponseWriter, force *time.Time, originHeaders map[string]string) { var cacheControl, expires string - var ttl int - if force != nil { + ttl := -1 + + if _, ok := originHeaders["Fallback-Image"]; ok && config.FallbackImageTTL > 0 { + ttl = config.FallbackImageTTL + } + + if force != nil && (ttl < 0 || force.Before(time.Now().Add(time.Duration(ttl)*time.Second))) { rw.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d, public", int(time.Until(*force).Seconds()))) rw.Header().Set("Expires", force.Format(http.TimeFormat)) return } - if config.CacheControlPassthrough && originHeaders != nil { + if config.CacheControlPassthrough && ttl < 0 && originHeaders != nil { if val, ok := originHeaders["Cache-Control"]; ok && len(val) > 0 { cacheControl = val } @@ -75,9 +80,8 @@ func setCacheControl(rw http.ResponseWriter, force *time.Time, originHeaders map } if len(cacheControl) == 0 && len(expires) == 0 { - ttl = config.TTL - if _, ok := originHeaders["Fallback-Image"]; ok && config.FallbackImageTTL > 0 { - ttl = config.FallbackImageTTL + if ttl < 0 { + ttl = config.TTL } cacheControl = fmt.Sprintf("max-age=%d, public", ttl) expires = time.Now().Add(time.Second * time.Duration(ttl)).Format(http.TimeFormat) From 35dcf5a2613b80185f71d71d71d3acf3567fb4c0 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 1 Aug 2023 18:45:27 +0300 Subject: [PATCH 042/138] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b60c4db499..bafc7e9f6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Unreleased] ### Change - Don't report `The image request is cancelled` errors. +- (pro) Change the `/info` endpoint behavior to return only the first EXIF/XMP/IPTC block data of JPEG if the image contains multiple metadata blocks of the same type. ### Fix - Fix the `Cache-Control` and `Expires` headers behavior when both `IMGPROXY_CACHE_CONTROL_PASSTHROUGH` and `IMGPROXY_FALLBACK_IMAGE_TTL` configs are set. From a020a7603ef4856a3ee4b77dc086014ffa2b7b0a Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 1 Aug 2023 19:48:10 +0300 Subject: [PATCH 043/138] Fix reporting image loading errors --- CHANGELOG.md | 3 +++ router/logging.go | 2 +- vips/vips.go | 6 ++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bafc7e9f6c..b468b311eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ - Don't report `The image request is cancelled` errors. - (pro) Change the `/info` endpoint behavior to return only the first EXIF/XMP/IPTC block data of JPEG if the image contains multiple metadata blocks of the same type. +### Fix +- Fix reporting image loading errors. + ### Fix - Fix the `Cache-Control` and `Expires` headers behavior when both `IMGPROXY_CACHE_CONTROL_PASSTHROUGH` and `IMGPROXY_FALLBACK_IMAGE_TTL` configs are set. - (pro) Fix the `IMGPROXY_FALLBACK_IMAGE_TTL` config behavior when the `fallback_image_url` processing option is used. diff --git a/router/logging.go b/router/logging.go index 1c2b02414c..df84117ed7 100644 --- a/router/logging.go +++ b/router/logging.go @@ -24,7 +24,7 @@ func LogResponse(reqID string, r *http.Request, status int, err *ierrors.Error, var level log.Level switch { - case status >= 500: + case status >= 500 || (err != nil && err.Unexpected): level = log.ErrorLevel case status >= 400: level = log.WarnLevel diff --git a/vips/vips.go b/vips/vips.go index 3154ec9626..829d4c7667 100644 --- a/vips/vips.go +++ b/vips/vips.go @@ -155,12 +155,14 @@ func Error() error { defer C.vips_error_clear() errstr := strings.TrimSpace(C.GoString(C.vips_error_buffer())) + err := ierrors.NewUnexpected(errstr, 1) if strings.Contains(errstr, "load_buffer: ") { - return ierrors.New(422, errstr, "Broken or unsupported image") + err.StatusCode = 422 + err.PublicMessage = "Broken or unsupported image" } - return ierrors.NewUnexpected(errstr, 1) + return err } func hasOperation(name string) bool { From 3557fa2c4e30cb4453532f0a09a4e425980e4b5e Mon Sep 17 00:00:00 2001 From: DarthSim Date: Wed, 2 Aug 2023 21:32:51 +0300 Subject: [PATCH 044/138] Add S3 multi-region mode --- config/config.go | 3 + docs/configuration.md | 2 + docs/serving_files_from_s3.md | 17 ++-- transport/s3/s3.go | 149 ++++++++++++++++++++++++++-------- transport/s3/s3_test.go | 19 ++++- 5 files changed, 146 insertions(+), 44 deletions(-) diff --git a/config/config.go b/config/config.go index 1397546dd0..de0729970c 100644 --- a/config/config.go +++ b/config/config.go @@ -103,6 +103,7 @@ var ( S3Region string S3Endpoint string S3AssumeRoleArn string + S3MultiRegion bool GCSEnabled bool GCSKey string @@ -298,6 +299,7 @@ func Reset() { S3Region = "" S3Endpoint = "" S3AssumeRoleArn = "" + S3MultiRegion = false GCSEnabled = false GCSKey = "" ABSEnabled = false @@ -497,6 +499,7 @@ func Configure() error { configurators.String(&S3Region, "IMGPROXY_S3_REGION") configurators.String(&S3Endpoint, "IMGPROXY_S3_ENDPOINT") configurators.String(&S3AssumeRoleArn, "IMGPROXY_S3_ASSUME_ROLE_ARN") + configurators.Bool(&S3MultiRegion, "IMGPROXY_S3_MULTI_REGION") configurators.Bool(&GCSEnabled, "IMGPROXY_USE_GCS") configurators.String(&GCSKey, "IMGPROXY_GCS_KEY") diff --git a/docs/configuration.md b/docs/configuration.md index 1d8e11a72d..c8b41f87a4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -366,7 +366,9 @@ Check out the [Serving local files](serving_local_files.md) guide to learn more. imgproxy can process files from Amazon S3 buckets, but this feature is disabled by default. To enable it, set `IMGPROXY_USE_S3` to `true`: * `IMGPROXY_USE_S3`: when `true`, enables image fetching from Amazon S3 buckets. Default: `false` +* `IMGPROXY_S3_REGION`: an S3 buckets region * `IMGPROXY_S3_ENDPOINT`: a custom S3 endpoint to being used by imgproxy +* `IMGPROXY_S3_MULTI_REGION`: when `true`, allows using S3 buckets from different regions Check out the [Serving files from S3](serving_files_from_s3.md) guide to learn more. diff --git a/docs/serving_files_from_s3.md b/docs/serving_files_from_s3.md index 3487433549..a45c630c53 100644 --- a/docs/serving_files_from_s3.md +++ b/docs/serving_files_from_s3.md @@ -6,8 +6,9 @@ imgproxy can process images from S3 buckets. To use this feature, do the followi 2. [Set up the necessary credentials](#set-up-credentials) to grant access to your bucket. 3. _(optional)_ Specify the AWS region with `IMGPROXY_S3_REGION` or `AWS_REGION`. Default: `us-west-1` 4. _(optional)_ Specify the S3 endpoint with `IMGPROXY_S3_ENDPOINT`. -5. _(optional)_ Specify the AWS IAM Role to Assume with `IMGPROXY_S3_ASSUME_ROLE_ARN` -6. Use `s3://%bucket_name/%file_key` as the source image URL. +5. _(optional)_ Set the `IMGPROXY_S3_MULTI_REGION` environment variable to be `true`. +6. _(optional)_ Specify the AWS IAM Role to Assume with `IMGPROXY_S3_ASSUME_ROLE_ARN` +7. Use `s3://%bucket_name/%file_key` as the source image URL. If you need to specify the version of the source object, you can use the query string of the source URL: @@ -54,11 +55,17 @@ aws_secret_access_key = %secret_access_key S3 access credentials may be acquired by assuming a role using STS. To do so specify the IAM Role arn with the `IMGPROXY_S3_ASSUME_ROLE_ARN` environment variable. This approach still requires you to provide initial AWS credentials by using one of the ways described above. The provided credentials role should allow assuming the role with provided ARN. -## Minio +## Multi-Region mode -[Minio](https://github.com/minio/minio) is an object storage server released under Apache License v2.0. It is compatible with Amazon S3, so it can be used with imgproxy. +By default, imgproxy allows using S3 buckets located in a single region specified with `IMGPROXY_S3_REGION` or `AWS_REGION`. If your buckets are located in different regions, set `IMGPROXY_S3_MULTI_REGION` environment variable to be `true` to enable multi-region mode. In this mode, imgproxy will make an additional request to determine the bucket's region when the bucket is accessed for the first time. -To use Minio as source images provider, do the following: +In this mode, imgroxy uses a region specified with `IMGPROXY_S3_REGION` or `AWS_REGION` to determine the endpoint to which it should send the bucket's region determination request. Thus, it's a good idea to use one of these variables to specify a region closest to the imgproxy instance. + +## MinIO + +[MinIO](https://github.com/minio/minio) is an object storage server released under Apache License v2.0. It is compatible with Amazon S3, so it can be used with imgproxy. + +To use MinIO as source images provider, do the following: * Set up Amazon S3 support as usual using environment variables or a shared config file. * Specify an endpoint with `IMGPROXY_S3_ENDPOINT`. Use the `http://...` endpoint to disable SSL. diff --git a/transport/s3/s3.go b/transport/s3/s3.go index 3033d07abf..60036985aa 100644 --- a/transport/s3/s3.go +++ b/transport/s3/s3.go @@ -1,10 +1,12 @@ package s3 import ( + "context" "fmt" "io" http "net/http" "strings" + "sync" "time" "github.com/aws/aws-sdk-go/aws" @@ -13,6 +15,7 @@ import ( "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/imgproxy/imgproxy/v3/config" defaultTransport "github.com/imgproxy/imgproxy/v3/transport" @@ -20,26 +23,28 @@ import ( // transport implements RoundTripper for the 's3' protocol. type transport struct { - svc *s3.S3 + session *session.Session + defaultClient *s3.S3 + + clientsByRegion map[string]*s3.S3 + clientsByBucket map[string]*s3.S3 + + mu sync.RWMutex } func New() (http.RoundTripper, error) { - s3Conf := aws.NewConfig() + conf := aws.NewConfig() trans, err := defaultTransport.New(false) if err != nil { return nil, err } - s3Conf.HTTPClient = &http.Client{Transport: trans} - - if len(config.S3Region) != 0 { - s3Conf.Region = aws.String(config.S3Region) - } + conf.HTTPClient = &http.Client{Transport: trans} if len(config.S3Endpoint) != 0 { - s3Conf.Endpoint = aws.String(config.S3Endpoint) - s3Conf.S3ForcePathStyle = aws.Bool(true) + conf.Endpoint = aws.String(config.S3Endpoint) + conf.S3ForcePathStyle = aws.Bool(true) } sess, err := session.NewSession() @@ -47,18 +52,35 @@ func New() (http.RoundTripper, error) { return nil, fmt.Errorf("Can't create S3 session: %s", err) } - if len(config.S3AssumeRoleArn) != 0 { - s3Conf.Credentials = stscreds.NewCredentials(sess, config.S3AssumeRoleArn) + if len(config.S3Region) != 0 { + sess.Config.Region = aws.String(config.S3Region) } if sess.Config.Region == nil || len(*sess.Config.Region) == 0 { sess.Config.Region = aws.String("us-west-1") } - return transport{s3.New(sess, s3Conf)}, nil + if len(config.S3AssumeRoleArn) != 0 { + conf.Credentials = stscreds.NewCredentials(sess, config.S3AssumeRoleArn) + } + + client := s3.New(sess, conf) + + clientRegion := "us-west-1" + if client.Config.Region != nil { + clientRegion = *client.Config.Region + } + + return &transport{ + session: sess, + defaultClient: client, + + clientsByRegion: map[string]*s3.S3{clientRegion: client}, + clientsByBucket: make(map[string]*s3.S3), + }, nil } -func (t transport) RoundTrip(req *http.Request) (resp *http.Response, err error) { +func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { input := &s3.GetObjectInput{ Bucket: aws.String(req.URL.Host), Key: aws.String(req.URL.Path), @@ -86,7 +108,12 @@ func (t transport) RoundTrip(req *http.Request) (resp *http.Response, err error) } } - s3req, _ := t.svc.GetObjectRequest(input) + client, err := t.getClient(req.Context(), *input.Bucket) + if err != nil { + return handleError(req, err) + } + + s3req, _ := client.GetObjectRequest(input) s3req.SetContext(req.Context()) if err := s3req.Send(); err != nil { @@ -94,29 +121,81 @@ func (t transport) RoundTrip(req *http.Request) (resp *http.Response, err error) s3req.HTTPResponse.Body.Close() } - if s3err, ok := err.(awserr.Error); ok && s3err.Code() == request.CanceledErrorCode { - if e := s3err.OrigErr(); e != nil { - return nil, e - } - } + return handleError(req, err) + } + + return s3req.HTTPResponse, nil +} - if s3err, ok := err.(awserr.RequestFailure); !ok || s3err.StatusCode() < 100 || s3err.StatusCode() == 301 { - return nil, err - } else { - body := strings.NewReader(s3err.Message()) - return &http.Response{ - StatusCode: s3err.StatusCode(), - Proto: "HTTP/1.0", - ProtoMajor: 1, - ProtoMinor: 0, - Header: http.Header{}, - ContentLength: int64(body.Len()), - Body: io.NopCloser(body), - Close: false, - Request: s3req.HTTPRequest, - }, nil +func (t *transport) getClient(ctx context.Context, bucket string) (*s3.S3, error) { + if !config.S3MultiRegion { + return t.defaultClient, nil + } + + var client *s3.S3 + + func() { + t.mu.RLock() + defer t.mu.RUnlock() + client = t.clientsByBucket[bucket] + }() + + if client != nil { + return client, nil + } + + t.mu.Lock() + defer t.mu.Unlock() + + // Check again if someone did this before us + if client = t.clientsByBucket[bucket]; client != nil { + return client, nil + } + + region, err := s3manager.GetBucketRegionWithClient(ctx, t.defaultClient, bucket) + if err != nil { + return nil, err + } + + if client = t.clientsByRegion[region]; client != nil { + t.clientsByBucket[bucket] = client + return client, nil + } + + conf := t.defaultClient.Config.Copy() + conf.Region = aws.String(region) + + client = s3.New(t.session, conf) + + t.clientsByRegion[region] = client + t.clientsByBucket[bucket] = client + + return client, nil +} + +func handleError(req *http.Request, err error) (*http.Response, error) { + if s3err, ok := err.(awserr.Error); ok && s3err.Code() == request.CanceledErrorCode { + if e := s3err.OrigErr(); e != nil { + return nil, e } } - return s3req.HTTPResponse, nil + s3err, ok := err.(awserr.RequestFailure) + if !ok || s3err.StatusCode() < 100 || s3err.StatusCode() == 301 { + return nil, err + } + + body := strings.NewReader(s3err.Message()) + + return &http.Response{ + StatusCode: s3err.StatusCode(), + Proto: "HTTP/1.0", + ProtoMajor: 1, + ProtoMinor: 0, + Header: http.Header{}, + ContentLength: int64(body.Len()), + Body: io.NopCloser(body), + Close: false, + Request: req, + }, nil } diff --git a/transport/s3/s3_test.go b/transport/s3/s3_test.go index 80b1b8d24d..03259e5914 100644 --- a/transport/s3/s3_test.go +++ b/transport/s3/s3_test.go @@ -2,6 +2,7 @@ package s3 import ( "bytes" + "context" "net/http" "net/http/httptest" "os" @@ -43,12 +44,12 @@ func (s *S3TestSuite) SetupSuite() { s.transport, err = New() require.Nil(s.T(), err) - svc := s.transport.(transport).svc + err = backend.CreateBucket("test") + require.Nil(s.T(), err) - _, err = svc.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String("test"), - }) + svc, err := s.transport.(*transport).getClient(context.Background(), "test") require.Nil(s.T(), err) + require.NotNil(s.T(), svc) _, err = svc.PutObject(&s3.PutObjectInput{ Body: bytes.NewReader(make([]byte, 32)), @@ -70,6 +71,7 @@ func (s *S3TestSuite) SetupSuite() { func (s *S3TestSuite) TearDownSuite() { s.server.Close() + config.Reset() } func (s *S3TestSuite) TestRoundTripWithETagDisabledReturns200() { @@ -155,6 +157,15 @@ func (s *S3TestSuite) TestRoundTripWithUpdatedLastModifiedReturns200() { require.Equal(s.T(), http.StatusOK, response.StatusCode) } +func (s *S3TestSuite) TestRoundTripWithMultiregionEnabledReturns200() { + config.S3MultiRegion = true + request, _ := http.NewRequest("GET", "s3://test/foo/test.png", nil) + + response, err := s.transport.RoundTrip(request) + require.Nil(s.T(), err) + require.Equal(s.T(), 200, response.StatusCode) +} + func TestS3Transport(t *testing.T) { suite.Run(t, new(S3TestSuite)) } From 64fc31f3f3aa2e081deb97bf23e18ffa8aa8b19c Mon Sep 17 00:00:00 2001 From: DarthSim Date: Wed, 2 Aug 2023 21:38:31 +0300 Subject: [PATCH 045/138] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b468b311eb..8b144bf7b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog ## [Unreleased] +### Add +- Add [multi-region mode](https://docs.imgproxy.net/latest/serving_files_from_s3?id=multi-region-mode) to S3 integration. + ### Change - Don't report `The image request is cancelled` errors. - (pro) Change the `/info` endpoint behavior to return only the first EXIF/XMP/IPTC block data of JPEG if the image contains multiple metadata blocks of the same type. From 1d210dde3b1595c3c528f2894a58f93be131fd72 Mon Sep 17 00:00:00 2001 From: Aaron Date: Sat, 24 Jun 2023 18:30:01 -0700 Subject: [PATCH 046/138] Add Clojure signature example --- examples/signature.clj | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 examples/signature.clj diff --git a/examples/signature.clj b/examples/signature.clj new file mode 100644 index 0000000000..2a383ba839 --- /dev/null +++ b/examples/signature.clj @@ -0,0 +1,33 @@ +(ns imgproxy-test + (:require + [clojure.test :refer [deftest is]]) + (:import + (java.util Base64) + (javax.crypto Mac) + (javax.crypto.spec SecretKeySpec))) + +(defn hex-string-to-byte-array + [hex] + (let [res (byte-array (/ (count hex) 2))] + (dotimes [i (count res)] + (aset res + i + (unchecked-byte (bit-or (bit-shift-left (Character/digit (nth hex (* i 2)) 16) 4) + (Character/digit (nth hex (+ (* i 2) 1)) 16))))) + res)) + +(defn sign-path + [key salt path] + (let [mac (doto (Mac/getInstance "HmacSHA256") (.init (SecretKeySpec. key "HmacSHA256"))) + _ (.update mac salt) + hash (.doFinal mac (.getBytes path "UTF-8")) + encoded-hash (.. (Base64/getUrlEncoder) withoutPadding (encodeToString hash))] + (str "/" encoded-hash path))) + +(deftest test-with-hmac-base64-img-proxy-test + (is + (= + "/m3k5QADfcKPDj-SDI2AIogZbC3FlAXszuwhtWXYqavc/rs:fit:300:300/plain/http://img.example.com/pretty/image.jpg" + (sign-path (hex-string-to-byte-array "943b421c9eb07c830af81030552c86009268de4e532ba2ee2eab8247c6da0881") + (hex-string-to-byte-array "520f986b998545b4785e0defbc4f3c1203f22de2374a3d53cb7a7fe9fea309c5") + "/rs:fit:300:300/plain/http://img.example.com/pretty/image.jpg")))) From 417a53d3ce9aaeb6f6b50cdec58067928eda2e6e Mon Sep 17 00:00:00 2001 From: Aaron Date: Sat, 24 Jun 2023 18:30:31 -0700 Subject: [PATCH 047/138] Fix missing character in java example --- examples/signature.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/signature.java b/examples/signature.java index 3d85848857..aaa7f373d0 100644 --- a/examples/signature.java +++ b/examples/signature.java @@ -20,7 +20,7 @@ public void testWithJavaHmacApacheBase64ImgProxyTest() throws Exception { String pathWithHash = signPath(key, salt, path); - assertEquals("/m3k5QADfcKPDj-SDI2AIogZbC3FlAXszuwhtWXYqavc/rs:fit:300:300/plain/http://img.example.com/pretty/image.jp", pathWithHash); + assertEquals("/m3k5QADfcKPDj-SDI2AIogZbC3FlAXszuwhtWXYqavc/rs:fit:300:300/plain/http://img.example.com/pretty/image.jpg", pathWithHash); } public static String signPath(byte[] key, byte[] salt, String path) throws Exception { From e976424c1e958cac4b49b86a66f1c6ce7fb0aa5b Mon Sep 17 00:00:00 2001 From: Sergey Alexandrovich Date: Fri, 11 Aug 2023 20:12:56 +0600 Subject: [PATCH 048/138] Create FUNDING.yml --- .github/FUNDING.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000000..22675661ab --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: imgproxy From e172b14377c766ffacf1bdec62294e737150c4b1 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 15 Aug 2023 17:55:01 +0300 Subject: [PATCH 049/138] [docs] Add anchors for configs on the configuration page --- docs/assets/docsify-init.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/assets/docsify-init.js b/docs/assets/docsify-init.js index 3b01c4e4c2..573d8b599c 100644 --- a/docs/assets/docsify-init.js +++ b/docs/assets/docsify-init.js @@ -26,6 +26,8 @@ const proLink = `"; +const configRegex = /^\* `([^`]+)`:/gm; + const defaultVersions = [["latest", "latest"]]; const configureDocsify = (additionalVersions, latestVersion, latestTag) => { @@ -124,6 +126,10 @@ const configureDocsify = (additionalVersions, latestVersion, latestTag) => { hook.beforeEach((content, next) => { content = content.replaceAll(proBadgeRegex, proLink); content = content.replaceAll(oldProBadge, proLink); + + if (vm.route.path.endsWith('/configuration')) + content = content.replaceAll(configRegex, '* $1:'); + next(content); }) From 07e34a45f2fc1c22a9307a459843d2a0198fa39e Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 15 Aug 2023 19:54:42 +0300 Subject: [PATCH 050/138] Rename concurrency to workers --- CHANGELOG.md | 3 +-- config/config.go | 13 +++++++------ docs/cloud_watch.md | 2 +- docs/configuration.md | 2 +- docs/generating_the_url.md | 2 +- imagedata/read.go | 2 +- metrics/cloudwatch/cloudwatch.go | 11 ++++++++++- processing_handler.go | 6 +++--- transport/transport.go | 2 +- 9 files changed, 26 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b144bf7b4..14fd1b3fc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Unreleased] ### Add - Add [multi-region mode](https://docs.imgproxy.net/latest/serving_files_from_s3?id=multi-region-mode) to S3 integration. +- Add `IMGPROXY_WORKERS` alias for the `IMGPROXY_CONCURRENCY` config. ### Change - Don't report `The image request is cancelled` errors. @@ -10,8 +11,6 @@ ### Fix - Fix reporting image loading errors. - -### Fix - Fix the `Cache-Control` and `Expires` headers behavior when both `IMGPROXY_CACHE_CONTROL_PASSTHROUGH` and `IMGPROXY_FALLBACK_IMAGE_TTL` configs are set. - (pro) Fix the `IMGPROXY_FALLBACK_IMAGE_TTL` config behavior when the `fallback_image_url` processing option is used. diff --git a/config/config.go b/config/config.go index de0729970c..036808ab0e 100644 --- a/config/config.go +++ b/config/config.go @@ -26,7 +26,7 @@ var ( KeepAliveTimeout int ClientKeepAliveTimeout int DownloadTimeout int - Concurrency int + Workers int RequestsQueueSize int MaxClients int @@ -219,7 +219,7 @@ func Reset() { KeepAliveTimeout = 10 ClientKeepAliveTimeout = 90 DownloadTimeout = 5 - Concurrency = runtime.GOMAXPROCS(0) * 2 + Workers = runtime.GOMAXPROCS(0) * 2 RequestsQueueSize = 0 MaxClients = 2048 @@ -400,7 +400,8 @@ func Configure() error { configurators.Int(&KeepAliveTimeout, "IMGPROXY_KEEP_ALIVE_TIMEOUT") configurators.Int(&ClientKeepAliveTimeout, "IMGPROXY_CLIENT_KEEP_ALIVE_TIMEOUT") configurators.Int(&DownloadTimeout, "IMGPROXY_DOWNLOAD_TIMEOUT") - configurators.Int(&Concurrency, "IMGPROXY_CONCURRENCY") + configurators.Int(&Workers, "IMGPROXY_CONCURRENCY") + configurators.Int(&Workers, "IMGPROXY_WORKERS") configurators.Int(&RequestsQueueSize, "IMGPROXY_REQUESTS_QUEUE_SIZE") configurators.Int(&MaxClients, "IMGPROXY_MAX_CLIENTS") @@ -625,8 +626,8 @@ func Configure() error { return fmt.Errorf("Download timeout should be greater than 0, now - %d\n", DownloadTimeout) } - if Concurrency <= 0 { - return fmt.Errorf("Concurrency should be greater than 0, now - %d\n", Concurrency) + if Workers <= 0 { + return fmt.Errorf("Workers number should be greater than 0, now - %d\n", Workers) } if RequestsQueueSize < 0 { @@ -634,7 +635,7 @@ func Configure() error { } if MaxClients < 0 { - return fmt.Errorf("Concurrency should be greater than or equal 0, now - %d\n", MaxClients) + return fmt.Errorf("Max clients number should be greater than or equal 0, now - %d\n", MaxClients) } if TTL <= 0 { diff --git a/docs/cloud_watch.md b/docs/cloud_watch.md index 5388958868..f626c78ded 100644 --- a/docs/cloud_watch.md +++ b/docs/cloud_watch.md @@ -11,7 +11,7 @@ imgproxy sends the following metrics to CloudWatch: * `RequestsInProgress`: the number of requests currently in progress * `ImagesInProgress`: the number of images currently in progress -* `ConcurrencyUtilization`: the percentage of imgproxy's concurrency utilization. Calculated as `RequestsInProgress / IMGPROXY_CONCURRENCY * 100` +* `WorkersUtilization`, `ConcurrencyUtilization`: the percentage of imgproxy's workers utilization. Calculated as `RequestsInProgress / IMGPROXY_WORKERS * 100` * `BufferSize`: a summary of the download buffers sizes (in bytes) * `BufferDefaultSize`: calibrated default buffer size (in bytes) * `BufferMaxSize`: calibrated maximum buffer size (in bytes) diff --git a/docs/configuration.md b/docs/configuration.md index c8b41f87a4..e8150689dc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -33,7 +33,7 @@ echo $(xxd -g 2 -l 64 -p /dev/random | tr -d '\n') * `IMGPROXY_KEEP_ALIVE_TIMEOUT`: the maximum duration (in seconds) to wait for the next request before closing the connection. When set to `0`, keep-alive is disabled. Default: `10` * `IMGPROXY_CLIENT_KEEP_ALIVE_TIMEOUT`: the maximum duration (in seconds) to wait for the next request before closing the HTTP client connection. The HTTP client is used to download source images. When set to `0`, keep-alive is disabled. Default: `90` * `IMGPROXY_DOWNLOAD_TIMEOUT`: the maximum duration (in seconds) for downloading the source image. Default: `5` -* `IMGPROXY_CONCURRENCY`: the maximum number of image requests to be processed simultaneously. Requests that exceed this limit are put in the queue. Default: the number of CPU cores multiplied by two +* `IMGPROXY_WORKERS`: _(alias: `IMGPROXY_CONCURRENCY`)_ the maximum number of images an imgproxy instance can process simultaneously without creating a queue. Default: the number of CPU cores multiplied by two * `IMGPROXY_REQUESTS_QUEUE_SIZE`: the maximum number of image requests that can be put in the queue. Requests that exceed this limit are rejected with `429` HTTP status. When set to `0`, the requests queue is unlimited. Default: `0` * `IMGPROXY_MAX_CLIENTS`: the maximum number of simultaneous active connections. When set to `0`, connection limit is disabled. Default: `2048` * `IMGPROXY_TTL`: a duration (in seconds) sent via the `Expires` and `Cache-Control: max-age` HTTP headers. Default: `31536000` (1 year) diff --git a/docs/generating_the_url.md b/docs/generating_the_url.md index a4572601a3..2a15929a77 100644 --- a/docs/generating_the_url.md +++ b/docs/generating_the_url.md @@ -746,7 +746,7 @@ When set to `1`, `t` or `true`, imgproxy will respond with a raw unprocessed, an * While the `skip_processing` option has some conditions to skip the processing, the `raw` option allows to skip processing no matter what * With the `raw` option set, imgproxy doesn't check the source image's type, resolution, and file size. Basically, the `raw` option allows streaming of any file type * With the `raw` option set, imgproxy won't download the whole image to the memory. Instead, it will stream the source image directly to the response lowering memory usage -* The requests with the `raw` option set are not limited by the `IMGPROXY_CONCURRENCY` config +* The requests with the `raw` option set are not limited by the `IMGPROXY_WORKERS` config Default: `false` diff --git a/imagedata/read.go b/imagedata/read.go index 531528e35e..0ab9e4c363 100644 --- a/imagedata/read.go +++ b/imagedata/read.go @@ -18,7 +18,7 @@ var ErrSourceImageTypeNotSupported = ierrors.New(422, "Source image type not sup var downloadBufPool *bufpool.Pool func initRead() { - downloadBufPool = bufpool.New("download", config.Concurrency, config.DownloadBufferSize) + downloadBufPool = bufpool.New("download", config.Workers, config.DownloadBufferSize) } func readAndCheckImage(r io.Reader, contentLength int, secopts security.Options) (*ImageData, error) { diff --git a/metrics/cloudwatch/cloudwatch.go b/metrics/cloudwatch/cloudwatch.go index b0b1fac050..49d55ea7bd 100644 --- a/metrics/cloudwatch/cloudwatch.go +++ b/metrics/cloudwatch/cloudwatch.go @@ -232,7 +232,16 @@ func runMetricsCollector() { MetricName: aws.String("ConcurrencyUtilization"), Unit: aws.String("Percent"), Value: aws.Float64( - stats.RequestsInProgress() / float64(config.Concurrency) * 100.0, + stats.RequestsInProgress() / float64(config.Workers) * 100.0, + ), + }) + + metrics = append(metrics, &cloudwatch.MetricDatum{ + Dimensions: []*cloudwatch.Dimension{dimension}, + MetricName: aws.String("WorkersUtilization"), + Unit: aws.String("Percent"), + Value: aws.Float64( + stats.RequestsInProgress() / float64(config.Workers) * 100.0, ), }) diff --git a/processing_handler.go b/processing_handler.go index 22b2a6bb24..ad1d787698 100644 --- a/processing_handler.go +++ b/processing_handler.go @@ -37,10 +37,10 @@ var ( func initProcessingHandler() { if config.RequestsQueueSize > 0 { - queueSem = semaphore.New(config.RequestsQueueSize + config.Concurrency) + queueSem = semaphore.New(config.RequestsQueueSize + config.Workers) } - processingSem = semaphore.New(config.Concurrency) + processingSem = semaphore.New(config.Workers) vary := make([]string, 0) @@ -282,7 +282,7 @@ func handleProcessing(reqID string, rw http.ResponseWriter, r *http.Request) { } } - // The heavy part start here, so we need to restrict concurrency + // The heavy part start here, so we need to restrict workers number var processingSemToken *semaphore.Token func() { defer metrics.StartQueueSegment(ctx)() diff --git a/transport/transport.go b/transport/transport.go index 15ce16dcb9..cd60f74956 100644 --- a/transport/transport.go +++ b/transport/transport.go @@ -30,7 +30,7 @@ func New(verifyNetworks bool) (*http.Transport, error) { Proxy: http.ProxyFromEnvironment, DialContext: dialer.DialContext, MaxIdleConns: 100, - MaxIdleConnsPerHost: config.Concurrency + 1, + MaxIdleConnsPerHost: config.Workers + 1, IdleConnTimeout: time.Duration(config.ClientKeepAliveTimeout) * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, From 3b3720fbb507bc41a8936c732d2cd5abc9d0f693 Mon Sep 17 00:00:00 2001 From: Ray Date: Sun, 6 Aug 2023 14:00:19 +1000 Subject: [PATCH 051/138] Fix typo in cloud_watch.md --- docs/cloud_watch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cloud_watch.md b/docs/cloud_watch.md index f626c78ded..ac90ffaa6c 100644 --- a/docs/cloud_watch.md +++ b/docs/cloud_watch.md @@ -1,6 +1,6 @@ # Amazon CloudWatch -imgproxy can send its metrics to AmazonCloudFront. To use this feature, do the following: +imgproxy can send its metrics to Amazon CloudWatch. To use this feature, do the following: 1. Set the `IMGPROXY_CLOUD_WATCH_SERVICE_NAME` environment variable. imgproxy will use the value of this variable as a value for the `ServiceName` dimension. 2. [Set up the necessary credentials](#set-up-credentials) to grant access to CloudWatch. From 93b1dc575740cee9828f8fb95fd73702436f0637 Mon Sep 17 00:00:00 2001 From: DarthSim Date: Tue, 28 Feb 2023 19:40:56 +0300 Subject: [PATCH 052/138] Load env variables from env files or secrets --- CHANGELOG.md | 1 + config/loadenv/aws.go | 121 +++++++++++++++++++ config/loadenv/gcp.go | 82 +++++++++++++ config/loadenv/loadenv.go | 21 ++++ config/loadenv/local_file.go | 38 ++++++ docs/_sidebar.md | 4 +- docs/cloud_watch.md | 2 - docs/loading_environment_variables.md | 160 ++++++++++++++++++++++++++ docs/serving_files_from_s3.md | 2 - go.mod | 2 + go.sum | 4 + main.go | 5 + 12 files changed, 437 insertions(+), 5 deletions(-) create mode 100644 config/loadenv/aws.go create mode 100644 config/loadenv/gcp.go create mode 100644 config/loadenv/loadenv.go create mode 100644 config/loadenv/local_file.go create mode 100644 docs/loading_environment_variables.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 14fd1b3fc3..9988ef3e6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Unreleased] ### Add - Add [multi-region mode](https://docs.imgproxy.net/latest/serving_files_from_s3?id=multi-region-mode) to S3 integration. +- Add the ability to [load environment variables](https://docs.imgproxy.net/latest/loading_environment_variables) from a file or a cloud secret. - Add `IMGPROXY_WORKERS` alias for the `IMGPROXY_CONCURRENCY` config. ### Change diff --git a/config/loadenv/aws.go b/config/loadenv/aws.go new file mode 100644 index 0000000000..abc184456e --- /dev/null +++ b/config/loadenv/aws.go @@ -0,0 +1,121 @@ +package loadenv + +import ( + "fmt" + "os" + "strings" + + "github.com/DarthSim/godotenv" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/secretsmanager" + "github.com/aws/aws-sdk-go/service/ssm" +) + +func loadAWSSecret() error { + secretID := os.Getenv("IMGPROXY_ENV_AWS_SECRET_ID") + secretVersionID := os.Getenv("IMGPROXY_ENV_AWS_SECRET_VERSION_ID") + secretVersionStage := os.Getenv("IMGPROXY_ENV_AWS_SECRET_VERSION_STAGE") + secretRegion := os.Getenv("IMGPROXY_ENV_AWS_SECRET_REGION") + + if len(secretID) == 0 { + return nil + } + + sess, err := session.NewSession() + if err != nil { + return fmt.Errorf("Can't create AWS Secrets Manager session: %s", err) + } + + conf := aws.NewConfig() + + if len(secretRegion) != 0 { + conf.Region = aws.String(secretRegion) + } + + svc := secretsmanager.New(sess, conf) + + input := secretsmanager.GetSecretValueInput{SecretId: aws.String(secretID)} + if len(secretVersionID) > 0 { + input.VersionId = aws.String(secretVersionID) + } else if len(secretVersionStage) > 0 { + input.VersionStage = aws.String(secretVersionStage) + } + + output, err := svc.GetSecretValue(&input) + if err != nil { + return fmt.Errorf("Can't retrieve config from AWS Secrets Manager: %s", err) + } + + if output.SecretString == nil { + return nil + } + + envmap, err := godotenv.Unmarshal(*output.SecretString) + if err != nil { + return fmt.Errorf("Can't parse config from AWS Secrets Manager: %s", err) + } + + for k, v := range envmap { + if err = os.Setenv(k, v); err != nil { + return fmt.Errorf("Can't set %s env variable from AWS Secrets Manager: %s", k, err) + } + } + + return nil +} + +func loadAWSSystemManagerParams() error { + paramsPath := os.Getenv("IMGPROXY_ENV_AWS_SSM_PARAMETERS_PATH") + paramsRegion := os.Getenv("IMGPROXY_ENV_AWS_SSM_PARAMETERS_REGION") + + if len(paramsPath) == 0 { + return nil + } + + sess, err := session.NewSession() + if err != nil { + return fmt.Errorf("Can't create AWS SSM session: %s", err) + } + + conf := aws.NewConfig() + + if len(paramsRegion) != 0 { + conf.Region = aws.String(paramsRegion) + } + + svc := ssm.New(sess, conf) + + input := ssm.GetParametersByPathInput{ + Path: aws.String(paramsPath), + WithDecryption: aws.Bool(true), + } + + output, err := svc.GetParametersByPath(&input) + if err != nil { + return fmt.Errorf("Can't retrieve parameters from AWS SSM: %s", err) + } + + for _, p := range output.Parameters { + if p == nil || p.Name == nil || p.Value == nil { + continue + } + + if p.DataType == nil || *p.DataType != "text" { + continue + } + + name := *p.Name + + env := strings.ReplaceAll( + strings.TrimPrefix(strings.TrimPrefix(name, paramsPath), "/"), + "/", "_", + ) + + if err = os.Setenv(env, *p.Value); err != nil { + return fmt.Errorf("Can't set %s env variable from AWS SSM: %s", env, err) + } + } + + return nil +} diff --git a/config/loadenv/gcp.go b/config/loadenv/gcp.go new file mode 100644 index 0000000000..0551f99bf5 --- /dev/null +++ b/config/loadenv/gcp.go @@ -0,0 +1,82 @@ +package loadenv + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + secretmanager "cloud.google.com/go/secretmanager/apiv1" + "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" + "github.com/DarthSim/godotenv" + "google.golang.org/api/option" +) + +func loadGCPSecret() error { + secretID := os.Getenv("IMGPROXY_ENV_GCP_SECRET_ID") + secretVersion := os.Getenv("IMGPROXY_ENV_GCP_SECRET_VERSION_ID") + secretProject := os.Getenv("IMGPROXY_ENV_GCP_SECRET_PROJECT_ID") + secretKey := os.Getenv("IMGPROXY_ENV_GCP_KEY") + + if len(secretID) == 0 { + return nil + } + + if len(secretVersion) == 0 { + secretVersion = "latest" + } + + var ( + client *secretmanager.Client + err error + ) + + ctx, ctxcancel := context.WithTimeout(context.Background(), time.Minute) + defer ctxcancel() + + opts := []option.ClientOption{} + + if len(secretKey) > 0 { + opts = append(opts, option.WithCredentialsJSON([]byte(secretKey))) + } + + client, err = secretmanager.NewClient(ctx, opts...) + + if err != nil { + return fmt.Errorf("Can't create Google Cloud Secret Manager client: %s", err) + } + + req := secretmanagerpb.AccessSecretVersionRequest{ + Name: fmt.Sprintf("projects/%s/secrets/%s/versions/%s", secretProject, secretID, secretVersion), + } + + resp, err := client.AccessSecretVersion(ctx, &req) + if err != nil { + return fmt.Errorf("Can't get Google Cloud Secret Manager secret: %s", err) + } + + payload := resp.GetPayload() + if payload == nil { + return errors.New("Can't get Google Cloud Secret Manager secret: payload is empty") + } + + data := payload.GetData() + + if len(data) == 0 { + return nil + } + + envmap, err := godotenv.Unmarshal(string(data)) + if err != nil { + return fmt.Errorf("Can't parse config from Google Cloud Secrets Manager: %s", err) + } + + for k, v := range envmap { + if err = os.Setenv(k, v); err != nil { + return fmt.Errorf("Can't set %s env variable from Google Cloud Secrets Manager: %s", k, err) + } + } + + return nil +} diff --git a/config/loadenv/loadenv.go b/config/loadenv/loadenv.go new file mode 100644 index 0000000000..528da63df9 --- /dev/null +++ b/config/loadenv/loadenv.go @@ -0,0 +1,21 @@ +package loadenv + +func Load() error { + if err := loadAWSSecret(); err != nil { + return err + } + + if err := loadAWSSystemManagerParams(); err != nil { + return err + } + + if err := loadGCPSecret(); err != nil { + return err + } + + if err := loadLocalFile(); err != nil { + return err + } + + return nil +} diff --git a/config/loadenv/local_file.go b/config/loadenv/local_file.go new file mode 100644 index 0000000000..d7f561197c --- /dev/null +++ b/config/loadenv/local_file.go @@ -0,0 +1,38 @@ +package loadenv + +import ( + "fmt" + "os" + + "github.com/DarthSim/godotenv" +) + +func loadLocalFile() error { + path := os.Getenv("IMGPROXY_ENV_LOCAL_FILE_PATH") + + if len(path) == 0 { + return nil + } + + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("Can't read loacal environment file: %s", err) + } + + if len(data) == 0 { + return nil + } + + envmap, err := godotenv.Unmarshal(string(data)) + if err != nil { + return fmt.Errorf("Can't parse config from local file: %s", err) + } + + for k, v := range envmap { + if err = os.Setenv(k, v); err != nil { + return fmt.Errorf("Can't set %s env variable from local file: %s", k, err) + } + } + + return nil +} diff --git a/docs/_sidebar.md b/docs/_sidebar.md index 55ad9e67b0..8e2ba05053 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -1,7 +1,9 @@ * [Getting started](GETTING_STARTED.md) * [Pro version](https://imgproxy.net/#pro) * [Installation](installation.md) -* [Configuration](configuration.md) +* Configuration + * [Configuration](configuration.md) + * [Loading environment variables](loading_environment_variables.md) * Generating the URL * [Generating the URL](generating_the_url.md) * [Getting the image info](getting_the_image_info.md) diff --git a/docs/cloud_watch.md b/docs/cloud_watch.md index ac90ffaa6c..696e096e96 100644 --- a/docs/cloud_watch.md +++ b/docs/cloud_watch.md @@ -42,8 +42,6 @@ AWS_ACCESS_KEY_ID=my_access_key AWS_SECRET_ACCESS_KEY=my_secret_key imgproxy docker run -e AWS_ACCESS_KEY_ID=my_access_key -e AWS_SECRET_ACCESS_KEY=my_secret_key -it darthsim/imgproxy ``` -This is the recommended method when using dockerized imgproxy. - #### Shared credentials file Alternatively, you can create the `.aws/credentials` file in your home directory with the following content: diff --git a/docs/loading_environment_variables.md b/docs/loading_environment_variables.md new file mode 100644 index 0000000000..f0a208bb6e --- /dev/null +++ b/docs/loading_environment_variables.md @@ -0,0 +1,160 @@ +# Loading environment variables + +imgproxy can load environment variables from various sources such as: + +* [Local file](#local-file) +* [AWS Secrets Manager](#aws-secrets-manager) +* [AWS Systems Manager Parameter Store](#aws-systems-manager-parameter-store) +* [Google Cloud Secret Manager](#google-cloud-secret-manager) + +## Local file + +You can create an [environment file](#environment-file-syntax) and configure imgproxy to read environment variables from it. + +* `IMGPROXY_ENV_LOCAL_FILE_PATH`: the path of the environmebt file to load + +## AWS Secrets Manager + +You can store the content of an [environment file](#environment-file-syntax) as an AWS Secrets Manager secret and configure imgproxy to read environment variables from it. + +* `IMGPROXY_ENV_AWS_SECRET_ID`: the ARN or name of the secret to load +* `IMGPROXY_ENV_AWS_SECRET_VERSION_ID`: _(optional)_ the unique identifier of the version of the secret to load +* `IMGPROXY_ENV_AWS_SECRET_VERSION_STAGE`: _(optional)_ the staging label of the version of the secret to load +* `IMGPROXY_ENV_AWS_SECRET_REGION`: _(optional)_ the region of the secret to load + +**📝 Note:** If both `IMGPROXY_ENV_AWS_SECRET_VERSION_ID` and `IMGPROXY_ENV_AWS_SECRET_VERSION_STAGE` are set, `IMGPROXY_ENV_AWS_SECRET_VERSION_STAGE` will be ignored + +### Set up AWS Secrets Manager credentials + +There are three ways to specify your AWS credentials. The credentials policy should allow performing the `secretsmanager:GetSecretValue` and `secretsmanager:ListSecretVersionIds` actions with the specified secret: + +#### IAM Roles + +If you're running imgproxy on an Amazon Web Services platform, you can use IAM roles to to get the security credentials to retrieve the secret. + +* **Elastic Container Service (ECS):** Assign an [IAM role to a task](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html). +* **Elastic Kubernetes Service (EKS):** Assign a [service account to a pod](https://docs.aws.amazon.com/eks/latest/userguide/pod-configuration.html). +* **Elastic Beanstalk:** Assign an [IAM role to an instance](https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/iam-instanceprofile.html). + +#### Environment variables + +You can specify an AWS Access Key ID and a Secret Access Key by setting the standard `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables. + +``` bash +AWS_ACCESS_KEY_ID=my_access_key AWS_SECRET_ACCESS_KEY=my_secret_key imgproxy + +# same for Docker +docker run -e AWS_ACCESS_KEY_ID=my_access_key -e AWS_SECRET_ACCESS_KEY=my_secret_key -it darthsim/imgproxy +``` + +#### Shared credentials file + +Alternatively, you can create the `.aws/credentials` file in your home directory with the following content: + +```ini +[default] +aws_access_key_id = %access_key_id +aws_secret_access_key = %secret_access_key +``` + +## AWS Systems Manager Parameter Store + +You can store multiple AWS Systems Manager Parameter Store entries and configure imgproxy to load their values to separate environment variables. + +* `IMGPROXY_ENV_AWS_SSM_PARAMETERS_PATH`: the [path](#aws-systems-manager-path) of the parameters to load +* `IMGPROXY_ENV_AWS_SSM_PARAMETERS_REGION`: _(optional)_ the region of the parameters to load + +### AWS Systems Manager path + +Let's assume that you created the following AWS Systems Manager parameters: + +* `/imgproxy/prod/IMGPROXY_KEY` +* `/imgproxy/prod/IMGPROXY_SALT` +* `/imgproxy/prod/IMGPROXY_CLOUD_WATCH/SERVICE_NAME` +* `/imgproxy/prod/IMGPROXY_CLOUD_WATCH/NAMESPACE` +* `/imgproxy/staging/IMGPROXY_KEY` + +If you set `IMGPROXY_ENV_AWS_SSM_PARAMETERS_PATH` to `/imgproxy/prod`, imgproxy will load these parameters the following way: + +* `/imgproxy/prod/IMGPROXY_KEY` value will be loaded to `IMGPROXY_KEY` +* `/imgproxy/prod/IMGPROXY_SALT` value will be loaded to `IMGPROXY_SALT` +* `/imgproxy/prod/IMGPROXY_CLOUD_WATCH/SERVICE_NAME` value will be loaded to `IMGPROXY_CLOUD_WATCH_SERVICE_NAME` +* `/imgproxy/prod/IMGPROXY_CLOUD_WATCH/NAMESPACE` value will be loaded to `IMGPROXY_CLOUD_WATCH_NAMESPACE` +* `/imgproxy/staging/IMGPROXY_KEY` will be ignored since its path is not `/imgproxy/prod` + +### Set up AWS Systems Manager credentials + +There are three ways to specify your AWS credentials. The credentials policy should allow performing the `ssm:GetParametersByPath` action with the specified parameters: + +#### IAM Roles + +If you're running imgproxy on an Amazon Web Services platform, you can use IAM roles to to get the security credentials to retrieve the secret. + +* **Elastic Container Service (ECS):** Assign an [IAM role to a task](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html). +* **Elastic Kubernetes Service (EKS):** Assign a [service account to a pod](https://docs.aws.amazon.com/eks/latest/userguide/pod-configuration.html). +* **Elastic Beanstalk:** Assign an [IAM role to an instance](https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/iam-instanceprofile.html). + +#### Environment variables + +You can specify an AWS Access Key ID and a Secret Access Key by setting the standard `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables. + +``` bash +AWS_ACCESS_KEY_ID=my_access_key AWS_SECRET_ACCESS_KEY=my_secret_key imgproxy + +# same for Docker +docker run -e AWS_ACCESS_KEY_ID=my_access_key -e AWS_SECRET_ACCESS_KEY=my_secret_key -it darthsim/imgproxy +``` + +#### Shared credentials file + +Alternatively, you can create the `.aws/credentials` file in your home directory with the following content: + +```ini +[default] +aws_access_key_id = %access_key_id +aws_secret_access_key = %secret_access_key +``` + +## Google Cloud Secret Manager + +You can store the content of an [environment file](#environment-file-syntax) in Google Cloud Secret Manager secret and configure imgproxy to read environment variables from it. + +* `IMGPROXY_ENV_GCP_SECRET_ID`: the name of the secret to load +* `IMGPROXY_ENV_GCP_SECRET_VERSION_ID`: _(optional)_ the unique identifier of the version of the secret to load +* `IMGPROXY_ENV_GCP_SECRET_PROJECT_ID`: the name or ID of the Google Cloud project that contains the secret + +### Setup credentials + +If you run imgproxy inside Google Cloud infrastructure (Compute Engine, Kubernetes Engine, App Engine, Cloud Functions, etc), and you have granted access to the specified secret to the service account, you probably don't need to do anything here. imgproxy will try to use the credentials provided by Google. + +Otherwise, set `IMGPROXY_ENV_GCP_KEY` environment variable to the content of Google Cloud JSON key. Get more info about JSON keys: [https://cloud.google.com/iam/docs/creating-managing-service-account-keys](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). + +## Environment file syntax + +The following syntax rules apply to environment files: + +* Blank lines are ignored +* Lines beginning with `#` are processed as comments and ignored +* Each line represents a key-value pair. Values can optionally be quoted: + * `VAR=VAL` -> `VAL` + * `VAR="VAL"` -> `VAL` + * `VAR='VAL'` -> `VAL` +* Unquoted and double-quoted (`"`) values have variable substitution applied: + * `VAR=${OTHER_VAR}` -> value of `OTHER_VAR` + * `VAR=$OTHER_VAR` -> value of `OTHER_VAR` + * `VAR="$OTHER_VAR"` -> value of `OTHER_VAR` + * `VAR="${OTHER_VAR}"` -> value of `OTHER_VAR` +* Single-quoted (`'`) values are used literally: + * `VAR='$OTHER_VAR'` -> `$OTHER_VAR` + * `VAR='${OTHER_VAR}'` -> `${OTHER_VAR}` +* Double quotes in double-quoted (`"`) values can be escaped with `\`: + * `VAR="{\"hello\": \"json\"}"` -> `{"hello": "json"}` +* Slash (`\`) in double-quoted values can be escaped with another slash: + * `VAR="some\\value"` -> `some\value` +* A new line can be added to double-quoted values using `\n`: + * `VAR="some\nvalue"` -> + ``` + some + value + ``` + diff --git a/docs/serving_files_from_s3.md b/docs/serving_files_from_s3.md index a45c630c53..3bd6c27b17 100644 --- a/docs/serving_files_from_s3.md +++ b/docs/serving_files_from_s3.md @@ -39,8 +39,6 @@ AWS_ACCESS_KEY_ID=my_access_key AWS_SECRET_ACCESS_KEY=my_secret_key imgproxy docker run -e AWS_ACCESS_KEY_ID=my_access_key -e AWS_SECRET_ACCESS_KEY=my_secret_key -it darthsim/imgproxy ``` -This is the recommended method when using dockerized imgproxy. - #### Shared credentials file Alternatively, you can create the `.aws/credentials` file in your home directory with the following content: diff --git a/go.mod b/go.mod index 0492aab511..8af8095c47 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 + github.com/DarthSim/godotenv v1.3.1 github.com/DataDog/datadog-go/v5 v5.3.0 github.com/airbrake/gobrake/v5 v5.6.1 github.com/aws/aws-sdk-go v1.44.260 @@ -58,6 +59,7 @@ require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.0.1 // indirect cloud.google.com/go/pubsub v1.30.0 // indirect + cloud.google.com/go/secretmanager v1.10.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/DataDog/appsec-internal-go v1.0.0 // indirect diff --git a/go.sum b/go.sum index 243ec11669..3b063cdee4 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/pubsub v1.30.0 h1:vCge8m7aUKBJYOgrZp7EsNDf6QMd2CAlXZqWTn3yq6s= cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= +cloud.google.com/go/secretmanager v1.10.0 h1:pu03bha7ukxF8otyPKTFdDz+rr9sE3YauS5PliDXK60= +cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -59,6 +61,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DarthSim/gofakes3 v0.0.0-20230502153341-3fc66d2bc272 h1:Gj21neabaU3DEwwVPG6/Vn0GSuDQ1n2m2z1qV33SCI0= github.com/DarthSim/gofakes3 v0.0.0-20230502153341-3fc66d2bc272/go.mod h1:Cnosl0cRZIfKjTMuH49sQog2LeNsU5Hf4WnPIDWIDV0= +github.com/DarthSim/godotenv v1.3.1 h1:NMWdswlRx2M9uPY4Ux8p/Q/rDs7A97OG89fECiQ/Tz0= +github.com/DarthSim/godotenv v1.3.1/go.mod h1:B3ySe1HYTUFFR6+TPyHyxPWjUdh48il0Blebg9p1cCc= github.com/DarthSim/opentelemetry-go-contrib/detectors/aws/ecs v0.0.0-20230510163401-1a377505ea6c h1:6XK2HjE3YbWRAl4nNpXFwiZ8LP+JZjxihvZw5ZUgyss= github.com/DarthSim/opentelemetry-go-contrib/detectors/aws/ecs v0.0.0-20230510163401-1a377505ea6c/go.mod h1:OshtJzwB+6SKoFM4ovJIbsHuwg7PpLGIbpaAOHJwyUU= github.com/DataDog/appsec-internal-go v1.0.0 h1:2u5IkF4DBj3KVeQn5Vg2vjPUtt513zxEYglcqnd500U= diff --git a/main.go b/main.go index ce09829a40..b2451b6057 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( "go.uber.org/automaxprocs/maxprocs" "github.com/imgproxy/imgproxy/v3/config" + "github.com/imgproxy/imgproxy/v3/config/loadenv" "github.com/imgproxy/imgproxy/v3/errorreport" "github.com/imgproxy/imgproxy/v3/gliblog" "github.com/imgproxy/imgproxy/v3/imagedata" @@ -35,6 +36,10 @@ func initialize() error { maxprocs.Set(maxprocs.Logger(log.Debugf)) + if err := loadenv.Load(); err != nil { + return err + } + if err := config.Configure(); err != nil { return err } From 2832aabf42cb05fb75c13530487ad633422cd54e Mon Sep 17 00:00:00 2001 From: DarthSim Date: Wed, 16 Aug 2023 14:10:27 +0300 Subject: [PATCH 053/138] Update docs style --- docs/assets/check.svg | 1 + docs/assets/cross.svg | 1 + docs/assets/docsify-init.js | 15 +++- docs/assets/note.svg | 1 + docs/assets/pro.svg | 4 +- docs/assets/style.css | 143 +++++++++++++++++++++++++++++------- docs/assets/warning.svg | 1 + 7 files changed, 133 insertions(+), 33 deletions(-) create mode 100644 docs/assets/check.svg create mode 100644 docs/assets/cross.svg create mode 100644 docs/assets/note.svg create mode 100644 docs/assets/warning.svg diff --git a/docs/assets/check.svg b/docs/assets/check.svg new file mode 100644 index 0000000000..f3de2daafc --- /dev/null +++ b/docs/assets/check.svg @@ -0,0 +1 @@ + diff --git a/docs/assets/cross.svg b/docs/assets/cross.svg new file mode 100644 index 0000000000..668c854bbd --- /dev/null +++ b/docs/assets/cross.svg @@ -0,0 +1 @@ + diff --git a/docs/assets/docsify-init.js b/docs/assets/docsify-init.js index 573d8b599c..0240bc6d71 100644 --- a/docs/assets/docsify-init.js +++ b/docs/assets/docsify-init.js @@ -35,7 +35,7 @@ const configureDocsify = (additionalVersions, latestVersion, latestTag) => { const versionAliases = {}; - const versionSelect = ['']; versions.forEach(([version, tag]) => { const value = version == latestVersion ? "" : version; versionSelect.push(`