From 72b8a07384862319c6af9f25ac1113bffd603a02 Mon Sep 17 00:00:00 2001 From: ItsOnlyBinary Date: Fri, 27 Nov 2020 18:44:57 +0000 Subject: [PATCH 01/20] [WIP] Make fileuploader handle creating embeddable html iframes --- fileuploader.config.example.toml | 8 + go.mod | 7 +- go.sum | 236 +++++++++++++++++++++ server/config.go | 9 + server/embed.go | 339 +++++++++++++++++++++++++++++++ server/uploadserver.go | 5 + templates/embed.html | 60 ++++++ 7 files changed, 663 insertions(+), 1 deletion(-) create mode 100644 server/embed.go create mode 100644 templates/embed.html diff --git a/fileuploader.config.example.toml b/fileuploader.config.example.toml index 1ad8ce1..33ebead 100644 --- a/fileuploader.config.example.toml +++ b/fileuploader.config.example.toml @@ -45,6 +45,14 @@ MaxAge = "24h" # 1 day IdentifiedMaxAge = "168h" # 1 week CheckInterval = "5m" +[Embed] +TemplatePath = "templates/embed.html" +CacheMaxAge = "1h" +CacheCleanInterval = "15m" +ImageCachePath = "image-cache" +ImageCacheMaxSize = 1073741824 + + # If EXTJWT is supported by the gateway or network, a validated token with an account present (when # the user is authenticated to an irc services account) will use the IdentifiedMaxAge setting above # instead of the base MaxAge. diff --git a/go.mod b/go.mod index 44cfa81..19726c9 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,9 @@ require ( github.com/OneOfOne/xxhash v1.2.7 // indirect github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 // indirect github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae + github.com/davecgh/go-spew v1.1.1 github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/dyatlov/go-oembed v0.0.0-20191103150536-a57c85b3b37c github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-gonic/gin v1.4.0 github.com/go-sql-driver/mysql v1.4.1 @@ -16,12 +18,15 @@ require ( github.com/golang/protobuf v1.3.2 // indirect github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect github.com/gorilla/websocket v1.4.1 // indirect + github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc github.com/jmoiron/sqlx v1.2.0 github.com/kiwiirc/webircgateway v0.0.0-20200226172020-f8a71090407a github.com/lib/pq v1.1.1 // indirect github.com/mattn/go-isatty v0.0.8 // indirect github.com/mattn/go-sqlite3 v1.10.0 + github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 // indirect + github.com/peterbourgon/diskv v0.0.0-20171120014656-2973218375c3 github.com/rs/zerolog v1.14.3 github.com/rubenv/sql-migrate v0.0.0-20190618074426-f4d34eae5a5c github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0 // indirect @@ -32,10 +37,10 @@ require ( github.com/ziutek/mymysql v1.5.4 // indirect golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d // indirect golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect - golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect google.golang.org/appengine v1.6.1 // indirect gopkg.in/Acconut/lockfile.v1 v1.1.0 gopkg.in/gorp.v1 v1.7.2 // indirect gopkg.in/ini.v1 v1.52.0 // indirect + willnorris.com/go/imageproxy v0.10.0 ) diff --git a/go.sum b/go.sum index a2ca4bd..e5c9978 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,16 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.37.1/go.mod h1:SAbnLi6YTSPKSI0dTUEOVLCkyPfKXK8n4ibqiMoj4ok= +contrib.go.opencensus.io/exporter/ocagent v0.4.9/go.mod h1:ueLzZcP7LPhPulEBukGn4aLh7Mx9YJwpVJ9nL2FYltw= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/Azure/azure-sdk-for-go v26.5.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-autorest v11.5.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo= +github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.4 h1:HZ+j9jn/+mcsaDSQRZuK00pXWdE25AQLtgm8kZct1Ew= github.com/OneOfOne/xxhash v1.2.4/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -7,16 +18,36 @@ github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/OneOfOne/xxhash v1.2.7 h1:fzrmmkskv067ZQbd9wERNGuxckWw67dyzoMG62p7LMo= github.com/OneOfOne/xxhash v1.2.7/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/PaulARoy/azurestoragecache v0.0.0-20170906084534-3c249a3ba788/go.mod h1:lY1dZd8HBzJ10eqKERHn3CU59tfhzcAVb2c0ZhIWSOk= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aws/aws-sdk-go v1.19.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 h1:y4B3+GPxKlrigF1ha5FFErxK+sr6sWxQovRMzwMhejo= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= +github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae h1:2Zmk+8cNvAGuY8AyvZuWpUdpQUAXwfom4ReVMe/CTIo= github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= +github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -24,17 +55,35 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/die-net/lrucache v0.0.0-20181227122439-19a39ef22a11/go.mod h1:ew0MSjCVDdtGMjF3kzLK9hwdgF5mOE8SbYVF3Rc7mkU= +github.com/disintegration/imaging v1.6.0 h1:nVPXRUUQ36Z7MNf0O77UzgnOb1mkMMor7lmJMJXc/mA= +github.com/disintegration/imaging v1.6.0/go.mod h1:xuIt+sRxDFrHS0drzXUlCJthkJ8k7lkkUojDSR247MQ= +github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/dyatlov/go-oembed v0.0.0-20191103150536-a57c85b3b37c h1:MEV1LrQtCBGacXajlT4CSuYWbZuLl/qaZVqwoOmwAbU= +github.com/dyatlov/go-oembed v0.0.0-20191103150536-a57c85b3b37c/go.mod h1:DjlDZiZGRRKbiJZmiEiiXozsBQAQzHmxwHKFeXifL2g= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3 h1:t8FVkw33L+wilf2QiWkw0UV77qRpcH/JHPKGpKa2E8g= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.4.0 h1:3tMoCCfM7ppqsR0ptz/wi1impNpT7/9wQtMZ8lr1mCQ= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/logger v1.0.0/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs= @@ -45,53 +94,96 @@ github.com/gobuffalo/packr v1.30.1/go.mod h1:ljMyFO2EcrnzsHsN99cvbq055Y9OhRrIavi github.com/gobuffalo/packr/v2 v2.5.1/go.mod h1:8f9c96ITobJlPzI44jj+4tHnEKNt0xXWSVlXRN9X1Iw= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc h1:f8eY6cV/x1x+HLjOp4r72s/31/V2aTUtg5oKRRPf8/Q= +github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/igm/sockjs-go v0.0.0-20181115114233-fd48fe90e436 h1:02X+bCfYLK5AdKlM6RYZ+LlI4MTIBwOf+hRENcv9Mpk= github.com/igm/sockjs-go v0.0.0-20181115114233-fd48fe90e436/go.mod h1:Yu6pvqjNniWNJe07LPObeCG6R77Qc97C6Kss0roF8tU= github.com/igm/sockjs-go v0.0.0-20191119074118-cd6986df5bcc h1:BKaqvDSmr77q6jHXHxdpsyZSSNntGktvWwvP+pg+cdg= github.com/igm/sockjs-go v0.0.0-20191119074118-cd6986df5bcc/go.mod h1:Yu6pvqjNniWNJe07LPObeCG6R77Qc97C6Kss0roF8tU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jamiealquiza/envy v1.1.0/go.mod h1:MP36BriGCLwEHhi1OU8E9569JNZrjWfCvzG7RsPnHus= +github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karrick/godirwalk v1.10.12/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kiwiirc/webircgateway v0.0.0-20190709195406-2c86038cda4f h1:vlP3syZmsRUeGOBk9eaC5cZTzC5w3d/m/A/b+fmGZZ4= github.com/kiwiirc/webircgateway v0.0.0-20190709195406-2c86038cda4f/go.mod h1:WoHdFzCnR24cCEdcImTt141bKmhYKs60eUH4Woav2Ls= github.com/kiwiirc/webircgateway v0.0.0-20200226172020-f8a71090407a h1:947kcsvRCG/vqoiCN9B7WgXoCfoMVMZQ70PBmoQUQO4= github.com/kiwiirc/webircgateway v0.0.0-20200226172020-f8a71090407a/go.mod h1:3OveolwWkB00OKF/05GgyHaPmwe0coJ5RIDk44WAZhw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/marstr/guid v0.0.0-20170427235115-8bdf7d1a087c/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= @@ -99,12 +191,27 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/muesli/smartcrop v0.2.1-0.20181030220600-548bbf0c0965 h1:BdOnvj+P+06ZFwYd07iFWXHPfRyrJd5sAXpX9+E8bxM= +github.com/muesli/smartcrop v0.2.1-0.20181030220600-548bbf0c0965/go.mod h1:i2fCI/UorTfgEpPPLWiFBv4pye+YAG78RwcQLUkocpI= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= +github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/orcaman/concurrent-map v0.0.0-20190107190726-7ed82d9cb717 h1:2v7IYkog9ZFN04bv5hkwjpyHkc6wujPPOVYDPp2rfwA= github.com/orcaman/concurrent-map v0.0.0-20190107190726-7ed82d9cb717/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/orcaman/concurrent-map v0.0.0-20190314100340-2693aad1ed75 h1:IV56VwUb9Ludyr7s53CMuEh4DdTnnQtEPLEgLyJ0kHI= @@ -112,9 +219,38 @@ github.com/orcaman/concurrent-map v0.0.0-20190314100340-2693aad1ed75/go.mod h1:L github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 h1:lNCW6THrCKBiJBpz8kbVGjC7MgdCGKwuvBgc7LoD6sw= github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v0.0.0-20171120014656-2973218375c3 h1:ZKRE3mqKoxObHs5oWjLnA1WxXhmlDDAVuE0VsuLIoNk= +github.com/peterbourgon/diskv v0.0.0-20171120014656-2973218375c3/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/peterbourgon/diskv v1.0.0 h1:bRU92KzrX3TQ6IYobfie/PnZkFC+1opBfHpf/PHPDoo= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= +github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -124,8 +260,12 @@ github.com/rs/zerolog v1.14.3/go.mod h1:3WXPzbXEEliJ+a6UFE4vhIxV8qR1EML6ngzP9ug4 github.com/rubenv/sql-migrate v0.0.0-20190618074426-f4d34eae5a5c h1:LCELEbde3/GT141OpHRs+jJZrI1tI3ayVd4VqW7Ui2U= github.com/rubenv/sql-migrate v0.0.0-20190618074426-f4d34eae5a5c/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= +github.com/satori/go.uuid v0.0.0-20180103174451-36e9d2ebbde5/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0 h1:X9XMOYjxEfAYSy3xK1DzO5dMkkWhs9E9UCcS1IERx2k= github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0/go.mod h1:Ad7IjTpvzZO8Fl0vh9AzQ+j/jYZfyp2diGwI8m5q+ns= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -150,6 +290,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tus/tusd v0.0.0-20190712143443-30811b6579c5 h1:YBHQiRr7TH20CDlIcoH2Fzqqm1C8asoj0NHlOK5Pm4I= github.com/tus/tusd v0.0.0-20190712143443-30811b6579c5/go.mod h1:BBkwF03jAYYdT5yGkoojt46c3NLjziO9OYu0zR4ptWg= github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= @@ -164,6 +306,13 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= +go.opencensus.io v0.19.1/go.mod h1:gug0GbSHa8Pafr0d2urOSgoXHZ6x/RUlaiT0d9pqb4A= +go.opencensus.io v0.19.2/go.mod h1:NO/8qkisMZLZ1FCsKNqtJPwc8/TaclWyY0B6wcYNg9M= +go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/build v0.0.0-20190314133821-5284462c4bec/go.mod h1:atTaCNAy0f16Ah5aV1gMSwgiKVHwu/JncqDpuRr7lS4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= @@ -174,18 +323,56 @@ golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200208060501-ecb85df21340/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190321063152-3fc05d484e9f h1:FO4MZ3N56GnxbqxGKqh+YTzUWQ2sDwtFQEZgLOxh9Jc= +golang.org/x/image v0.0.0-20190321063152-3fc05d484e9f/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -196,40 +383,89 @@ golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624180213-70d37148ca0c/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.2.0/go.mod h1:IfRCZScioGtypHNTlz3gFk67J8uePVW7uDTBzXuIkhU= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190321212433-e79c0c59cdb5/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= gopkg.in/Acconut/lockfile.v1 v1.1.0 h1:c5AMZOxgM1y+Zl8eSbaCENzVYp/LCaWosbQSXzb3FVI= gopkg.in/Acconut/lockfile.v1 v1.1.0/go.mod h1:6UCz3wJ8tSFUsPR6uP/j8uegEtDuEEqFxlpi0JI4Umw= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/gorp.v1 v1.7.2 h1:j3DWlAyGVv8whO7AcIWznQ2Yj7yJkn34B8s63GViAAw= gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.44.0 h1:YRJzTUp0kSYWUVFF5XAbDFfyiqwsl0Vb9R8TVP5eRi0= gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.52.0 h1:j+Lt/M1oPPejkniCg1TkWE2J3Eh1oZTsHSXzMTzUXn4= gopkg.in/ini.v1 v1.52.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +willnorris.com/go/gifresize v1.0.0 h1:GKS68zjNhHMqkgNTv4iFAO/j/sNcVSOHQ7SqmDAIAmM= +willnorris.com/go/gifresize v1.0.0/go.mod h1:eBM8gogBGCcaH603vxSpnfjwXIpq6nmnj/jauBDKtAk= +willnorris.com/go/imageproxy v0.10.0 h1:onR4Q88jfC+uYKaFaDKJH57xY6UDl9GJ/x7La6MmBsk= +willnorris.com/go/imageproxy v0.10.0/go.mod h1:2tWdKRneln3E9X/zwH1RINpQAQWPeUiNynZ7UQ9OROk= diff --git a/server/config.go b/server/config.go index 5dafb66..8c57014 100644 --- a/server/config.go +++ b/server/config.go @@ -45,6 +45,15 @@ type Config struct { } JwtSecretsByIssuer map[string]string Loggers []LoggerConfig + + // Embed Provider + Embed struct { + TemplatePath string + CacheMaxAge duration + CacheCleanInterval duration + ImageCachePath string + ImageCacheMaxSize uint64 + } } func NewConfig() *Config { diff --git a/server/embed.go b/server/embed.go new file mode 100644 index 0000000..699209d --- /dev/null +++ b/server/embed.go @@ -0,0 +1,339 @@ +package server + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "log" + "net/http" + "net/url" + "os" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/dyatlov/go-oembed/oembed" + "github.com/gin-gonic/gin" + "github.com/gregjones/httpcache/diskcache" + "github.com/peterbourgon/diskv" + "willnorris.com/go/imageproxy" +) + +type cacheItem struct { + url string + html string + created int64 + wg sync.WaitGroup +} + +// HTML template +var template string +var templateLock sync.RWMutex + +// In memory HTML cache +var cache = make(map[string]*cacheItem) +var cacheLock sync.Mutex +var cacheTicker *time.Ticker + +var embed *oembed.Oembed + +var imgProxy *imageproxy.Proxy + +// Used to detect possible image urls after after failed oembed match +var isImage = regexp.MustCompile(`\.(jpe?g|png|gifv?)$`) + +func (serv *UploadServer) registerEmbedHandlers(r *gin.Engine, cfg Config) error { + data, err := getProviders(false) + if err != nil { + return err + } + embed = oembed.NewOembed() + embed.ParseProviders(bytes.NewReader(*data)) + + // Start the cleanup ticker + startCleanupTicker( + cfg.Embed.CacheCleanInterval.Duration, + cfg.Embed.CacheMaxAge.Duration, + ) + + if err := loadTemplate(cfg.Embed.TemplatePath); err != nil { + log.Println("Failed to load template: " + err.Error()) + return nil + } + + rg := r.Group("/embed") + rg.GET("", handleEmbed) + + // Attach imageproxy + cache := diskCache(cfg.Embed.ImageCachePath, cfg.Embed.ImageCacheMaxSize) + imgProxy = imageproxy.NewProxy(nil, cache) + + ic := r.Group("/image-cache/*id") + ic.GET("", handleImageCache) + return nil +} + +func handleImageCache(c *gin.Context) { + r := c.Request + r.URL.Path = strings.Replace(r.URL.Path, "/image-cache", "", -1) + spew.Dump(r.URL.Path) + imgProxy.ServeHTTP(c.Writer, c.Request) +} + +func diskCache(path string, maxSize uint64) *diskcache.Cache { + d := diskv.New(diskv.Options{ + BasePath: path, + CacheSizeMax: maxSize, + // For file "c0ffee", store file as "c0/ff/c0ffee" + Transform: func(s string) []string { return []string{s[0:2], s[2:4]} }, + }) + return diskcache.NewWithDiskv(d) +} + +func handleEmbed(c *gin.Context) { + queryURL := c.Query("url") + if !isValidURL(queryURL) { + c.AbortWithStatus(http.StatusBadRequest) + return + } + + queryCenter := c.Query("center") + queryWidth := c.Query("width") + queryHeight := c.Query("height") + + // Convert queryCenter to boolean + center, err := strconv.ParseBool(queryCenter) + if err != nil { + center = false + } + + width, err := strconv.Atoi(queryWidth) + if err != nil { + width = 1000 + } + + height, err := strconv.Atoi(queryHeight) + if err != nil { + height = 400 + } + + hash := getHash(queryURL) + + spew.Dump(queryURL, center, height, width) + + cacheLock.Lock() + item, ok := cache[hash] + if !ok { + // Cache miss create new cache item + item = &cacheItem{ + url: queryURL, + html: "", + created: time.Now().Unix(), + } + + // Add to waitgroup so other clients can wait for the oEmbed result + item.wg.Add(1) + cache[hash] = item + + // Item added to cache, unlock so other requests can see the new item + cacheLock.Unlock() + + // Attempt to fetch oEmbed data + embedItem := embed.FindItem(queryURL) + if embedItem != nil { + options := oembed.Options{ + URL: queryURL, + MaxHeight: height, + MaxWidth: width, + } + info, err := embedItem.FetchOembed(options) + if err != nil { + // An unexpected error occurred + fmt.Printf("An error occured: %s\n", err.Error()) + } else if info.Status >= 300 { + // oEmbed returned an error status + fmt.Printf("Response status code is: %d\n", info.Status) + } else if info.HTML != "" { + // oEmbed returned embedable html + fmt.Printf("Oembed info:\n%s\n", info) + item.html = info.HTML + } else if info.Type == "photo" { + // oEmbed returned a photo type the url should be an image + fmt.Println("type photo " + info.URL) + item.html = getImageHTML(c, info.URL, height) + } else { + spew.Dump(info) + } + } + + // oEmbed did not return any html, maybe the url is direct to an image + if item.html == "" && isImage.MatchString(queryURL) { + item.html = getImageHTML(c, queryURL, height) + } + + // Still no html send an error to the parent + if item.html == "" { + item.html = "" + } + + // Decrease the waitgroup so other requests can complete + item.wg.Done() + } else { + log.Printf("Cache HIT") + // Cache hit unlock the cache + cacheLock.Unlock() + } + + // Wait until the cache item is fulfilled + item.wg.Wait() + + // Prepare html and send it to the client + style := "display: flex; justify-content: center;" + if !center { + style = "" + } + templateLock.RLock() + htmlData := strings.Replace(template, "{{body.html}}", item.html, -1) + templateLock.RUnlock() + htmlData = strings.Replace(htmlData, "/* style.body */", style, -1) + c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(htmlData)) +} + +func getProviders(force bool) (*[]byte, error) { + log.Println("Getting oEmbed Providers") + if _, err := os.Stat("providers.json"); force || os.IsNotExist(err) { + resp, err := http.Get("https://oembed.com/providers.json") + if err != nil { + return nil, errors.New("Failed to fetch oEmbed providers: " + err.Error()) + } + defer resp.Body.Close() + + log.Println("Fetched oEmbed Providers") + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.New("Failed to read oEmbed providers: " + err.Error()) + } + + log.Println("Read oEmbed Providers") + + // Unmarshal to temp interface to ensure valid json + var temp interface{} + err = json.Unmarshal(data, &temp) + if err != nil { + return nil, errors.New("Failed to parse oEmbed providers: " + err.Error()) + } + + log.Println("Tested oEmbed Providers") + + // Data appears to be valid json open providers file for writing + file, err := os.Create("providers.json") + if err != nil { + return nil, errors.New("Failed to create oEmbed providers: " + err.Error()) + } + defer file.Close() + + log.Println("Created oEmbed Providers") + + // Write providers.json + _, err = file.Write(data) + if err != nil { + return nil, errors.New("Failed to write oEmbed providers: " + err.Error()) + } + + log.Println("Written oEmbed Providers") + + return &data, nil + } + + // Open existing providers.json + file, err := os.Open("providers.json") + if err != nil { + return nil, errors.New("Failed to open oEmbed providers: " + err.Error()) + } + defer file.Close() + + // Read providers.json data + data, err := ioutil.ReadAll(file) + if err != nil { + return nil, errors.New("Failed to read oEmbed providers: " + err.Error()) + } + + return &data, nil +} + +func startCleanupTicker(cleanInterval, cacheMaxAge time.Duration) { + cacheTicker = time.NewTicker(cleanInterval) + go func() { + for range cacheTicker.C { + cleanCache(cacheMaxAge) + } + }() +} + +func cleanCache(cacheMaxAge time.Duration) { + createdBefore := time.Now().Unix() - int64(cacheMaxAge.Seconds()) + var expired []string + for hash, item := range cache { + if item.created >= createdBefore { + continue + } + expired = append(expired, hash) + } + + if len(expired) == 0 { + // Nothing to clean + log.Println("No cache items to clean") + return + } + + cacheLock.Lock() + for _, hash := range expired { + if hash == "" { + break + } + log.Println("Deleting cache item: " + hash) + delete(cache, hash) + } + cacheLock.Unlock() +} + +func loadTemplate(templatePath string) error { + html, err := ioutil.ReadFile(templatePath) + if err != nil { + return err + } + templateLock.Lock() + template = string(html) + templateLock.Unlock() + return nil +} + +func getImageHTML(c *gin.Context, url string, height int) string { + newURL := "http://" + if c.Request.TLS != nil { + newURL = "https://" + } + newURL += c.Request.Host + // fixed height proportional width + newURL += "/image-cache/x" + strconv.Itoa(height) + "/" + url + return "" +} + +func getHash(url string) string { + hasher := sha256.New() + hasher.Write([]byte(url)) + return hex.EncodeToString(hasher.Sum(nil)) +} + +func isValidURL(str string) bool { + u, err := url.Parse(str) + return err == nil && u.Host != "" +} diff --git a/server/uploadserver.go b/server/uploadserver.go index ca28a43..d861b13 100644 --- a/server/uploadserver.go +++ b/server/uploadserver.go @@ -76,6 +76,11 @@ func (serv *UploadServer) Run(replaceableHandler *ReplaceableHandler) error { return err } + err = serv.registerEmbedHandlers(serv.Router, serv.cfg) + if err != nil { + return err + } + // closed channel indicates that startup is complete close(serv.GetStartedChan()) diff --git a/templates/embed.html b/templates/embed.html new file mode 100644 index 0000000..12c5bcb --- /dev/null +++ b/templates/embed.html @@ -0,0 +1,60 @@ + + + + + + + +
+ {{body.html}} +
+ + From 83ac2e41e0f4f6511e7c06512b1667c6356a9c50 Mon Sep 17 00:00:00 2001 From: ItsOnlyBinary Date: Tue, 8 Dec 2020 19:57:47 +0000 Subject: [PATCH 02/20] Use shardedfilestore for image-proxy --- server/embed.go | 24 ++++---- server/image-proxy-cache.go | 111 ++++++++++++++++++++++++++++++++++++ server/uploadserver.go | 4 +- 3 files changed, 126 insertions(+), 13 deletions(-) create mode 100644 server/image-proxy-cache.go diff --git a/server/embed.go b/server/embed.go index 699209d..955ff60 100644 --- a/server/embed.go +++ b/server/embed.go @@ -21,8 +21,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/dyatlov/go-oembed/oembed" "github.com/gin-gonic/gin" - "github.com/gregjones/httpcache/diskcache" - "github.com/peterbourgon/diskv" "willnorris.com/go/imageproxy" ) @@ -72,7 +70,7 @@ func (serv *UploadServer) registerEmbedHandlers(r *gin.Engine, cfg Config) error rg.GET("", handleEmbed) // Attach imageproxy - cache := diskCache(cfg.Embed.ImageCachePath, cfg.Embed.ImageCacheMaxSize) + cache := serv.diskCache() imgProxy = imageproxy.NewProxy(nil, cache) ic := r.Group("/image-cache/*id") @@ -83,18 +81,20 @@ func (serv *UploadServer) registerEmbedHandlers(r *gin.Engine, cfg Config) error func handleImageCache(c *gin.Context) { r := c.Request r.URL.Path = strings.Replace(r.URL.Path, "/image-cache", "", -1) - spew.Dump(r.URL.Path) imgProxy.ServeHTTP(c.Writer, c.Request) + fmt.Println("------------------------------------------------------") } -func diskCache(path string, maxSize uint64) *diskcache.Cache { - d := diskv.New(diskv.Options{ - BasePath: path, - CacheSizeMax: maxSize, - // For file "c0ffee", store file as "c0/ff/c0ffee" - Transform: func(s string) []string { return []string{s[0:2], s[2:4]} }, - }) - return diskcache.NewWithDiskv(d) +func (serv *UploadServer) diskCache() *ImageProxyCache { + // d := diskv.New(diskv.Options{ + // BasePath: path, + // CacheSizeMax: maxSize, + // // For file "c0ffee", store file as "c0/ff/c0ffee" + // Transform: func(s string) []string { return []string{s[0:2], s[2:4]} }, + // }) + // return diskcache.NewWithDiskv(d) + d := NewImageProxyCache(serv.store, serv.log) + return d } func handleEmbed(c *gin.Context) { diff --git a/server/image-proxy-cache.go b/server/image-proxy-cache.go new file mode 100644 index 0000000..ccd8110 --- /dev/null +++ b/server/image-proxy-cache.go @@ -0,0 +1,111 @@ +package server + +import ( + "bytes" + "fmt" + "sync" + + "github.com/davecgh/go-spew/spew" + "github.com/kiwiirc/plugin-fileuploader/shardedfilestore" + "github.com/rs/zerolog" + "github.com/tus/tusd" +) + +// ImageProxyCache is an implementation of httpcache.Cache that supplements the in-memory map with persistent storage +type ImageProxyCache struct { + store *shardedfilestore.ShardedFileStore + log *zerolog.Logger + urlMap sync.Map +} + +// Get returns the response corresponding to key if present +func (c *ImageProxyCache) Get(key string) (resp []byte, ok bool) { + urlHash := getHash(key) + spew.Dump("Get", key, urlHash) + + idInterface, ok := c.urlMap.Load(urlHash) + if !ok { + fmt.Println("Not in map") + fmt.Println("") + return []byte{}, false + } + id := idInterface.(string) + + reader, err := c.store.GetReader(id) + if err != nil { + fmt.Println("No reader") + fmt.Println("") + c.urlMap.Delete(urlHash) + return []byte{}, false + } + + buffer := new(bytes.Buffer) + _, err = buffer.ReadFrom(reader) + if err != nil { + fmt.Println("Failed read") + fmt.Println("") + c.urlMap.Delete(urlHash) + return []byte{}, false + } + + bytes := buffer.Bytes() + + spew.Dump(len(bytes)) + fmt.Println("Got from cache") + fmt.Println("") + return bytes, true +} + +// Set saves a response to the cache as key +func (c *ImageProxyCache) Set(key string, resp []byte) { + urlHash := getHash(key) + spew.Dump("Set", key, len(resp), urlHash) + + metaData := tusd.MetaData{ + "Url": key, + } + fileInfo := tusd.FileInfo{ + Size: int64(len(resp)), + SizeIsDeferred: false, + MetaData: metaData, + IsFinal: false, + } + + id, err := c.store.NewUpload(fileInfo) + if err != nil { + c.log.Error(). + Err(err). + Msg("Failed to create new upload") + return + } + + _, err = c.store.WriteChunk(id, 0, bytes.NewReader(resp)) + if err != nil { + c.log.Error(). + Err(err). + Msg("Failed to write chunk") + return + } + + c.store.FinishUpload(id) + c.urlMap.Store(urlHash, id) + + fmt.Println("Set in cache") + fmt.Println("") +} + +// Delete removes the response with key from the cache +func (c *ImageProxyCache) Delete(key string) { + urlHash := getHash(key) + c.urlMap.Delete(urlHash) + spew.Dump("Delete", key) + fmt.Println("") +} + +// NewImageProxyCache returns a new Cache that will store files in basePath +func NewImageProxyCache(store *shardedfilestore.ShardedFileStore, log *zerolog.Logger) *ImageProxyCache { + return &ImageProxyCache{ + store: store, + log: log, + } +} diff --git a/server/uploadserver.go b/server/uploadserver.go index d861b13..f944f07 100644 --- a/server/uploadserver.go +++ b/server/uploadserver.go @@ -1,6 +1,7 @@ package server import ( + "context" "net/http" "sync" @@ -18,6 +19,7 @@ import ( type UploadServer struct { DBConn *db.DatabaseConnection Router *gin.Engine + ctx *RunContext cfg Config log *zerolog.Logger @@ -109,7 +111,7 @@ func (serv *UploadServer) Shutdown() { // wait for all requests to finish if serv.httpServer != nil { - serv.httpServer.Shutdown(nil) + serv.httpServer.Shutdown(context.TODO()) } // stop running FileStore GC cycles From 1eb851c30cf9cc730bf2dc3700660c8b701767d4 Mon Sep 17 00:00:00 2001 From: ItsOnlyBinary Date: Thu, 10 Dec 2020 15:37:01 +0000 Subject: [PATCH 03/20] Fixes and cleanup --- fileuploader.config.example.toml | 3 - go.mod | 1 - go.sum | 1 - noembed/noembed.go | 118 +++++++++++ server/embed.go | 331 ++++++++++++++++++++++--------- server/image-proxy-cache.go | 28 +-- 6 files changed, 360 insertions(+), 122 deletions(-) create mode 100644 noembed/noembed.go diff --git a/fileuploader.config.example.toml b/fileuploader.config.example.toml index 33ebead..d7fc244 100644 --- a/fileuploader.config.example.toml +++ b/fileuploader.config.example.toml @@ -49,9 +49,6 @@ CheckInterval = "5m" TemplatePath = "templates/embed.html" CacheMaxAge = "1h" CacheCleanInterval = "15m" -ImageCachePath = "image-cache" -ImageCacheMaxSize = 1073741824 - # If EXTJWT is supported by the gateway or network, a validated token with an account present (when # the user is authenticated to an irc services account) will use the IdentifiedMaxAge setting above diff --git a/go.mod b/go.mod index 19726c9..12afd2d 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/OneOfOne/xxhash v1.2.7 // indirect github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 // indirect github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae - github.com/davecgh/go-spew v1.1.1 github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dyatlov/go-oembed v0.0.0-20191103150536-a57c85b3b37c github.com/gin-contrib/sse v0.1.0 // indirect diff --git a/go.sum b/go.sum index e5c9978..74477db 100644 --- a/go.sum +++ b/go.sum @@ -51,7 +51,6 @@ github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= diff --git a/noembed/noembed.go b/noembed/noembed.go new file mode 100644 index 0000000..20e5d97 --- /dev/null +++ b/noembed/noembed.go @@ -0,0 +1,118 @@ +package noembed + +import ( + "encoding/json" + "errors" + "io" + "io/ioutil" + "net/http" + "regexp" + "strings" +) + +var noembedURL = "https://noembed.com/embed?url={url}" + +// NoEmbed represents this package +type NoEmbed struct { + data *Data +} + +// Data represents the data for noembed providers +type Data []struct { + Name string `json:"name"` + Patterns []Regex `json:"patterns"` +} + +// Response represents the data returned by noembed server +type Response struct { + AuthorName string `json:"author_name"` + AuthorURL string `json:"author_url"` + ProviderName string `json:"provider_name"` + ProviderURL string `json:"provider_url"` + Title string `json:"title"` + Type string `json:"type"` + URL string `json:"url"` + HTML string `json:"html"` + Version string `json:"version"` + ThumbnailURL string `json:"thumbnail_url"` + ThumbnailWidth int `json:"thumbnail_width,string"` + ThumbnailHeight int `json:"thumbnail_height,string"` + Width int `json:"width,string"` + Height int `json:"height,string"` +} + +// New returns a Noembed object +func New() *NoEmbed { + return &NoEmbed{} +} + +// ParseProviders parses the raw json obtained from noembed.com +func (n *NoEmbed) ParseProviders(buf io.Reader) error { + data, err := ioutil.ReadAll(buf) + if err != nil { + return err + } + + var noembedData Data + err = json.Unmarshal(data, &noembedData) + if err != nil { + return err + } + + n.data = &noembedData + return nil +} + +// Get returns a noembed response object +func (n *NoEmbed) Get(url string) (resp *Response, err error) { + if !n.ValidURL(url) { + err = errors.New("Unsupported URL") + return + } + + reqURL := strings.Replace(noembedURL, "{url}", url, 1) + + var httpResp *http.Response + httpResp, err = http.Get(reqURL) + if err != nil { + return + } + defer httpResp.Body.Close() + + var body []byte + body, err = ioutil.ReadAll(httpResp.Body) + if err != nil { + return + } + + err = json.Unmarshal(body, &resp) + if err != nil { + return + } + + return +} + +// ValidURL is used to test if a url is supported by noembed +func (n *NoEmbed) ValidURL(url string) bool { + for _, entry := range *n.data { + for _, pattern := range entry.Patterns { + if pattern.Regexp.MatchString(url) { + return true + } + } + } + return false +} + +// Regex Unmarshaler +type Regex struct { + regexp.Regexp +} + +// UnmarshalText used to unmarshal regexp's from text +func (r *Regex) UnmarshalText(text []byte) error { + reg, err := regexp.Compile(string(text)) + r.Regexp = *reg + return err +} diff --git a/server/embed.go b/server/embed.go index 955ff60..2c24b70 100644 --- a/server/embed.go +++ b/server/embed.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "encoding/json" "errors" - "fmt" "io/ioutil" "log" "net/http" @@ -18,9 +17,9 @@ import ( "sync" "time" - "github.com/davecgh/go-spew/spew" "github.com/dyatlov/go-oembed/oembed" "github.com/gin-gonic/gin" + "github.com/kiwiirc/plugin-fileuploader/noembed" "willnorris.com/go/imageproxy" ) @@ -31,73 +30,167 @@ type cacheItem struct { wg sync.WaitGroup } +type imgWaiterItem struct { + url string + status int + created int64 + wg sync.WaitGroup +} + // HTML template var template string var templateLock sync.RWMutex // In memory HTML cache var cache = make(map[string]*cacheItem) -var cacheLock sync.Mutex +var cacheMutex sync.Mutex var cacheTicker *time.Ticker -var embed *oembed.Oembed +// Image waiter +var imgWaiter = make(map[string]*imgWaiterItem) +var imgWaiterMutex sync.Mutex +var oEmbed *oembed.Oembed +var noEmbed *noembed.NoEmbed var imgProxy *imageproxy.Proxy -// Used to detect possible image urls after after failed oembed match +// Used to detect possible image urls var isImage = regexp.MustCompile(`\.(jpe?g|png|gifv?)$`) func (serv *UploadServer) registerEmbedHandlers(r *gin.Engine, cfg Config) error { - data, err := getProviders(false) + serv.log.Info(). + Msg("Starting embed handlers") + + // Prepare oEmbed provider + oembedJSON, err := getProvidersCached("https://oembed.com/providers.json", "oembed-providers.json", false) + if err != nil { + serv.log.Error(). + Err(err). + Msg("Failed to get oembed providers json") + return err + } + oEmbed = oembed.NewOembed() + err = oEmbed.ParseProviders(bytes.NewReader(*oembedJSON)) if err != nil { + serv.log.Error(). + Err(err). + Msg("Failed to parse oembed providers json") return err } - embed = oembed.NewOembed() - embed.ParseProviders(bytes.NewReader(*data)) + + // Prepare noEmbed provider + noembedJSON, err := getProvidersCached("https://noembed.com/providers", "noembed-providers.json", false) + if err != nil { + serv.log.Error(). + Err(err). + Msg("Failed to get noembed providers json") + return err + } + noEmbed = noembed.New() + err = noEmbed.ParseProviders(bytes.NewReader(*noembedJSON)) + if err != nil { + serv.log.Error(). + Err(err). + Msg("Failed to parse noembed providers json") + return err + } + + // Check config defaults + cacheCleanInterval := cfg.Embed.CacheCleanInterval.Duration + if cacheCleanInterval == time.Duration(0) { + cacheCleanInterval, _ = time.ParseDuration("15m") + } + + cacheMaxAge := cfg.Embed.CacheMaxAge.Duration + if cacheMaxAge == time.Duration(0) { + cacheMaxAge, _ = time.ParseDuration("1h") + } + + templatePath := cfg.Embed.TemplatePath + if templatePath == "" { + templatePath = "templates/embed.html" + } // Start the cleanup ticker - startCleanupTicker( - cfg.Embed.CacheCleanInterval.Duration, - cfg.Embed.CacheMaxAge.Duration, + serv.startCleanupTicker( + cacheCleanInterval, + cacheMaxAge, ) - if err := loadTemplate(cfg.Embed.TemplatePath); err != nil { - log.Println("Failed to load template: " + err.Error()) - return nil + // Load embed html template + if err := loadTemplate(templatePath); err != nil { + serv.log.Error(). + Err(err). + Msg("Failed to load template") + return err } + // Register our handler rg := r.Group("/embed") - rg.GET("", handleEmbed) + rg.GET("", serv.handleEmbed) - // Attach imageproxy - cache := serv.diskCache() - imgProxy = imageproxy.NewProxy(nil, cache) + // Create imageproxy and provide interface to shardedfilestore + imgCache := NewImageProxyCache(serv.store, serv.log) + imgProxy = imageproxy.NewProxy(nil, imgCache) + // Attach imageproxy ic := r.Group("/image-cache/*id") - ic.GET("", handleImageCache) + ic.GET("", serv.handleImageCache) + return nil } -func handleImageCache(c *gin.Context) { +func (serv *UploadServer) handleImageCache(c *gin.Context) { r := c.Request r.URL.Path = strings.Replace(r.URL.Path, "/image-cache", "", -1) - imgProxy.ServeHTTP(c.Writer, c.Request) - fmt.Println("------------------------------------------------------") -} -func (serv *UploadServer) diskCache() *ImageProxyCache { - // d := diskv.New(diskv.Options{ - // BasePath: path, - // CacheSizeMax: maxSize, - // // For file "c0ffee", store file as "c0/ff/c0ffee" - // Transform: func(s string) []string { return []string{s[0:2], s[2:4]} }, - // }) - // return diskcache.NewWithDiskv(d) - d := NewImageProxyCache(serv.store, serv.log) - return d + hash := getHash(r.URL.Path) + + serv.log.Debug(). + Msgf("Image request\n\turl: %s\n\thash: %s", r.URL.Path, hash) + + imgWaiterMutex.Lock() + item, ok := imgWaiter[hash] + if !ok { + // This is the first client to request this url + // create a waiter item and add it to the map + item = &imgWaiterItem{ + url: r.URL.Path, + created: time.Now().Unix(), + } + // Other requests will wait on this waitgroup once the mutex is unlocked + item.wg.Add(1) + imgWaiter[hash] = item + + // Other requests are currently waiting for this mutex + imgWaiterMutex.Unlock() + + // Pass this request to the image proxy + imgProxy.ServeHTTP(c.Writer, c.Request) + + // Image proxy is done, store resulting status + item.status = c.Writer.Status() + + // Ready for other clients to access this url + item.wg.Done() + } else { + // Not the first client to request this url + // We no longer need the mutex as we will use the waitgroup + imgWaiterMutex.Unlock() + item.wg.Wait() + + // Waitgroup is complete check if the first request was successful + if item.status == 200 { + // The first request was successful pass this request to the image proxy + imgProxy.ServeHTTP(c.Writer, c.Request) + } else { + // First request failed return its status code to the client + c.Status(item.status) + } + } } -func handleEmbed(c *gin.Context) { +func (serv *UploadServer) handleEmbed(c *gin.Context) { queryURL := c.Query("url") if !isValidURL(queryURL) { c.AbortWithStatus(http.StatusBadRequest) @@ -126,27 +219,35 @@ func handleEmbed(c *gin.Context) { hash := getHash(queryURL) - spew.Dump(queryURL, center, height, width) + serv.log.Debug(). + Msgf("Embed request\n\turl: %s\n\thash: %s", queryURL, hash) - cacheLock.Lock() + cacheMutex.Lock() item, ok := cache[hash] if !ok { // Cache miss create new cache item + serv.log.Debug(). + Msgf("HTML cache miss") item = &cacheItem{ url: queryURL, html: "", created: time.Now().Unix(), } - // Add to waitgroup so other clients can wait for the oEmbed result + // Add to waitgroup so other clients can wait for the embed result item.wg.Add(1) cache[hash] = item // Item added to cache, unlock so other requests can see the new item - cacheLock.Unlock() + cacheMutex.Unlock() + + // Check if the url looks like an image + if isImage.MatchString(queryURL) { + item.html = getImageHTML(c, queryURL, height) + } // Attempt to fetch oEmbed data - embedItem := embed.FindItem(queryURL) + embedItem := oEmbed.FindItem(queryURL) if embedItem != nil { options := oembed.Options{ URL: queryURL, @@ -155,27 +256,36 @@ func handleEmbed(c *gin.Context) { } info, err := embedItem.FetchOembed(options) if err != nil { - // An unexpected error occurred - fmt.Printf("An error occured: %s\n", err.Error()) + serv.log.Error(). + Err(err). + Msg("Unexpected error in oEmbed") } else if info.Status >= 300 { - // oEmbed returned an error status - fmt.Printf("Response status code is: %d\n", info.Status) + // oEmbed returned a bad status code + serv.log.Debug(). + Msgf("Bad response code from oEmbed: %d", info.Status) } else if info.HTML != "" { // oEmbed returned embedable html - fmt.Printf("Oembed info:\n%s\n", info) + serv.log.Debug(). + Msgf("oEmbed info:\n%s", info) item.html = info.HTML } else if info.Type == "photo" { // oEmbed returned a photo type the url should be an image - fmt.Println("type photo " + info.URL) + serv.log.Debug(). + Msgf("oEmbed info:\n%s", info) item.html = getImageHTML(c, info.URL, height) - } else { - spew.Dump(info) } } - // oEmbed did not return any html, maybe the url is direct to an image - if item.html == "" && isImage.MatchString(queryURL) { - item.html = getImageHTML(c, queryURL, height) + // No embedable html, time to try noembed + if item.html == "" { + noEmbedResp, err := noEmbed.Get(queryURL) + if err != nil { + serv.log.Error(). + Err(err). + Msg("Unexpected error in noEmbed") + } else { + item.html = noEmbedResp.HTML + } } // Still no html send an error to the parent @@ -186,9 +296,10 @@ func handleEmbed(c *gin.Context) { // Decrease the waitgroup so other requests can complete item.wg.Done() } else { - log.Printf("Cache HIT") // Cache hit unlock the cache - cacheLock.Unlock() + serv.log.Debug(). + Msg("HTML cache hit") + cacheMutex.Unlock() } // Wait until the cache item is fulfilled @@ -206,80 +317,79 @@ func handleEmbed(c *gin.Context) { c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(htmlData)) } -func getProviders(force bool) (*[]byte, error) { - log.Println("Getting oEmbed Providers") - if _, err := os.Stat("providers.json"); force || os.IsNotExist(err) { - resp, err := http.Get("https://oembed.com/providers.json") +func getProvidersCached(url string, filePath string, force bool) (*[]byte, error) { + var err error + if _, err = os.Stat(filePath); force || os.IsNotExist(err) { + var httpResp *http.Response + httpResp, err = http.Get(url) if err != nil { - return nil, errors.New("Failed to fetch oEmbed providers: " + err.Error()) + return nil, errors.New("Failed to fetch providers: " + err.Error()) } - defer resp.Body.Close() - - log.Println("Fetched oEmbed Providers") + defer httpResp.Body.Close() - data, err := ioutil.ReadAll(resp.Body) + var rawJSON []byte + rawJSON, err = ioutil.ReadAll(httpResp.Body) if err != nil { - return nil, errors.New("Failed to read oEmbed providers: " + err.Error()) + return nil, errors.New("Failed to read providers: " + err.Error()) } - log.Println("Read oEmbed Providers") - // Unmarshal to temp interface to ensure valid json var temp interface{} - err = json.Unmarshal(data, &temp) + err = json.Unmarshal(rawJSON, &temp) if err != nil { - return nil, errors.New("Failed to parse oEmbed providers: " + err.Error()) + return nil, errors.New("Failed to parse providers: " + err.Error()) } - log.Println("Tested oEmbed Providers") - // Data appears to be valid json open providers file for writing - file, err := os.Create("providers.json") + var file *os.File + file, err = os.Create(filePath) if err != nil { - return nil, errors.New("Failed to create oEmbed providers: " + err.Error()) + return nil, errors.New("Failed to create providers file: " + err.Error()) } defer file.Close() - log.Println("Created oEmbed Providers") - // Write providers.json - _, err = file.Write(data) + _, err = file.Write(rawJSON) if err != nil { - return nil, errors.New("Failed to write oEmbed providers: " + err.Error()) + return nil, errors.New("Failed to write providers: " + err.Error()) } - log.Println("Written oEmbed Providers") - - return &data, nil + return &rawJSON, nil + } else if err != nil { + return nil, errors.New("Failed to stat providers file: " + err.Error()) } - // Open existing providers.json - file, err := os.Open("providers.json") + // Open existing providers file + var file *os.File + file, err = os.Open(filePath) if err != nil { - return nil, errors.New("Failed to open oEmbed providers: " + err.Error()) + return nil, errors.New("Failed to open providers: " + err.Error()) } defer file.Close() - // Read providers.json data - data, err := ioutil.ReadAll(file) + // Read existing providers file + var rawJSON []byte + rawJSON, err = ioutil.ReadAll(file) if err != nil { - return nil, errors.New("Failed to read oEmbed providers: " + err.Error()) + return nil, errors.New("Failed to read providers: " + err.Error()) } - return &data, nil + return &rawJSON, nil } -func startCleanupTicker(cleanInterval, cacheMaxAge time.Duration) { +func (serv *UploadServer) startCleanupTicker(cleanInterval, cacheMaxAge time.Duration) { cacheTicker = time.NewTicker(cleanInterval) go func() { for range cacheTicker.C { - cleanCache(cacheMaxAge) + serv.cleanCache(cacheMaxAge) } }() } -func cleanCache(cacheMaxAge time.Duration) { +func (serv *UploadServer) cleanCache(cacheMaxAge time.Duration) { createdBefore := time.Now().Unix() - int64(cacheMaxAge.Seconds()) + + // Find expired items in HTML cache var expired []string for hash, item := range cache { if item.created >= createdBefore { @@ -288,21 +398,46 @@ func cleanCache(cacheMaxAge time.Duration) { expired = append(expired, hash) } - if len(expired) == 0 { - // Nothing to clean - log.Println("No cache items to clean") - return + // Find expired items in imgWaiter + var expiredWaiters []string + for hash, item := range imgWaiter { + if item.created >= createdBefore { + continue + } + expired = append(expiredWaiters, hash) + } + + // Remove expired items from HTML cache + if len(expired) > 0 { + serv.log.Debug(). + Msgf("Cleaning %d item from HTML cache", len(expired)) + + cacheMutex.Lock() + for _, hash := range expired { + if hash == "" { + break + } + log.Println("Deleting cache item: " + hash) + delete(cache, hash) + } + cacheMutex.Unlock() } - cacheLock.Lock() - for _, hash := range expired { - if hash == "" { - break + // Remove expired items from img waiter + if len(expiredWaiters) > 0 { + serv.log.Debug(). + Msgf("Cleaning %d item from img waiter cache", len(expired)) + + cacheMutex.Lock() + for _, hash := range expiredWaiters { + if hash == "" { + break + } + log.Println("Deleting cache item: " + hash) + delete(imgWaiter, hash) } - log.Println("Deleting cache item: " + hash) - delete(cache, hash) + cacheMutex.Unlock() } - cacheLock.Unlock() } func loadTemplate(templatePath string) error { diff --git a/server/image-proxy-cache.go b/server/image-proxy-cache.go index ccd8110..2771a7c 100644 --- a/server/image-proxy-cache.go +++ b/server/image-proxy-cache.go @@ -2,10 +2,8 @@ package server import ( "bytes" - "fmt" "sync" - "github.com/davecgh/go-spew/spew" "github.com/kiwiirc/plugin-fileuploader/shardedfilestore" "github.com/rs/zerolog" "github.com/tus/tusd" @@ -21,20 +19,20 @@ type ImageProxyCache struct { // Get returns the response corresponding to key if present func (c *ImageProxyCache) Get(key string) (resp []byte, ok bool) { urlHash := getHash(key) - spew.Dump("Get", key, urlHash) idInterface, ok := c.urlMap.Load(urlHash) if !ok { - fmt.Println("Not in map") - fmt.Println("") + // Not in map return []byte{}, false } id := idInterface.(string) reader, err := c.store.GetReader(id) if err != nil { - fmt.Println("No reader") - fmt.Println("") + // No file to read + c.log.Debug(). + Err(err). + Msg("Image missing from shardedfilestore, maybe it was cleaned") c.urlMap.Delete(urlHash) return []byte{}, false } @@ -42,24 +40,21 @@ func (c *ImageProxyCache) Get(key string) (resp []byte, ok bool) { buffer := new(bytes.Buffer) _, err = buffer.ReadFrom(reader) if err != nil { - fmt.Println("Failed read") - fmt.Println("") + // Read error + c.log.Debug(). + Err(err). + Msg("Failed to read image from shardedfilestore") c.urlMap.Delete(urlHash) return []byte{}, false } bytes := buffer.Bytes() - - spew.Dump(len(bytes)) - fmt.Println("Got from cache") - fmt.Println("") return bytes, true } // Set saves a response to the cache as key func (c *ImageProxyCache) Set(key string, resp []byte) { urlHash := getHash(key) - spew.Dump("Set", key, len(resp), urlHash) metaData := tusd.MetaData{ "Url": key, @@ -89,17 +84,12 @@ func (c *ImageProxyCache) Set(key string, resp []byte) { c.store.FinishUpload(id) c.urlMap.Store(urlHash, id) - - fmt.Println("Set in cache") - fmt.Println("") } // Delete removes the response with key from the cache func (c *ImageProxyCache) Delete(key string) { urlHash := getHash(key) c.urlMap.Delete(urlHash) - spew.Dump("Delete", key) - fmt.Println("") } // NewImageProxyCache returns a new Cache that will store files in basePath From 9dded0a1105ead906c49c69e1221c5792f8d4b44 Mon Sep 17 00:00:00 2001 From: ItsOnlyBinary Date: Fri, 11 Dec 2020 13:52:08 +0000 Subject: [PATCH 04/20] Rename embed.go && fixes --- noembed/noembed.go | 12 +++++++--- server/uploadserver.go | 2 +- server/{embed.go => web-preview.go} | 35 ++++++++++++++++------------- 3 files changed, 30 insertions(+), 19 deletions(-) rename server/{embed.go => web-preview.go} (95%) diff --git a/noembed/noembed.go b/noembed/noembed.go index 20e5d97..030f194 100644 --- a/noembed/noembed.go +++ b/noembed/noembed.go @@ -8,13 +8,15 @@ import ( "net/http" "regexp" "strings" + "time" ) var noembedURL = "https://noembed.com/embed?url={url}" // NoEmbed represents this package type NoEmbed struct { - data *Data + data *Data + httpClient *http.Client } // Data represents the data for noembed providers @@ -43,7 +45,11 @@ type Response struct { // New returns a Noembed object func New() *NoEmbed { - return &NoEmbed{} + return &NoEmbed{ + httpClient: &http.Client{ + Timeout: time.Second * 30, + }, + } } // ParseProviders parses the raw json obtained from noembed.com @@ -73,7 +79,7 @@ func (n *NoEmbed) Get(url string) (resp *Response, err error) { reqURL := strings.Replace(noembedURL, "{url}", url, 1) var httpResp *http.Response - httpResp, err = http.Get(reqURL) + httpResp, err = n.httpClient.Get(reqURL) if err != nil { return } diff --git a/server/uploadserver.go b/server/uploadserver.go index f944f07..6f58b2e 100644 --- a/server/uploadserver.go +++ b/server/uploadserver.go @@ -78,7 +78,7 @@ func (serv *UploadServer) Run(replaceableHandler *ReplaceableHandler) error { return err } - err = serv.registerEmbedHandlers(serv.Router, serv.cfg) + err = serv.registerWebPreviewHandlers(serv.Router, serv.cfg) if err != nil { return err } diff --git a/server/embed.go b/server/web-preview.go similarity index 95% rename from server/embed.go rename to server/web-preview.go index 2c24b70..72336f0 100644 --- a/server/embed.go +++ b/server/web-preview.go @@ -7,7 +7,6 @@ import ( "encoding/json" "errors" "io/ioutil" - "log" "net/http" "net/url" "os" @@ -37,6 +36,8 @@ type imgWaiterItem struct { wg sync.WaitGroup } +var httpClient *http.Client + // HTML template var template string var templateLock sync.RWMutex @@ -57,10 +58,14 @@ var imgProxy *imageproxy.Proxy // Used to detect possible image urls var isImage = regexp.MustCompile(`\.(jpe?g|png|gifv?)$`) -func (serv *UploadServer) registerEmbedHandlers(r *gin.Engine, cfg Config) error { +func (serv *UploadServer) registerWebPreviewHandlers(r *gin.Engine, cfg Config) error { serv.log.Info(). Msg("Starting embed handlers") + httpClient = &http.Client{ + Timeout: time.Second * 30, + } + // Prepare oEmbed provider oembedJSON, err := getProvidersCached("https://oembed.com/providers.json", "oembed-providers.json", false) if err != nil { @@ -127,7 +132,7 @@ func (serv *UploadServer) registerEmbedHandlers(r *gin.Engine, cfg Config) error // Register our handler rg := r.Group("/embed") - rg.GET("", serv.handleEmbed) + rg.GET("", serv.handleWebPreview) // Create imageproxy and provide interface to shardedfilestore imgCache := NewImageProxyCache(serv.store, serv.log) @@ -190,7 +195,7 @@ func (serv *UploadServer) handleImageCache(c *gin.Context) { } } -func (serv *UploadServer) handleEmbed(c *gin.Context) { +func (serv *UploadServer) handleWebPreview(c *gin.Context) { queryURL := c.Query("url") if !isValidURL(queryURL) { c.AbortWithStatus(http.StatusBadRequest) @@ -321,7 +326,7 @@ func getProvidersCached(url string, filePath string, force bool) (*[]byte, error var err error if _, err = os.Stat(filePath); force || os.IsNotExist(err) { var httpResp *http.Response - httpResp, err = http.Get(url) + httpResp, err = httpClient.Get(url) if err != nil { return nil, errors.New("Failed to fetch providers: " + err.Error()) } @@ -414,11 +419,11 @@ func (serv *UploadServer) cleanCache(cacheMaxAge time.Duration) { cacheMutex.Lock() for _, hash := range expired { - if hash == "" { - break - } - log.Println("Deleting cache item: " + hash) delete(cache, hash) + serv.log.Info(). + Str("event", "expired"). + Str("hash", hash). + Msg("Pruned from HTML cache") } cacheMutex.Unlock() } @@ -428,15 +433,15 @@ func (serv *UploadServer) cleanCache(cacheMaxAge time.Duration) { serv.log.Debug(). Msgf("Cleaning %d item from img waiter cache", len(expired)) - cacheMutex.Lock() + imgWaiterMutex.Lock() for _, hash := range expiredWaiters { - if hash == "" { - break - } - log.Println("Deleting cache item: " + hash) delete(imgWaiter, hash) + serv.log.Info(). + Str("event", "expired"). + Str("hash", hash). + Msg("Pruned from image waiter") } - cacheMutex.Unlock() + imgWaiterMutex.Unlock() } } From 69dd56e5245f10af4f74b6c57b15b7eeb6741782 Mon Sep 17 00:00:00 2001 From: ItsOnlyBinary Date: Mon, 14 Dec 2020 16:03:29 +0000 Subject: [PATCH 05/20] Make fallback embed more generalised && bundle html templates in the binary --- fallback-embed/fallback-embed-provider.go | 125 +++++++++++++ fileuploader.config.example.toml | 10 +- main.go | 2 + noembed/noembed.go | 124 ------------- scripts/generate-templates.go | 27 +++ server/config.go | 14 +- server/web-preview.go | 217 +++++++++++----------- templates/templates.go | 65 +++++++ 8 files changed, 346 insertions(+), 238 deletions(-) create mode 100644 fallback-embed/fallback-embed-provider.go delete mode 100644 noembed/noembed.go create mode 100644 scripts/generate-templates.go create mode 100644 templates/templates.go diff --git a/fallback-embed/fallback-embed-provider.go b/fallback-embed/fallback-embed-provider.go new file mode 100644 index 0000000..637fcef --- /dev/null +++ b/fallback-embed/fallback-embed-provider.go @@ -0,0 +1,125 @@ +package fallbackembed + +import ( + "encoding/json" + "errors" + "io" + "io/ioutil" + "net/http" + "regexp" + "strconv" + "strings" + "time" +) + +// FallbackEmbed represents this package +type FallbackEmbed struct { + data *Data + httpClient *http.Client + targetKey string + providerURL string +} + +// Data represents the data for FallbackEmbed providers +type Data []struct { + Name string `json:"name"` + Patterns []Regex `json:"patterns"` +} + +// New returns a FallbackEmbed object +func New(providerURL, targetKey string) *FallbackEmbed { + obj := &FallbackEmbed{ + httpClient: &http.Client{ + Timeout: time.Second * 30, + }, + providerURL: providerURL, + targetKey: targetKey, + } + + return obj +} + +// ParseProviders parses the raw json obtained from noembed.com +func (f *FallbackEmbed) ParseProviders(buf io.Reader) error { + data, err := ioutil.ReadAll(buf) + if err != nil { + return err + } + + var providerData Data + err = json.Unmarshal(data, &providerData) + if err != nil { + return err + } + + f.data = &providerData + return nil +} + +// Get returns html string +func (f *FallbackEmbed) Get(url string, width int, height int) (html string, err error) { + if !f.ValidURL(url) { + return + } + + // Do replacements + reqURL := strings.Replace(f.providerURL, "{url}", url, 1) + reqURL = strings.Replace(reqURL, "{width}", strconv.Itoa(width), 1) + reqURL = strings.Replace(reqURL, "{height}", strconv.Itoa(height), 1) + + var httpResp *http.Response + httpResp, err = f.httpClient.Get(reqURL) + if err != nil { + return + } + defer httpResp.Body.Close() + + var body []byte + body, err = ioutil.ReadAll(httpResp.Body) + if err != nil { + return + } + + // Try to parse json response + resp := make(map[string]interface{}) + err = json.Unmarshal(body, &resp) + if err != nil { + return + } + + // Check targetKey exists + if jsonVal, ok := resp[f.targetKey]; ok { + // Check targetVal is string + if htmlString, ok := jsonVal.(string); ok { + html = htmlString + return + } + } + + err = errors.New("Failed to get target json key") + return +} + +// ValidURL is used to test if a url is supported by noembed +func (f *FallbackEmbed) ValidURL(url string) bool { + for _, entry := range *f.data { + for _, pattern := range entry.Patterns { + if pattern.Regexp.MatchString(url) { + return true + } + } + } + return false +} + +// Regex Unmarshaler +type Regex struct { + regexp.Regexp +} + +// UnmarshalText used to unmarshal regexp's from text +func (r *Regex) UnmarshalText(text []byte) error { + reg, err := regexp.Compile(string(text)) + r.Regexp = *reg + return err +} diff --git a/fileuploader.config.example.toml b/fileuploader.config.example.toml index d7fc244..6d82a08 100644 --- a/fileuploader.config.example.toml +++ b/fileuploader.config.example.toml @@ -45,11 +45,17 @@ MaxAge = "24h" # 1 day IdentifiedMaxAge = "168h" # 1 week CheckInterval = "5m" -[Embed] -TemplatePath = "templates/embed.html" +[WebPreview] +TemplatesDirectory = "templates" CacheMaxAge = "1h" CacheCleanInterval = "15m" +# Fallback provider specific +FallbackProviderDisabled = false +FallbackProviderURL = "https://noembed.com/embed?url={url}" +FallbackProviderFile = "fallback-providers.json" +FallbackProviderJsonKey = "html" + # If EXTJWT is supported by the gateway or network, a validated token with an account present (when # the user is authenticated to an irc services account) will use the IdentifiedMaxAge setting above # instead of the base MaxAge. diff --git a/main.go b/main.go index 3ca320e..9fdc915 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,8 @@ import ( "github.com/kiwiirc/plugin-fileuploader/server" ) +//go:generate go run ./scripts/generate-templates.go + func main() { var configPath = flag.String("config", "fileuploader.config.toml", "path to config file") flag.Parse() diff --git a/noembed/noembed.go b/noembed/noembed.go deleted file mode 100644 index 030f194..0000000 --- a/noembed/noembed.go +++ /dev/null @@ -1,124 +0,0 @@ -package noembed - -import ( - "encoding/json" - "errors" - "io" - "io/ioutil" - "net/http" - "regexp" - "strings" - "time" -) - -var noembedURL = "https://noembed.com/embed?url={url}" - -// NoEmbed represents this package -type NoEmbed struct { - data *Data - httpClient *http.Client -} - -// Data represents the data for noembed providers -type Data []struct { - Name string `json:"name"` - Patterns []Regex `json:"patterns"` -} - -// Response represents the data returned by noembed server -type Response struct { - AuthorName string `json:"author_name"` - AuthorURL string `json:"author_url"` - ProviderName string `json:"provider_name"` - ProviderURL string `json:"provider_url"` - Title string `json:"title"` - Type string `json:"type"` - URL string `json:"url"` - HTML string `json:"html"` - Version string `json:"version"` - ThumbnailURL string `json:"thumbnail_url"` - ThumbnailWidth int `json:"thumbnail_width,string"` - ThumbnailHeight int `json:"thumbnail_height,string"` - Width int `json:"width,string"` - Height int `json:"height,string"` -} - -// New returns a Noembed object -func New() *NoEmbed { - return &NoEmbed{ - httpClient: &http.Client{ - Timeout: time.Second * 30, - }, - } -} - -// ParseProviders parses the raw json obtained from noembed.com -func (n *NoEmbed) ParseProviders(buf io.Reader) error { - data, err := ioutil.ReadAll(buf) - if err != nil { - return err - } - - var noembedData Data - err = json.Unmarshal(data, &noembedData) - if err != nil { - return err - } - - n.data = &noembedData - return nil -} - -// Get returns a noembed response object -func (n *NoEmbed) Get(url string) (resp *Response, err error) { - if !n.ValidURL(url) { - err = errors.New("Unsupported URL") - return - } - - reqURL := strings.Replace(noembedURL, "{url}", url, 1) - - var httpResp *http.Response - httpResp, err = n.httpClient.Get(reqURL) - if err != nil { - return - } - defer httpResp.Body.Close() - - var body []byte - body, err = ioutil.ReadAll(httpResp.Body) - if err != nil { - return - } - - err = json.Unmarshal(body, &resp) - if err != nil { - return - } - - return -} - -// ValidURL is used to test if a url is supported by noembed -func (n *NoEmbed) ValidURL(url string) bool { - for _, entry := range *n.data { - for _, pattern := range entry.Patterns { - if pattern.Regexp.MatchString(url) { - return true - } - } - } - return false -} - -// Regex Unmarshaler -type Regex struct { - regexp.Regexp -} - -// UnmarshalText used to unmarshal regexp's from text -func (r *Regex) UnmarshalText(text []byte) error { - reg, err := regexp.Compile(string(text)) - r.Regexp = *reg - return err -} diff --git a/scripts/generate-templates.go b/scripts/generate-templates.go new file mode 100644 index 0000000..f88bc61 --- /dev/null +++ b/scripts/generate-templates.go @@ -0,0 +1,27 @@ +package main + +import ( + "io" + "io/ioutil" + "os" + "path" + "strings" +) + +const templatesDir = "./templates" + +func main() { + files, _ := ioutil.ReadDir(templatesDir) + + out, _ := os.Create(path.Join(templatesDir, "templates.go")) + out.Write([]byte("package templates\n\nvar Get = map[string]string{\n")) + for _, fileInfo := range files { + if strings.HasSuffix(fileInfo.Name(), ".html") { + out.Write([]byte(strings.TrimSuffix(fileInfo.Name(), ".html") + ": `")) + file, _ := os.Open(path.Join(templatesDir, fileInfo.Name())) + io.Copy(out, file) + out.Write([]byte("`,\n")) + } + } + out.Write([]byte("}\n")) +} diff --git a/server/config.go b/server/config.go index 8c57014..80c1969 100644 --- a/server/config.go +++ b/server/config.go @@ -46,13 +46,17 @@ type Config struct { JwtSecretsByIssuer map[string]string Loggers []LoggerConfig - // Embed Provider - Embed struct { - TemplatePath string + // WebPreview config options + WebPreview struct { + TemplatesDirectory string CacheMaxAge duration CacheCleanInterval duration - ImageCachePath string - ImageCacheMaxSize uint64 + + // Fallback provider configuration + FallbackProviderDisabled bool + FallbackProviderURL string + FallbackProviderFile string + FallbackProviderJsonKey string } } diff --git a/server/web-preview.go b/server/web-preview.go index 72336f0..042220f 100644 --- a/server/web-preview.go +++ b/server/web-preview.go @@ -1,15 +1,16 @@ package server import ( + "bufio" "bytes" "crypto/sha256" "encoding/hex" - "encoding/json" "errors" "io/ioutil" "net/http" "net/url" "os" + "path" "regexp" "strconv" "strings" @@ -18,7 +19,8 @@ import ( "github.com/dyatlov/go-oembed/oembed" "github.com/gin-gonic/gin" - "github.com/kiwiirc/plugin-fileuploader/noembed" + fallbackembed "github.com/kiwiirc/plugin-fileuploader/fallback-embed" + "github.com/kiwiirc/plugin-fileuploader/templates" "willnorris.com/go/imageproxy" ) @@ -52,7 +54,8 @@ var imgWaiter = make(map[string]*imgWaiterItem) var imgWaiterMutex sync.Mutex var oEmbed *oembed.Oembed -var noEmbed *noembed.NoEmbed +var fallbackEmbed *fallbackembed.FallbackEmbed +var fallbackEmbedDisabled bool var imgProxy *imageproxy.Proxy // Used to detect possible image urls @@ -60,14 +63,47 @@ var isImage = regexp.MustCompile(`\.(jpe?g|png|gifv?)$`) func (serv *UploadServer) registerWebPreviewHandlers(r *gin.Engine, cfg Config) error { serv.log.Info(). - Msg("Starting embed handlers") + Msg("Starting web preview handlers") httpClient = &http.Client{ Timeout: time.Second * 30, } + // Check config defaults + cacheCleanInterval := cfg.WebPreview.CacheCleanInterval.Duration + if cacheCleanInterval == time.Duration(0) { + cacheCleanInterval, _ = time.ParseDuration("15m") + } + + cacheMaxAge := cfg.WebPreview.CacheMaxAge.Duration + if cacheMaxAge == time.Duration(0) { + cacheMaxAge, _ = time.ParseDuration("1h") + } + + templatesDir := cfg.WebPreview.TemplatesDirectory + if templatesDir == "" { + templatesDir = "templates" + } + + fallbackProviderURL := cfg.WebPreview.FallbackProviderURL + if fallbackProviderURL == "" { + fallbackProviderURL = "https://noembed.com/embed?url={url}" + } + + fallbackProviderFile := cfg.WebPreview.FallbackProviderFile + if fallbackProviderFile == "" { + fallbackProviderFile = "fallback-providers.json" + } + + fallbackProviderJsonKey := cfg.WebPreview.FallbackProviderJsonKey + if fallbackProviderJsonKey == "" { + fallbackProviderJsonKey = "html" + } + + fallbackEmbedDisabled = cfg.WebPreview.FallbackProviderDisabled + // Prepare oEmbed provider - oembedJSON, err := getProvidersCached("https://oembed.com/providers.json", "oembed-providers.json", false) + oembedJSON, err := getEmbedProviders("https://oembed.com/providers.json") if err != nil { serv.log.Error(). Err(err). @@ -75,7 +111,7 @@ func (serv *UploadServer) registerWebPreviewHandlers(r *gin.Engine, cfg Config) return err } oEmbed = oembed.NewOembed() - err = oEmbed.ParseProviders(bytes.NewReader(*oembedJSON)) + err = oEmbed.ParseProviders(oembedJSON) if err != nil { serv.log.Error(). Err(err). @@ -83,37 +119,42 @@ func (serv *UploadServer) registerWebPreviewHandlers(r *gin.Engine, cfg Config) return err } - // Prepare noEmbed provider - noembedJSON, err := getProvidersCached("https://noembed.com/providers", "noembed-providers.json", false) - if err != nil { - serv.log.Error(). - Err(err). - Msg("Failed to get noembed providers json") - return err - } - noEmbed = noembed.New() - err = noEmbed.ParseProviders(bytes.NewReader(*noembedJSON)) - if err != nil { - serv.log.Error(). - Err(err). - Msg("Failed to parse noembed providers json") - return err - } - - // Check config defaults - cacheCleanInterval := cfg.Embed.CacheCleanInterval.Duration - if cacheCleanInterval == time.Duration(0) { - cacheCleanInterval, _ = time.ParseDuration("15m") - } - - cacheMaxAge := cfg.Embed.CacheMaxAge.Duration - if cacheMaxAge == time.Duration(0) { - cacheMaxAge, _ = time.ParseDuration("1h") - } - - templatePath := cfg.Embed.TemplatePath - if templatePath == "" { - templatePath = "templates/embed.html" + if !fallbackEmbedDisabled { + if _, err := os.Stat(fallbackProviderFile); err == nil { + // Fallback provider file exists attempt to read and parse it + file, err := os.Open(fallbackProviderFile) + if err != nil { + serv.log.Error(). + Err(err). + Msg("Failed to open fallback providers json") + return err + } + err = fallbackEmbed.ParseProviders(bufio.NewReader(file)) + if err != nil { + serv.log.Error(). + Err(err). + Msg("Failed to parse fallback providers json") + return err + } + } else if strings.HasPrefix(fallbackProviderURL, "https://noembed.com/") { + // No fallback provider file and the url is noembed.com + // attempt to fetch and parse the providers.json from noembed.com + noembedJSON, err := getEmbedProviders("https://noembed.com/providers") + if err != nil { + serv.log.Error(). + Err(err). + Msg("Failed to get fallback providers json") + return err + } + fallbackEmbed = fallbackembed.New(fallbackProviderURL, fallbackProviderJsonKey) + err = fallbackEmbed.ParseProviders(noembedJSON) + if err != nil { + serv.log.Error(). + Err(err). + Msg("Failed to parse noembed providers json") + return err + } + } } // Start the cleanup ticker @@ -123,12 +164,7 @@ func (serv *UploadServer) registerWebPreviewHandlers(r *gin.Engine, cfg Config) ) // Load embed html template - if err := loadTemplate(templatePath); err != nil { - serv.log.Error(). - Err(err). - Msg("Failed to load template") - return err - } + serv.initTemplates(templatesDir) // Register our handler rg := r.Group("/embed") @@ -281,15 +317,15 @@ func (serv *UploadServer) handleWebPreview(c *gin.Context) { } } - // No embedable html, time to try noembed - if item.html == "" { - noEmbedResp, err := noEmbed.Get(queryURL) + // No embedable html, time to try our fallback provider + if item.html == "" && !fallbackEmbedDisabled { + noEmbedResp, err := fallbackEmbed.Get(queryURL, width, height) if err != nil { serv.log.Error(). Err(err). Msg("Unexpected error in noEmbed") } else { - item.html = noEmbedResp.HTML + item.html = noEmbedResp } } @@ -322,64 +358,19 @@ func (serv *UploadServer) handleWebPreview(c *gin.Context) { c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(htmlData)) } -func getProvidersCached(url string, filePath string, force bool) (*[]byte, error) { - var err error - if _, err = os.Stat(filePath); force || os.IsNotExist(err) { - var httpResp *http.Response - httpResp, err = httpClient.Get(url) - if err != nil { - return nil, errors.New("Failed to fetch providers: " + err.Error()) - } - defer httpResp.Body.Close() - - var rawJSON []byte - rawJSON, err = ioutil.ReadAll(httpResp.Body) - if err != nil { - return nil, errors.New("Failed to read providers: " + err.Error()) - } - - // Unmarshal to temp interface to ensure valid json - var temp interface{} - err = json.Unmarshal(rawJSON, &temp) - if err != nil { - return nil, errors.New("Failed to parse providers: " + err.Error()) - } - - // Data appears to be valid json open providers file for writing - var file *os.File - file, err = os.Create(filePath) - if err != nil { - return nil, errors.New("Failed to create providers file: " + err.Error()) - } - defer file.Close() - - // Write providers.json - _, err = file.Write(rawJSON) - if err != nil { - return nil, errors.New("Failed to write providers: " + err.Error()) - } - - return &rawJSON, nil - } else if err != nil { - return nil, errors.New("Failed to stat providers file: " + err.Error()) - } - - // Open existing providers file - var file *os.File - file, err = os.Open(filePath) +func getEmbedProviders(url string) (*bytes.Reader, error) { + var httpResp *http.Response + httpResp, err := httpClient.Get(url) if err != nil { - return nil, errors.New("Failed to open providers: " + err.Error()) + return nil, errors.New("Failed to fetch embed providers: " + err.Error()) } - defer file.Close() + defer httpResp.Body.Close() - // Read existing providers file - var rawJSON []byte - rawJSON, err = ioutil.ReadAll(file) + body, err := ioutil.ReadAll(httpResp.Body) if err != nil { - return nil, errors.New("Failed to read providers: " + err.Error()) + return nil, errors.New("Failed to read embed providers: " + err.Error()) } - - return &rawJSON, nil + return bytes.NewReader(body), nil } func (serv *UploadServer) startCleanupTicker(cleanInterval, cacheMaxAge time.Duration) { @@ -409,7 +400,7 @@ func (serv *UploadServer) cleanCache(cacheMaxAge time.Duration) { if item.created >= createdBefore { continue } - expired = append(expiredWaiters, hash) + expiredWaiters = append(expiredWaiters, hash) } // Remove expired items from HTML cache @@ -420,7 +411,7 @@ func (serv *UploadServer) cleanCache(cacheMaxAge time.Duration) { cacheMutex.Lock() for _, hash := range expired { delete(cache, hash) - serv.log.Info(). + serv.log.Debug(). Str("event", "expired"). Str("hash", hash). Msg("Pruned from HTML cache") @@ -431,12 +422,12 @@ func (serv *UploadServer) cleanCache(cacheMaxAge time.Duration) { // Remove expired items from img waiter if len(expiredWaiters) > 0 { serv.log.Debug(). - Msgf("Cleaning %d item from img waiter cache", len(expired)) + Msgf("Cleaning %d item from img waiter cache", len(expiredWaiters)) imgWaiterMutex.Lock() for _, hash := range expiredWaiters { delete(imgWaiter, hash) - serv.log.Info(). + serv.log.Debug(). Str("event", "expired"). Str("hash", hash). Msg("Pruned from image waiter") @@ -445,15 +436,27 @@ func (serv *UploadServer) cleanCache(cacheMaxAge time.Duration) { } } -func loadTemplate(templatePath string) error { +func (serv *UploadServer) initTemplates(templatesDir string) { + templatePath := path.Join(templatesDir, "Embed.html") + if _, err := os.Stat(templatePath); os.IsNotExist(err) { + // No template file use content from binary + templateLock.Lock() + template, _ = templates.Get["Embed"] + templateLock.Unlock() + return + } + + // Template file exists read it from disk html, err := ioutil.ReadFile(templatePath) if err != nil { - return err + serv.log.Error(). + Err(err). + Str("path", templatePath). + Msg("Failed to read embed template") } templateLock.Lock() template = string(html) templateLock.Unlock() - return nil } func getImageHTML(c *gin.Context, url string, height int) string { diff --git a/templates/templates.go b/templates/templates.go new file mode 100644 index 0000000..ff4613e --- /dev/null +++ b/templates/templates.go @@ -0,0 +1,65 @@ +package templates + +var Get = map[string]string{ + "Embed": ` + + + + + + +
+ {{body.html}} +
+ + +`, +} From 42ca4665ecfcca09457f6277c5add491beb41fff Mon Sep 17 00:00:00 2001 From: ItsOnlyBinary Date: Tue, 15 Dec 2020 14:31:00 +0000 Subject: [PATCH 06/20] Fix template scrollbars --- scripts/generate-templates.go | 2 +- server/web-preview.go | 19 ++++++++++++++++--- templates/embed.html | 4 +--- templates/templates.go | 6 ++---- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/scripts/generate-templates.go b/scripts/generate-templates.go index f88bc61..772a713 100644 --- a/scripts/generate-templates.go +++ b/scripts/generate-templates.go @@ -17,7 +17,7 @@ func main() { out.Write([]byte("package templates\n\nvar Get = map[string]string{\n")) for _, fileInfo := range files { if strings.HasSuffix(fileInfo.Name(), ".html") { - out.Write([]byte(strings.TrimSuffix(fileInfo.Name(), ".html") + ": `")) + out.Write([]byte("\"" + strings.TrimSuffix(fileInfo.Name(), ".html") + "\"" + ": `")) file, _ := os.Open(path.Join(templatesDir, fileInfo.Name())) io.Copy(out, file) out.Write([]byte("`,\n")) diff --git a/server/web-preview.go b/server/web-preview.go index 042220f..1d5cab6 100644 --- a/server/web-preview.go +++ b/server/web-preview.go @@ -347,14 +347,27 @@ func (serv *UploadServer) handleWebPreview(c *gin.Context) { item.wg.Wait() // Prepare html and send it to the client - style := "display: flex; justify-content: center;" + style := ` + body { + display: flex; justify-content: center; + }` if !center { - style = "" + style = ` + body { + overflow: hidden; + } + #kiwi-embed-container { + position: absolute; + top: 0; + left: 0; + bottom: 0; + overflow: auto; + }` } templateLock.RLock() htmlData := strings.Replace(template, "{{body.html}}", item.html, -1) templateLock.RUnlock() - htmlData = strings.Replace(htmlData, "/* style.body */", style, -1) + htmlData = strings.Replace(htmlData, "/* style.extras */", style, -1) c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(htmlData)) } diff --git a/templates/embed.html b/templates/embed.html index 12c5bcb..73ca15a 100644 --- a/templates/embed.html +++ b/templates/embed.html @@ -6,12 +6,10 @@ margin: 0; padding: 0; } - body { - /* style.body */ - } .kiwi-embed-image { display: block; } + /* style.extras */ " + item.html = "" } // Decrease the waitgroup so other requests can complete From 9bd5f38f33bde4292d137203c3a37047b98d005c Mon Sep 17 00:00:00 2001 From: ItsOnlyBinary Date: Fri, 18 Dec 2020 16:58:51 +0000 Subject: [PATCH 11/20] Delete completed upload instead of move if the file already exists in shardedfilestore --- shardedfilestore/shardedfilestore.go | 38 +++++++++++++++++++--------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/shardedfilestore/shardedfilestore.go b/shardedfilestore/shardedfilestore.go index 29ce355..1860187 100644 --- a/shardedfilestore/shardedfilestore.go +++ b/shardedfilestore/shardedfilestore.go @@ -200,7 +200,7 @@ func RemoveWithDirs(path string, basePath string) (err error) { return err } - empty, err := isDirEmpty(parent); + empty, err := isDirEmpty(parent) if empty { err = os.Remove(parent) } @@ -431,14 +431,28 @@ func (store *ShardedFileStore) FinishUpload(id string) error { newPath := store.completeBinPath(hash) os.MkdirAll(filepath.Dir(newPath), defaultDirectoryPerm) oldPath := store.incompleteBinPath(id) - err = os.Rename(oldPath, newPath) - if err != nil { - store.log.Error(). - Err(err). - Str("oldPath", oldPath). - Str("newPath", newPath). - Msg("Failed to rename") + + if _, err := os.Stat(newPath); err != nil { + // file needs moving to the sharded filestore + err = os.Rename(oldPath, newPath) + if err != nil { + store.log.Error(). + Err(err). + Str("oldPath", oldPath). + Str("newPath", newPath). + Msg("Failed to rename") + } + } else { + // file already exists just remove the tempoary upload + err = os.Remove(oldPath) + if err != nil { + store.log.Error(). + Err(err). + Str("oldPath", oldPath). + Msg("Failed to remove") + } } + return err } @@ -465,8 +479,8 @@ func isDirEmpty(path string) (bool, error) { defer f.Close() _, err = f.Readdirnames(1) - if err == io.EOF { - return true, nil - } - return false, err + if err == io.EOF { + return true, nil + } + return false, err } From c4b7c14e99f9199259a852a9cd0b3eeecf43a9b0 Mon Sep 17 00:00:00 2001 From: ItsOnlyBinary Date: Tue, 22 Dec 2020 13:21:25 +0000 Subject: [PATCH 12/20] Switch around registering handlers, as tusd adds middlewares to the main gin component --- server/uploadserver.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/uploadserver.go b/server/uploadserver.go index 7d5f4b9..148cb9a 100644 --- a/server/uploadserver.go +++ b/server/uploadserver.go @@ -74,12 +74,12 @@ func (serv *UploadServer) Run(replaceableHandler *ReplaceableHandler) error { serv.log, ) - err := serv.registerTusHandlers(serv.Router, serv.store) + err = serv.registerWebPreviewHandlers(serv.Router, serv.cfg) if err != nil { return err } - err = serv.registerWebPreviewHandlers(serv.Router, serv.cfg) + err := serv.registerTusHandlers(serv.Router, serv.store) if err != nil { return err } From 517cd32436e17a02a408cd2d0add7769b27ca146 Mon Sep 17 00:00:00 2001 From: ItsOnlyBinary Date: Tue, 22 Dec 2020 13:45:00 +0000 Subject: [PATCH 13/20] Fix merge issue --- server/uploadserver.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/uploadserver.go b/server/uploadserver.go index 148cb9a..d16bed5 100644 --- a/server/uploadserver.go +++ b/server/uploadserver.go @@ -74,12 +74,12 @@ func (serv *UploadServer) Run(replaceableHandler *ReplaceableHandler) error { serv.log, ) - err = serv.registerWebPreviewHandlers(serv.Router, serv.cfg) + err := serv.registerWebPreviewHandlers(serv.Router, serv.cfg) if err != nil { return err } - err := serv.registerTusHandlers(serv.Router, serv.store) + err = serv.registerTusHandlers(serv.Router, serv.store) if err != nil { return err } From ed5fad9fba29936eea90b260b4ca84a007b19c38 Mon Sep 17 00:00:00 2001 From: ItsOnlyBinary Date: Mon, 4 Jan 2021 16:13:23 +0000 Subject: [PATCH 14/20] Correctly encode url in fallback embed && better error handling --- fallback-embed/fallback-embed-provider.go | 18 ++++++++++++++---- server/web-preview.go | 8 +++++--- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/fallback-embed/fallback-embed-provider.go b/fallback-embed/fallback-embed-provider.go index 637fcef..10ff07c 100644 --- a/fallback-embed/fallback-embed-provider.go +++ b/fallback-embed/fallback-embed-provider.go @@ -6,6 +6,7 @@ import ( "io" "io/ioutil" "net/http" + "net/url" "regexp" "strconv" "strings" @@ -57,13 +58,13 @@ func (f *FallbackEmbed) ParseProviders(buf io.Reader) error { } // Get returns html string -func (f *FallbackEmbed) Get(url string, width int, height int) (html string, err error) { - if !f.ValidURL(url) { +func (f *FallbackEmbed) Get(wantedURL string, width int, height int) (html string, err error) { + if !f.ValidURL(wantedURL) { return } // Do replacements - reqURL := strings.Replace(f.providerURL, "{url}", url, 1) + reqURL := strings.Replace(f.providerURL, "{url}", url.QueryEscape(wantedURL), 1) reqURL = strings.Replace(reqURL, "{width}", strconv.Itoa(width), 1) reqURL = strings.Replace(reqURL, "{height}", strconv.Itoa(height), 1) @@ -96,7 +97,16 @@ func (f *FallbackEmbed) Get(url string, width int, height int) (html string, err } } - err = errors.New("Failed to get target json key") + // Check for error key in json response + if jsonVal, ok := resp["error"]; ok { + // Check error is string + if errorString, ok := jsonVal.(string); ok { + err = errors.New(errorString) + return + } + } + + err = errors.New(`Fallback embed provider did not include a JSON property of "` + f.targetKey + `"`) return } diff --git a/server/web-preview.go b/server/web-preview.go index 884f0cf..a3a233a 100644 --- a/server/web-preview.go +++ b/server/web-preview.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "errors" "io/ioutil" + "log" "net/http" "net/url" "os" @@ -233,6 +234,7 @@ func (serv *UploadServer) handleImageCache(c *gin.Context) { func (serv *UploadServer) handleWebPreview(c *gin.Context) { queryURL := c.Query("url") + log.Printf("queryURL: " + queryURL) if !isValidURL(queryURL) { c.AbortWithStatus(http.StatusBadRequest) return @@ -319,13 +321,13 @@ func (serv *UploadServer) handleWebPreview(c *gin.Context) { // No embedable html, time to try our fallback provider if item.html == "" && !fallbackEmbedDisabled { - noEmbedResp, err := fallbackEmbed.Get(queryURL, width, height) + fallbackEmbedResp, err := fallbackEmbed.Get(queryURL, width, height) if err != nil { serv.log.Error(). Err(err). - Msg("Unexpected error in noEmbed") + Msg("Unexpected error in fallback embed") } else { - item.html = noEmbedResp + item.html = fallbackEmbedResp } } From 8a1aff6476409e13e96b8167d65bc2d17f6f5fe1 Mon Sep 17 00:00:00 2001 From: ItsOnlyBinary Date: Mon, 4 Jan 2021 20:14:48 +0000 Subject: [PATCH 15/20] Better web-preview startup, allow local oEmbed providers json --- fileuploader.config.example.toml | 1 + server/config.go | 1 + server/uploadserver.go | 8 +- server/web-preview.go | 133 ++++++++++++++++++++----------- 4 files changed, 90 insertions(+), 53 deletions(-) diff --git a/fileuploader.config.example.toml b/fileuploader.config.example.toml index bf5ebf3..cbc5ceb 100644 --- a/fileuploader.config.example.toml +++ b/fileuploader.config.example.toml @@ -47,6 +47,7 @@ DeletedMaxAge = "720h" # 30 days CheckInterval = "5m" [WebPreview] +OembedProviderFile = "oembed-providers.json" TemplatesDirectory = "templates" CacheMaxAge = "1h" CacheCleanInterval = "15m" diff --git a/server/config.go b/server/config.go index e8ecc6e..6bf7c0d 100644 --- a/server/config.go +++ b/server/config.go @@ -49,6 +49,7 @@ type Config struct { // WebPreview config options WebPreview struct { + OembedProviderFile string TemplatesDirectory string CacheMaxAge duration CacheCleanInterval duration diff --git a/server/uploadserver.go b/server/uploadserver.go index d16bed5..b6e80aa 100644 --- a/server/uploadserver.go +++ b/server/uploadserver.go @@ -74,12 +74,10 @@ func (serv *UploadServer) Run(replaceableHandler *ReplaceableHandler) error { serv.log, ) - err := serv.registerWebPreviewHandlers(serv.Router, serv.cfg) - if err != nil { - return err - } + // If this fails to start it will log its own errors and not register any handlers + serv.registerWebPreviewHandlers(serv.Router, serv.cfg) - err = serv.registerTusHandlers(serv.Router, serv.store) + err := serv.registerTusHandlers(serv.Router, serv.store) if err != nil { return err } diff --git a/server/web-preview.go b/server/web-preview.go index a3a233a..871c1ae 100644 --- a/server/web-preview.go +++ b/server/web-preview.go @@ -6,8 +6,8 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "io" "io/ioutil" - "log" "net/http" "net/url" "os" @@ -86,6 +86,11 @@ func (serv *UploadServer) registerWebPreviewHandlers(r *gin.Engine, cfg Config) templatesDir = "templates" } + oembedProviderFile := cfg.WebPreview.OembedProviderFile + if oembedProviderFile == "" { + oembedProviderFile = "oembed-providers.json" + } + fallbackProviderURL := cfg.WebPreview.FallbackProviderURL if fallbackProviderURL == "" { fallbackProviderURL = "https://noembed.com/embed?url={url}" @@ -104,7 +109,7 @@ func (serv *UploadServer) registerWebPreviewHandlers(r *gin.Engine, cfg Config) fallbackEmbedDisabled = cfg.WebPreview.FallbackProviderDisabled // Prepare oEmbed provider - oembedJSON, err := getEmbedProviders("https://oembed.com/providers.json") + oembedJSON, err := serv.getProviderFileOrURL(oembedProviderFile, "https://oembed.com/providers.json") if err != nil { serv.log.Error(). Err(err). @@ -121,40 +126,17 @@ func (serv *UploadServer) registerWebPreviewHandlers(r *gin.Engine, cfg Config) } if !fallbackEmbedDisabled { - if _, err := os.Stat(fallbackProviderFile); err == nil { - // Fallback provider file exists attempt to read and parse it - file, err := os.Open(fallbackProviderFile) - if err != nil { - serv.log.Error(). - Err(err). - Msg("Failed to open fallback providers json") - return err - } - err = fallbackEmbed.ParseProviders(bufio.NewReader(file)) - if err != nil { - serv.log.Error(). - Err(err). - Msg("Failed to parse fallback providers json") - return err - } - } else if strings.HasPrefix(fallbackProviderURL, "https://noembed.com/") { - // No fallback provider file and the url is noembed.com - // attempt to fetch and parse the providers.json from noembed.com - noembedJSON, err := getEmbedProviders("https://noembed.com/providers") - if err != nil { - serv.log.Error(). - Err(err). - Msg("Failed to get fallback providers json") - return err - } - fallbackEmbed = fallbackembed.New(fallbackProviderURL, fallbackProviderJsonKey) - err = fallbackEmbed.ParseProviders(noembedJSON) - if err != nil { - serv.log.Error(). - Err(err). - Msg("Failed to parse noembed providers json") - return err - } + err := serv.initFallbackProvider( + fallbackProviderFile, + fallbackProviderURL, + fallbackProviderJsonKey, + ) + + if err != nil { + serv.log.Error(). + Err(err). + Msg("Failed to init fallback provider, disabling") + fallbackEmbedDisabled = true } } @@ -234,7 +216,7 @@ func (serv *UploadServer) handleImageCache(c *gin.Context) { func (serv *UploadServer) handleWebPreview(c *gin.Context) { queryURL := c.Query("url") - log.Printf("queryURL: " + queryURL) + if !isValidURL(queryURL) { c.AbortWithStatus(http.StatusBadRequest) return @@ -373,19 +355,25 @@ func (serv *UploadServer) handleWebPreview(c *gin.Context) { c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(htmlData)) } -func getEmbedProviders(url string) (*bytes.Reader, error) { - var httpResp *http.Response - httpResp, err := httpClient.Get(url) - if err != nil { - return nil, errors.New("Failed to fetch embed providers: " + err.Error()) - } - defer httpResp.Body.Close() +func (serv *UploadServer) getProviderFileOrURL(filename, url string) (io.Reader, error) { + if _, err := os.Stat(filename); err == nil { + serv.log.Debug(). + Str("file", filename). + Msg("Fetching embed providers from") + file, err := os.Open(filename) + if err == nil { + return file, nil + } - body, err := ioutil.ReadAll(httpResp.Body) - if err != nil { - return nil, errors.New("Failed to read embed providers: " + err.Error()) + serv.log.Error(). + Err(err). + Msg("Failed to open providers json") } - return bytes.NewReader(body), nil + + serv.log.Debug(). + Str("url", url). + Msg("Fetching embed providers from") + return getEmbedProviders(url) } func (serv *UploadServer) startCleanupTicker(cleanInterval, cacheMaxAge time.Duration) { @@ -451,6 +439,40 @@ func (serv *UploadServer) cleanCache(cacheMaxAge time.Duration) { } } +func (serv *UploadServer) initFallbackProvider(fallbackProviderFile, fallbackProviderURL, fallbackProviderJsonKey string) error { + if _, err := os.Stat(fallbackProviderFile); err == nil { + // Fallback provider file exists attempt to read and parse it + file, err := os.Open(fallbackProviderFile) + if err != nil { + return errors.New("Failed to open fallback providers json: " + err.Error()) + } + err = fallbackEmbed.ParseProviders(bufio.NewReader(file)) + if err != nil { + return errors.New("Failed to parse fallback providers json: " + err.Error()) + } + + return nil + } + + if strings.HasPrefix(fallbackProviderURL, "https://noembed.com/") { + // No fallback provider file and the url is noembed.com + // attempt to fetch and parse the providers.json from noembed.com + noembedJSON, err := getEmbedProviders("https://noembed.com/providers") + if err != nil { + return errors.New("Failed to get fallback providers json: " + err.Error()) + } + fallbackEmbed = fallbackembed.New(fallbackProviderURL, fallbackProviderJsonKey) + err = fallbackEmbed.ParseProviders(noembedJSON) + if err != nil { + return errors.New("Failed to parse fallback providers json: " + err.Error()) + } + + return nil + } + + return errors.New("Tried to init fallback provider without provider json file") +} + func (serv *UploadServer) initTemplates(templatesDir string) { templatePath := path.Join(templatesDir, "webpreview.html") if _, err := os.Stat(templatePath); os.IsNotExist(err) { @@ -474,6 +496,21 @@ func (serv *UploadServer) initTemplates(templatesDir string) { templateLock.Unlock() } +func getEmbedProviders(url string) (*bytes.Reader, error) { + var httpResp *http.Response + httpResp, err := httpClient.Get(url) + if err != nil { + return nil, errors.New("Failed to fetch embed providers: " + err.Error()) + } + defer httpResp.Body.Close() + + body, err := ioutil.ReadAll(httpResp.Body) + if err != nil { + return nil, errors.New("Failed to read embed providers: " + err.Error()) + } + return bytes.NewReader(body), nil +} + func getImageHTML(c *gin.Context, url string, height int) string { newURL := "http://" if c.Request.TLS != nil { From 3010bad0eba32b98764bed171a5ac6529441172f Mon Sep 17 00:00:00 2001 From: ItsOnlyBinary Date: Thu, 7 Jan 2021 19:38:35 +0000 Subject: [PATCH 16/20] Create webpreview plugin components --- .../src/components/WebPreview.vue | 147 ++++++++++++++++++ .../src/fileuploader-entry.js | 9 ++ server/web-preview.go | 3 +- templates/templates.go | 1 + templates/webpreview.html | 1 + 5 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 fileuploader-kiwiirc-plugin/src/components/WebPreview.vue diff --git a/fileuploader-kiwiirc-plugin/src/components/WebPreview.vue b/fileuploader-kiwiirc-plugin/src/components/WebPreview.vue new file mode 100644 index 0000000..fb4ed0b --- /dev/null +++ b/fileuploader-kiwiirc-plugin/src/components/WebPreview.vue @@ -0,0 +1,147 @@ +