diff --git a/.golangci.yaml b/.golangci.yaml index fcae1c9c..4ad5d09e 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,8 +1,6 @@ --- run: - # mochi in not written with generics (a.t.m), - # so we can check with 1.17 - go: "1.17" + go: "1.22" timeout: "5m" output: sort-results: true @@ -15,6 +13,7 @@ linters-settings: gosec: excludes: - "G505" # Allow SHA1 usage + - "G115" # FIXME: remove after https://github.com/securego/gosec/issues/1187 resolve linters: enable: - "bidichk" diff --git a/dist/example_config.yaml b/dist/example_config.yaml index 372f52db..0148a699 100644 --- a/dist/example_config.yaml +++ b/dist/example_config.yaml @@ -214,6 +214,10 @@ prehooks: # configuration: # hash_list: # - "a1b2c3d4e5a1b2c3d4e5a1b2c3d4e5a1b2c3d4e5" +# Path to watch new torrent files (only for initial_source: 'directory') +# path: "some/path" +# Time between two directory checks +# period: 5m # true - whitelist mode, false - blacklist # invert: false # Name of storage context where store hash list diff --git a/docs/middleware/torrent_approval.md b/docs/middleware/torrent_approval.md index 4828ead6..22c65686 100644 --- a/docs/middleware/torrent_approval.md +++ b/docs/middleware/torrent_approval.md @@ -25,7 +25,7 @@ There are two sources of hashes: `list` and `directory`. * `directory` will watch for `*.torrent` files in specified path and append/delete records from storage. This source will parse all existing - files at start and then watch for new files to add, or for delete events + files at start and then periodically watch for new files to add, or for delete events to remove hash from storage. Note: if storage is not `memory`, and `preserve` option set to `true`, records @@ -37,7 +37,8 @@ to storage) won't delete it. This middleware provides the following parameters for configuration: - `initial_source` - source type: `list` or `directory` -- `preserve`: - save source provided data into storage +- `storage` - storage configuration to store data, structure is same as global `storage` section. +If `name` is empty or `internal` global storage will be used - `configuration` - options for specified source - `list`: - `hash_list` - list of HEX encoded hashes @@ -46,6 +47,7 @@ This middleware provides the following parameters for configuration: It may be redis hash key, DB table name etc. - `directory`: - `path` - directory to watch + - `period` - time between two directory checks - `invert` and `storage_ctx` has the same meanins as `list`'s options Configuration example: @@ -58,9 +60,13 @@ mochi: - name: torrent approval options: initial_source: list - preserve: true + storage: + name: internal + config: configuration: hash_list: [ "AAA", "BBB" ] + path: "some/path" + period: 1m invert: false storage_ctx: APPROVED_HASH ``` diff --git a/go.mod b/go.mod index 6b25c6b5..36bc4496 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/sot-tech/mochi go 1.22 require ( - code.cloudfoundry.org/go-diodes v0.0.0-20240730232652-ce6331b0e7c0 + code.cloudfoundry.org/go-diodes v0.0.0-20240813203737-5032edb05ceb github.com/MicahParks/jwkset v0.5.18 github.com/MicahParks/keyfunc/v3 v3.3.3 github.com/PowerDNS/lmdb-go v1.9.2 @@ -15,11 +15,12 @@ require ( github.com/libp2p/go-reuseport v0.4.0 github.com/minio/sha256-simd v1.0.1 github.com/mitchellh/mapstructure v1.5.0 - github.com/prometheus/client_golang v1.19.1 + github.com/prometheus/client_golang v1.20.2 github.com/redis/go-redis/v9 v9.6.1 github.com/rs/zerolog v1.33.0 github.com/stretchr/testify v1.9.0 github.com/valyala/fasthttp v1.55.0 + github.com/zeebo/bencode v1.0.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -35,7 +36,6 @@ require ( github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/huandu/xstrings v1.5.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect @@ -50,16 +50,16 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/common v0.57.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/savsgio/gotils v0.0.0-20240704082632-aef3928b8a38 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect golang.org/x/crypto v0.26.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect golang.org/x/net v0.28.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.23.0 // indirect + golang.org/x/sys v0.24.0 // indirect golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.6.0 // indirect google.golang.org/protobuf v1.34.2 // indirect diff --git a/go.sum b/go.sum index 375c1ff9..5e6fd0c1 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -code.cloudfoundry.org/go-diodes v0.0.0-20240730232652-ce6331b0e7c0 h1:8B0xmGIfbmq8zCjVI00vwEzqanSGcrxvccau4bdjGRk= -code.cloudfoundry.org/go-diodes v0.0.0-20240730232652-ce6331b0e7c0/go.mod h1:QtvWqJPmrug53E4Vl1SNYn4jorWAjDJQr/DcAssb4TU= +code.cloudfoundry.org/go-diodes v0.0.0-20240813203737-5032edb05ceb h1:Last39ehN2b866DrM0B5FPIbdBFouL9humzWTqVTEX4= +code.cloudfoundry.org/go-diodes v0.0.0-20240813203737-5032edb05ceb/go.mod h1:PfkH7ePa4EhyC6VRisbZJOaIHhas7ofB0heANM//Z6Y= crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk= crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -88,8 +88,6 @@ github.com/fasthttp/router v1.5.2/go.mod h1:C8EY53ozOwpONyevc/V7Gr8pqnEjwnkFFqPo github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= @@ -133,8 +131,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190309154008-847fc94819f9/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -175,6 +173,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -206,8 +206,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo/v2 v2.19.1 h1:QXgq3Z8Crl5EL1WBAC98A5sEBHARrAJNzAmMxzLcRF0= -github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA= +github.com/onsi/ginkgo/v2 v2.20.0 h1:PE84V2mHqoT1sglvHc8ZdQtPcwmvvt29WLEEO3xmdZw= +github.com/onsi/ginkgo/v2 v2.20.0/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= @@ -223,8 +223,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg= +github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -234,8 +234,8 @@ github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQy github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY= +github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -280,6 +280,8 @@ github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8 github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/zeebo/bencode v1.0.0 h1:zgop0Wu1nu4IexAZeCZ5qbsjU4O1vMrfCrVgUjbHVuA= +github.com/zeebo/bencode v1.0.0/go.mod h1:Ct7CkrWIQuLWAy9M3atFHYq4kG9Ao/SsY5cdtCXmp9Y= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -288,8 +290,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -330,8 +332,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= @@ -344,8 +346,8 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= diff --git a/middleware/torrentapproval/container/directory/directory.go b/middleware/torrentapproval/container/directory/directory.go index 16714c82..e41065c5 100644 --- a/middleware/torrentapproval/container/directory/directory.go +++ b/middleware/torrentapproval/container/directory/directory.go @@ -6,22 +6,34 @@ package directory import ( "context" + "crypto/sha1" "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" - "github.com/anacrolix/torrent/metainfo" - "github.com/anacrolix/torrent/util/dirwatch" "github.com/minio/sha256-simd" + "github.com/zeebo/bencode" + "github.com/sot-tech/mochi/bittorrent" "github.com/sot-tech/mochi/middleware/torrentapproval/container" "github.com/sot-tech/mochi/middleware/torrentapproval/container/list" "github.com/sot-tech/mochi/pkg/conf" "github.com/sot-tech/mochi/pkg/log" + "github.com/sot-tech/mochi/pkg/str2bytes" "github.com/sot-tech/mochi/storage" ) var logger = log.NewLogger("middleware/torrent approval/directory") +const ( + defaultPeriod = time.Minute + maxTorrentSize = 10 * 1024 * 1024 +) + func init() { container.Register("directory", build) } @@ -32,6 +44,8 @@ type Config struct { list.Config // Path in filesystem where torrent files stored and should be watched Path string + // Period is time between two Path checks + Period time.Duration } func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, error) { @@ -46,7 +60,7 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er Storage: st, StorageCtx: c.StorageCtx, }, - watcher: nil, + closed: make(chan bool), } if len(d.StorageCtx) == 0 { logger.Warn(). @@ -56,77 +70,132 @@ func build(conf conf.MapConfig, st storage.DataStorage) (container.Container, er Msg("falling back to default configuration") d.StorageCtx = container.DefaultStorageCtxName } - var w *dirwatch.Instance - if w, err = dirwatch.New(c.Path); err != nil { - return nil, fmt.Errorf("unable to initialize directory watch: %w", err) + if c.Period == 0 { + logger.Warn(). + Str("name", "Period"). + Dur("provided", 0). + Dur("default", defaultPeriod). + Msg("falling back to default configuration") + c.Period = defaultPeriod } - d.watcher = w - go func() { - for event := range d.watcher.Events { - var mi *metainfo.MetaInfo - if mi, err = metainfo.LoadFromFile(event.TorrentFilePath); err == nil { - s256 := sha256.New() - s256.Write(mi.InfoBytes) - v2hash, _ := bittorrent.NewInfoHash(s256.Sum(nil)) - switch event.Change { - case dirwatch.Added: - var name string - if info, err := mi.UnmarshalInfo(); err == nil { - name = info.Name - } else { - logger.Error(). - Err(err). - Str("file", event.TorrentFilePath). - Stringer("infoHash", event.InfoHash). - Stringer("infoHashV2", v2hash). - Msg("unable to unmarshal torrent info") + go d.runScan(c.Path, c.Period) + return d, err +} + +// BencodeRawBytes wrapper for byte slice to get raw 'info' section from +// torrent file +type BencodeRawBytes []byte + +// UnmarshalBencode just appends raw byte slice to result +func (ba *BencodeRawBytes) UnmarshalBencode(in []byte) error { + *ba = append([]byte(nil), in...) + return nil +} + +type torrentRawInfoStruct struct { + Info BencodeRawBytes `bencode:"info"` +} + +type torrentNameInfoStruct struct { + Name string `bencode:"name"` +} + +func (d *directory) runScan(path string, period time.Duration) { + t := time.NewTicker(period) + defer t.Stop() + files := make(map[string][2]bittorrent.InfoHash) + tmpFiles := make(map[string]bool) + // nolint:gosec + s1, s2 := sha1.New(), sha256.New() + for { + select { + case <-d.closed: + return + case <-t.C: + logger.Debug().Msg("starting directory scan") + if entries, err := os.ReadDir(path); err == nil { + for _, e := range entries { + if !e.IsDir() && strings.ToLower(filepath.Ext(e.Name())) == ".torrent" { + tmpFiles[filepath.Join(path, e.Name())] = true } - if len(name) == 0 { - name = list.DUMMY + } + for p := range tmpFiles { + if _, exists := files[p]; !exists { + var f *os.File + if f, err = os.Open(p); err == nil { + var info torrentRawInfoStruct + err = bencode.NewDecoder(io.LimitReader(f, maxTorrentSize)).Decode(&info) + _ = f.Close() + if err == nil { + s1.Write(info.Info) + h1, _ := bittorrent.NewInfoHash(s1.Sum(nil)) + s1.Reset() + + s2.Write(info.Info) + h2, _ := bittorrent.NewInfoHash(s2.Sum(nil)) + s2.Reset() + + files[p] = [2]bittorrent.InfoHash{h1, h2} + var name torrentNameInfoStruct + if err := bencode.DecodeBytes(info.Info, &name); err != nil { + logger.Warn(). + Err(err). + Str("file", p). + Msg("unable to unmarshal torrent info") + } + if len(name.Name) == 0 { + name.Name = list.DUMMY + } + bName := str2bytes.StringToBytes(name.Name) + logger.Err(d.Storage.Put(context.Background(), d.StorageCtx, storage.Entry{ + Key: h1.RawString(), + Value: bName, + }, storage.Entry{ + Key: h2.RawString(), + Value: bName, + }, storage.Entry{ + Key: h2.TruncateV1().RawString(), + Value: bName, + })). + Str("file", p). + Stringer("infoHash", h1). + Stringer("infoHashV2", h2). + Msg("added torrent to approval list") + } + } + if err != nil { + logger.Warn().Err(err).Str("file", p).Msg("unable to read file") + } + } + } + for p, ih := range files { + if _, isOk := tmpFiles[p]; !isOk { + delete(files, p) + logger.Err(d.Storage.Delete(context.Background(), d.StorageCtx, ih[0].RawString(), + ih[1].RawString(), ih[1].TruncateV1().RawString())). + Str("file", p). + Stringer("infoHash", ih[1]). + Stringer("infoHashV2", ih[1]). + Msg("deleted torrent from approval list") } - bName := []byte(name) - logger.Err(d.Storage.Put(context.Background(), d.StorageCtx, storage.Entry{ - Key: event.InfoHash.AsString(), - Value: bName, - }, storage.Entry{ - Key: v2hash.RawString(), - Value: bName, - }, storage.Entry{ - Key: v2hash.TruncateV1().RawString(), - Value: bName, - })). - Str("action", "add"). - Str("file", event.TorrentFilePath). - Stringer("infoHash", event.InfoHash). - Stringer("infoHashV2", v2hash). - Msg("approval torrent watcher event") - case dirwatch.Removed: - logger.Err(d.Storage.Delete(context.Background(), c.StorageCtx, event.InfoHash.AsString(), v2hash.RawString(), v2hash.TruncateV1().RawString())). - Str("action", "delete"). - Str("file", event.TorrentFilePath). - Stringer("infoHash", event.InfoHash). - Stringer("infoHashV2", v2hash). - Msg("approval torrent watcher event") } + clear(tmpFiles) } else { - logger.Error().Err(err). - Str("file", event.TorrentFilePath). - Msg("unable to load torrent file") + logger.Warn().Err(err).Msg("unable to get directory content") } } - }() - return d, err + } } type directory struct { list.List - watcher *dirwatch.Instance + closed chan bool } // Close closes watching of torrent directory func (d *directory) Close() error { - if d.watcher != nil { - d.watcher.Close() + if d.closed != nil { + close(d.closed) } return nil } diff --git a/middleware/torrentapproval/torrentapproval_test.go b/middleware/torrentapproval/torrentapproval_test.go index daaf84f6..883a5094 100644 --- a/middleware/torrentapproval/torrentapproval_test.go +++ b/middleware/torrentapproval/torrentapproval_test.go @@ -95,6 +95,7 @@ func TestHandleAnnounce(t *testing.T) { } else { require.Equal(t, err, ErrTorrentUnapproved) } + _ = h.(*hook).Close() }) } }