diff --git a/go.mod b/go.mod index 7858341767..71057d6d3c 100644 --- a/go.mod +++ b/go.mod @@ -58,12 +58,12 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/jinzhu/copier v0.3.5 github.com/kubernetes-csi/csi-lib-utils v0.6.1 - github.com/longhorn/backing-image-manager v1.7.0-rc1 + github.com/longhorn/backing-image-manager v1.7.0-dev.0.20240823042906-1ae3d5073f60 github.com/longhorn/backupstore v0.0.0-20240823072635-7afd6aa10d3e github.com/longhorn/go-common-libs v0.0.0-20240821134112-907f57efd48f github.com/longhorn/go-iscsi-helper v0.0.0-20240811043302-df8de353dd58 github.com/longhorn/go-spdk-helper v0.0.0-20240820144231-33c0873802ff - github.com/longhorn/longhorn-engine v1.7.0-rc3 + github.com/longhorn/longhorn-engine v1.7.0-dev.0.20240824053610-9d2b194f765f github.com/longhorn/longhorn-instance-manager v1.7.0-dev.0.20240824081444-10108a7c3ffc github.com/longhorn/longhorn-share-manager v1.7.0-rc1 github.com/pkg/errors v0.9.1 @@ -119,11 +119,9 @@ require ( require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/RoaringBitmap/roaring v1.9.4 // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/avast/retry-go v3.0.0+incompatible github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.12.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect @@ -164,7 +162,6 @@ require ( github.com/moby/sys/mountinfo v0.7.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mschoch/smat v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect @@ -183,8 +180,8 @@ require ( go.etcd.io/etcd/api/v3 v3.5.10 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect go.etcd.io/etcd/client/v3 v3.5.10 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect go.opentelemetry.io/otel v1.19.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect diff --git a/go.sum b/go.sum index 35909860df..bde7bfa73e 100644 --- a/go.sum +++ b/go.sum @@ -770,8 +770,6 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ= -github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= @@ -806,8 +804,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= -github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= @@ -1221,8 +1217,8 @@ github.com/kubernetes-csi/csi-lib-utils v0.6.1 h1:+AZ58SRSRWh2vmMoWAAGcv7x6fIyBM github.com/kubernetes-csi/csi-lib-utils v0.6.1/go.mod h1:GVmlUmxZ+SUjVLXicRFjqWUUvWez0g0Y78zNV9t7KfQ= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/longhorn/backing-image-manager v1.7.0-rc1 h1:kD106yhLtofxDwXkvKs77VAFzTVdJnbxsD2m28X4tp0= -github.com/longhorn/backing-image-manager v1.7.0-rc1/go.mod h1:ZXD/+yKwMer/eZzwQ2ev/eAyLvih7WNq6NDgfUpvQ+8= +github.com/longhorn/backing-image-manager v1.7.0-dev.0.20240823042906-1ae3d5073f60 h1:iZrC1Xq2k9fUVFkeZrD6X6UxaPZNr3jBZ5tyo+Efbhg= +github.com/longhorn/backing-image-manager v1.7.0-dev.0.20240823042906-1ae3d5073f60/go.mod h1:uRo+kg+oA0+5jcFtBcajOzAv6pXjy7gKUgwMEndn698= github.com/longhorn/backupstore v0.0.0-20240823072635-7afd6aa10d3e h1:Kvt/BqiHKaOlTPSM2HFz0VupuQM0laln67rMIDCRYxQ= github.com/longhorn/backupstore v0.0.0-20240823072635-7afd6aa10d3e/go.mod h1:N4cqNhSs4VUw9aGbO2OfyiIvJL7/L53hUrNiT73UN+U= github.com/longhorn/go-common-libs v0.0.0-20240821134112-907f57efd48f h1:hjqUs3WVodkzrWwlUMVsnKAlom3uohoNlhZBGLsRvQY= @@ -1231,8 +1227,8 @@ github.com/longhorn/go-iscsi-helper v0.0.0-20240811043302-df8de353dd58 h1:fzLAnC github.com/longhorn/go-iscsi-helper v0.0.0-20240811043302-df8de353dd58/go.mod h1:TobRDCXmF0Ni+jz6+nLJamw3uVu+gNDZoZre1JczGwc= github.com/longhorn/go-spdk-helper v0.0.0-20240820144231-33c0873802ff h1:8vR29tkbmzmdqRVtOo5kL7Rs7nfhA6duXsmetIh1Tbg= github.com/longhorn/go-spdk-helper v0.0.0-20240820144231-33c0873802ff/go.mod h1:Bzz7kGNYikAJqpmeV3cgN8jP1y9M+/oaiBc5iolIxuA= -github.com/longhorn/longhorn-engine v1.7.0-rc3 h1:YTt++OeSrEOlifz++8VAOH/aJ4lGShD2TaJP1ZaQ3Uw= -github.com/longhorn/longhorn-engine v1.7.0-rc3/go.mod h1:2Hq/3QzW4fF2yUg+kauiAT3ps5WCKLMkrwXW2Wyfj9o= +github.com/longhorn/longhorn-engine v1.7.0-dev.0.20240824053610-9d2b194f765f h1:Nsal/5akxiEyoBL+M0NOXiV5R96ACEgC64rK5w0VKcY= +github.com/longhorn/longhorn-engine v1.7.0-dev.0.20240824053610-9d2b194f765f/go.mod h1:E1ec7ub7SNGvASDtiFHL1dXX4bhEQiroBixD2GGeRbQ= github.com/longhorn/longhorn-instance-manager v1.7.0-dev.0.20240824081444-10108a7c3ffc h1:BDcwW/MevFEkQSRoiNDm4E5a2+vqa3lZDEm7CQ0UHGs= github.com/longhorn/longhorn-instance-manager v1.7.0-dev.0.20240824081444-10108a7c3ffc/go.mod h1:tUatwl4OazNIwSEK1HQQCOwW9KSIJRzTfQH8WNI2eFY= github.com/longhorn/longhorn-share-manager v1.7.0-rc1 h1:LsSkSajhG8tCfORKKfwK+8XHVrT/8rI9DRWb7fuoVls= @@ -1284,8 +1280,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= -github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -1548,10 +1542,12 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 h1:ZOLJc06r4CB42laIXg/7udr0pbZyuAihN10A/XuiQRY= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 h1:KfYpVmrjI7JuToy5k8XV3nkapjWx48k4E4JOtVstzQI= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 h1:RsQi0qJ2imFfCvZabqzM9cNXBG8k6gXMv1A0cXRmH6A= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0/go.mod h1:vsh3ySueQCiKPxFLvjWC4Z135gIa34TQ/NSqkDTZYUM= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI= diff --git a/vendor/github.com/RoaringBitmap/roaring/.drone.yml b/vendor/github.com/RoaringBitmap/roaring/.drone.yml deleted file mode 100644 index 7936bfe8df..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/.drone.yml +++ /dev/null @@ -1,19 +0,0 @@ -kind: pipeline -name: default - -workspace: - base: /go - path: src/github.com/RoaringBitmap/roaring - -steps: -- name: test - image: golang - commands: - - go get -t - - go test - - go build -tags appengine - - go test -tags appengine - - GOARCH=386 go build - - GOARCH=386 go test - - GOARCH=arm go build - - GOARCH=arm64 go build diff --git a/vendor/github.com/RoaringBitmap/roaring/.gitignore b/vendor/github.com/RoaringBitmap/roaring/.gitignore deleted file mode 100644 index 851f323dba..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -*~ -roaring-fuzz.zip -workdir -coverage.out -testdata/all3.classic diff --git a/vendor/github.com/RoaringBitmap/roaring/.gitmodules b/vendor/github.com/RoaringBitmap/roaring/.gitmodules deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/vendor/github.com/RoaringBitmap/roaring/AUTHORS b/vendor/github.com/RoaringBitmap/roaring/AUTHORS deleted file mode 100644 index 26ec99de9d..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/AUTHORS +++ /dev/null @@ -1,11 +0,0 @@ -# This is the official list of roaring authors for copyright purposes. - -Todd Gruben (@tgruben), -Daniel Lemire (@lemire), -Elliot Murphy (@statik), -Bob Potter (@bpot), -Tyson Maly (@tvmaly), -Will Glynn (@willglynn), -Brent Pedersen (@brentp) -Maciej Biłas (@maciej), -Joe Nall (@joenall) diff --git a/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS b/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS deleted file mode 100644 index 1a8da9cc0f..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS +++ /dev/null @@ -1,18 +0,0 @@ -# This is the official list of roaring contributors - -Todd Gruben (@tgruben), -Daniel Lemire (@lemire), -Elliot Murphy (@statik), -Bob Potter (@bpot), -Tyson Maly (@tvmaly), -Will Glynn (@willglynn), -Brent Pedersen (@brentp), -Jason E. Aten (@glycerine), -Vali Malinoiu (@0x4139), -Forud Ghafouri (@fzerorubigd), -Joe Nall (@joenall), -(@fredim), -Edd Robinson (@e-dard), -Alexander Petrov (@alldroll), -Guy Molinari (@guymolinari), -Ling Jin (@JinLingChristopher) diff --git a/vendor/github.com/RoaringBitmap/roaring/LICENSE b/vendor/github.com/RoaringBitmap/roaring/LICENSE deleted file mode 100644 index 3ccdd00084..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/LICENSE +++ /dev/null @@ -1,235 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2016 by the authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -================================================================================ - -Portions of runcontainer.go are from the Go standard library, which is licensed -under: - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - * Neither the name of Google Inc. nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/RoaringBitmap/roaring/LICENSE-2.0.txt b/vendor/github.com/RoaringBitmap/roaring/LICENSE-2.0.txt deleted file mode 100644 index aff5f9999b..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/LICENSE-2.0.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2016 by the authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/RoaringBitmap/roaring/README.md b/vendor/github.com/RoaringBitmap/roaring/README.md deleted file mode 100644 index f6705dfe3a..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/README.md +++ /dev/null @@ -1,413 +0,0 @@ -# roaring - -[![GoDoc](https://godoc.org/github.com/RoaringBitmap/roaring?status.svg)](https://godoc.org/github.com/RoaringBitmap/roaring) [![Go Report Card](https://goreportcard.com/badge/RoaringBitmap/roaring)](https://goreportcard.com/report/github.com/RoaringBitmap/roaring) - -![Go-CI](https://github.com/RoaringBitmap/roaring/workflows/Go-CI/badge.svg) -![Go-ARM-CI](https://github.com/RoaringBitmap/roaring/workflows/Go-ARM-CI/badge.svg) -![Go-Windows-CI](https://github.com/RoaringBitmap/roaring/workflows/Go-Windows-CI/badge.svg) -============= - -This is a go version of the Roaring bitmap data structure. - -Roaring bitmaps are used by several major systems such as [Apache Lucene][lucene] and derivative systems such as [Solr][solr] and -[Elasticsearch][elasticsearch], [Apache Druid (Incubating)][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [anacrolix/torrent][anacrolix/torrent], [Whoosh][whoosh], [Redpanda](https://github.com/redpanda-data/redpanda), [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin]. The YouTube SQL Engine, [Google Procella](https://research.google/pubs/pub48388/), uses Roaring bitmaps for indexing. - -[lucene]: https://lucene.apache.org/ -[solr]: https://lucene.apache.org/solr/ -[elasticsearch]: https://www.elastic.co/products/elasticsearch -[druid]: https://druid.apache.org/ -[spark]: https://spark.apache.org/ -[opensearchserver]: http://www.opensearchserver.com -[anacrolix/torrent]: https://github.com/anacrolix/torrent -[whoosh]: https://bitbucket.org/mchaput/whoosh/wiki/Home -[pilosa]: https://www.pilosa.com/ -[kylin]: http://kylin.apache.org/ -[pinot]: http://github.com/linkedin/pinot/wiki -[vsts]: https://www.visualstudio.com/team-services/ -[atlas]: https://github.com/Netflix/atlas - -Roaring bitmaps are found to work well in many important applications: - -> Use Roaring for bitmap compression whenever possible. Do not use other bitmap compression methods ([Wang et al., SIGMOD 2017](http://db.ucsd.edu/wp-content/uploads/2017/03/sidm338-wangA.pdf)) - - -The ``roaring`` Go library is used by -* [anacrolix/torrent] -* [InfluxDB](https://www.influxdata.com) -* [Pilosa](https://www.pilosa.com/) -* [Bleve](http://www.blevesearch.com) -* [Weaviate](https://github.com/weaviate/weaviate) -* [lindb](https://github.com/lindb/lindb) -* [Elasticell](https://github.com/deepfabric/elasticell) -* [SourceGraph](https://github.com/sourcegraph/sourcegraph) -* [M3](https://github.com/m3db/m3) -* [trident](https://github.com/NetApp/trident) -* [Husky](https://www.datadoghq.com/blog/engineering/introducing-husky/) -* [FrostDB](https://github.com/polarsignals/frostdb) - -This library is used in production in several systems, it is part of the [Awesome Go collection](https://awesome-go.com). - - -There are also [Java](https://github.com/RoaringBitmap/RoaringBitmap) and [C/C++](https://github.com/RoaringBitmap/CRoaring) versions. The Java, C, C++ and Go version are binary compatible: e.g, you can save bitmaps -from a Java program and load them back in Go, and vice versa. We have a [format specification](https://github.com/RoaringBitmap/RoaringFormatSpec). - - -This code is licensed under Apache License, Version 2.0 (ASL2.0). - -Copyright 2016-... by the authors. - -When should you use a bitmap? -=================================== - - -Sets are a fundamental abstraction in -software. They can be implemented in various -ways, as hash sets, as trees, and so forth. -In databases and search engines, sets are often an integral -part of indexes. For example, we may need to maintain a set -of all documents or rows (represented by numerical identifier) -that satisfy some property. Besides adding or removing -elements from the set, we need fast functions -to compute the intersection, the union, the difference between sets, and so on. - - -To implement a set -of integers, a particularly appealing strategy is the -bitmap (also called bitset or bit vector). Using n bits, -we can represent any set made of the integers from the range -[0,n): the ith bit is set to one if integer i is present in the set. -Commodity processors use words of W=32 or W=64 bits. By combining many such words, we can -support large values of n. Intersections, unions and differences can then be implemented - as bitwise AND, OR and ANDNOT operations. -More complicated set functions can also be implemented as bitwise operations. - -When the bitset approach is applicable, it can be orders of -magnitude faster than other possible implementation of a set (e.g., as a hash set) -while using several times less memory. - -However, a bitset, even a compressed one is not always applicable. For example, if -you have 1000 random-looking integers, then a simple array might be the best representation. -We refer to this case as the "sparse" scenario. - -When should you use compressed bitmaps? -=================================== - -An uncompressed BitSet can use a lot of memory. For example, if you take a BitSet -and set the bit at position 1,000,000 to true and you have just over 100kB. That is over 100kB -to store the position of one bit. This is wasteful even if you do not care about memory: -suppose that you need to compute the intersection between this BitSet and another one -that has a bit at position 1,000,001 to true, then you need to go through all these zeroes, -whether you like it or not. That can become very wasteful. - -This being said, there are definitively cases where attempting to use compressed bitmaps is wasteful. -For example, if you have a small universe size. E.g., your bitmaps represent sets of integers -from [0,n) where n is small (e.g., n=64 or n=128). If you can use uncompressed BitSet and -it does not blow up your memory usage, then compressed bitmaps are probably not useful -to you. In fact, if you do not need compression, then a BitSet offers remarkable speed. - -The sparse scenario is another use case where compressed bitmaps should not be used. -Keep in mind that random-looking data is usually not compressible. E.g., if you have a small set of -32-bit random integers, it is not mathematically possible to use far less than 32 bits per integer, -and attempts at compression can be counterproductive. - -How does Roaring compares with the alternatives? -================================================== - - -Most alternatives to Roaring are part of a larger family of compressed bitmaps that are run-length-encoded -bitmaps. They identify long runs of 1s or 0s and they represent them with a marker word. -If you have a local mix of 1s and 0, you use an uncompressed word. - -There are many formats in this family: - -* Oracle's BBC is an obsolete format at this point: though it may provide good compression, -it is likely much slower than more recent alternatives due to excessive branching. -* WAH is a patented variation on BBC that provides better performance. -* Concise is a variation on the patented WAH. It some specific instances, it can compress -much better than WAH (up to 2x better), but it is generally slower. -* EWAH is both free of patent, and it is faster than all the above. On the downside, it -does not compress quite as well. It is faster because it allows some form of "skipping" -over uncompressed words. So though none of these formats are great at random access, EWAH -is better than the alternatives. - - - -There is a big problem with these formats however that can hurt you badly in some cases: there is no random access. If you want to check whether a given value is present in the set, you have to start from the beginning and "uncompress" the whole thing. This means that if you want to intersect a big set with a large set, you still have to uncompress the whole big set in the worst case... - -Roaring solves this problem. It works in the following manner. It divides the data into chunks of 216 integers -(e.g., [0, 216), [216, 2 x 216), ...). Within a chunk, it can use an uncompressed bitmap, a simple list of integers, -or a list of runs. Whatever format it uses, they all allow you to check for the presence of any one value quickly -(e.g., with a binary search). The net result is that Roaring can compute many operations much faster than run-length-encoded -formats like WAH, EWAH, Concise... Maybe surprisingly, Roaring also generally offers better compression ratios. - - - - - -### References - -- Daniel Lemire, Owen Kaser, Nathan Kurz, Luca Deri, Chris O'Hara, François Saint-Jacques, Gregory Ssi-Yan-Kai, Roaring Bitmaps: Implementation of an Optimized Software Library, Software: Practice and Experience 48 (4), 2018 [arXiv:1709.07821](https://arxiv.org/abs/1709.07821) -- Samy Chambi, Daniel Lemire, Owen Kaser, Robert Godin, -Better bitmap performance with Roaring bitmaps, -Software: Practice and Experience 46 (5), 2016.[arXiv:1402.6407](http://arxiv.org/abs/1402.6407) This paper used data from http://lemire.me/data/realroaring2014.html -- Daniel Lemire, Gregory Ssi-Yan-Kai, Owen Kaser, Consistently faster and smaller compressed bitmaps with Roaring, Software: Practice and Experience 46 (11), 2016. [arXiv:1603.06549](http://arxiv.org/abs/1603.06549) - -### Dependencies - -Dependencies are fetched automatically by giving the `-t` flag to `go get`. - -they include - - github.com/bits-and-blooms/bitset - - github.com/mschoch/smat - - github.com/glycerine/go-unsnap-stream - - github.com/philhofer/fwd - - github.com/jtolds/gls - -Note that the smat library requires Go 1.6 or better. - -#### Installation - - - go get -t github.com/RoaringBitmap/roaring - -### Instructions for contributors - -Using bash or other common shells: -``` -$ git clone git@github.com:RoaringBitmap/roaring.git -$ export GO111MODULE=on -$ go mod tidy -$ go test -v -``` - -### Example - -Here is a simplified but complete example: - -```go -package main - -import ( - "fmt" - "github.com/RoaringBitmap/roaring" - "bytes" -) - - -func main() { - // example inspired by https://github.com/fzandona/goroar - fmt.Println("==roaring==") - rb1 := roaring.BitmapOf(1, 2, 3, 4, 5, 100, 1000) - fmt.Println(rb1.String()) - - rb2 := roaring.BitmapOf(3, 4, 1000) - fmt.Println(rb2.String()) - - rb3 := roaring.New() - fmt.Println(rb3.String()) - - fmt.Println("Cardinality: ", rb1.GetCardinality()) - - fmt.Println("Contains 3? ", rb1.Contains(3)) - - rb1.And(rb2) - - rb3.Add(1) - rb3.Add(5) - - rb3.Or(rb1) - - // computes union of the three bitmaps in parallel using 4 workers - roaring.ParOr(4, rb1, rb2, rb3) - // computes intersection of the three bitmaps in parallel using 4 workers - roaring.ParAnd(4, rb1, rb2, rb3) - - - // prints 1, 3, 4, 5, 1000 - i := rb3.Iterator() - for i.HasNext() { - fmt.Println(i.Next()) - } - fmt.Println() - - // next we include an example of serialization - buf := new(bytes.Buffer) - rb1.WriteTo(buf) // we omit error handling - newrb:= roaring.New() - newrb.ReadFrom(buf) - if rb1.Equals(newrb) { - fmt.Println("I wrote the content to a byte stream and read it back.") - } - // you can iterate over bitmaps using ReverseIterator(), Iterator, ManyIterator() -} -``` - -If you wish to use serialization and handle errors, you might want to -consider the following sample of code: - -```go - rb := BitmapOf(1, 2, 3, 4, 5, 100, 1000) - buf := new(bytes.Buffer) - size,err:=rb.WriteTo(buf) - if err != nil { - t.Errorf("Failed writing") - } - newrb:= New() - size,err=newrb.ReadFrom(buf) - if err != nil { - t.Errorf("Failed reading") - } - if ! rb.Equals(newrb) { - t.Errorf("Cannot retrieve serialized version") - } -``` - -Given N integers in [0,x), then the serialized size in bytes of -a Roaring bitmap should never exceed this bound: - -`` 8 + 9 * ((long)x+65535)/65536 + 2 * N `` - -That is, given a fixed overhead for the universe size (x), Roaring -bitmaps never use more than 2 bytes per integer. You can call -``BoundSerializedSizeInBytes`` for a more precise estimate. - -### 64-bit Roaring - -By default, roaring is used to stored unsigned 32-bit integers. However, we also offer -an extension dedicated to 64-bit integers. It supports roughly the same functions: - -```go -package main - -import ( - "fmt" - "github.com/RoaringBitmap/roaring/roaring64" - "bytes" -) - - -func main() { - // example inspired by https://github.com/fzandona/goroar - fmt.Println("==roaring64==") - rb1 := roaring64.BitmapOf(1, 2, 3, 4, 5, 100, 1000) - fmt.Println(rb1.String()) - - rb2 := roaring64.BitmapOf(3, 4, 1000) - fmt.Println(rb2.String()) - - rb3 := roaring64.New() - fmt.Println(rb3.String()) - - fmt.Println("Cardinality: ", rb1.GetCardinality()) - - fmt.Println("Contains 3? ", rb1.Contains(3)) - - rb1.And(rb2) - - rb3.Add(1) - rb3.Add(5) - - rb3.Or(rb1) - - - - // prints 1, 3, 4, 5, 1000 - i := rb3.Iterator() - for i.HasNext() { - fmt.Println(i.Next()) - } - fmt.Println() - - // next we include an example of serialization - buf := new(bytes.Buffer) - rb1.WriteTo(buf) // we omit error handling - newrb:= roaring64.New() - newrb.ReadFrom(buf) - if rb1.Equals(newrb) { - fmt.Println("I wrote the content to a byte stream and read it back.") - } - // you can iterate over bitmaps using ReverseIterator(), Iterator, ManyIterator() -} -``` - -Only the 32-bit roaring format is standard and cross-operable between Java, C++, C and Go. There is no guarantee that the 64-bit versions are compatible. - -### Documentation - -Current documentation is available at https://pkg.go.dev/github.com/RoaringBitmap/roaring and https://pkg.go.dev/github.com/RoaringBitmap/roaring/roaring64 - -### Goroutine safety - -In general, it should not generally be considered safe to access -the same bitmaps using different goroutines--they are left -unsynchronized for performance. Should you want to access -a Bitmap from more than one goroutine, you should -provide synchronization. Typically this is done by using channels to pass -the *Bitmap around (in Go style; so there is only ever one owner), -or by using `sync.Mutex` to serialize operations on Bitmaps. - -### Coverage - -We test our software. For a report on our test coverage, see - -https://coveralls.io/github/RoaringBitmap/roaring?branch=master - -### Benchmark - -Type - - go test -bench Benchmark -run - - -To run benchmarks on [Real Roaring Datasets](https://github.com/RoaringBitmap/real-roaring-datasets) -run the following: - -```sh -go get github.com/RoaringBitmap/real-roaring-datasets -BENCH_REAL_DATA=1 go test -bench BenchmarkRealData -run - -``` - -### Iterative use - -You can use roaring with gore: - -- go get -u github.com/motemen/gore -- Make sure that ``$GOPATH/bin`` is in your ``$PATH``. -- go get github.com/RoaringBitmap/roaring - -```go -$ gore -gore version 0.2.6 :help for help -gore> :import github.com/RoaringBitmap/roaring -gore> x:=roaring.New() -gore> x.Add(1) -gore> x.String() -"{1}" -``` - - -### Fuzzy testing - -You can help us test further the library with fuzzy testing: - - go get github.com/dvyukov/go-fuzz/go-fuzz - go get github.com/dvyukov/go-fuzz/go-fuzz-build - go test -tags=gofuzz -run=TestGenerateSmatCorpus - go-fuzz-build github.com/RoaringBitmap/roaring - go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 -func FuzzSmat - -Let it run, and if the # of crashers is > 0, check out the reports in -the workdir where you should be able to find the panic goroutine stack -traces. - -You may also replace `-func FuzzSmat` by `-func FuzzSerializationBuffer` or `-func FuzzSerializationStream`. - -### Alternative in Go - -There is a Go version wrapping the C/C++ implementation https://github.com/RoaringBitmap/gocroaring - -For an alternative implementation in Go, see https://github.com/fzandona/goroar -The two versions were written independently. - - -### Mailing list/discussion group - -https://groups.google.com/forum/#!forum/roaring-bitmaps diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go deleted file mode 100644 index 80fa676ef6..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go +++ /dev/null @@ -1,1101 +0,0 @@ -package roaring - -import ( - "fmt" -) - -type arrayContainer struct { - content []uint16 -} - -func (ac *arrayContainer) String() string { - s := "{" - for it := ac.getShortIterator(); it.hasNext(); { - s += fmt.Sprintf("%v, ", it.next()) - } - return s + "}" -} - -func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int { - if i < 0 { - panic("negative index") - } - if len(ac.content) == 0 { - return i - } - _ = x[len(ac.content)-1+i] - _ = ac.content[len(ac.content)-1] - for k := 0; k < len(ac.content); k++ { - x[k+i] = - uint32(ac.content[k]) | mask - } - return i + len(ac.content) -} - -func (ac *arrayContainer) iterate(cb func(x uint16) bool) bool { - iterator := shortIterator{ac.content, 0} - - for iterator.hasNext() { - if !cb(iterator.next()) { - return false - } - } - - return true -} - -func (ac *arrayContainer) getShortIterator() shortPeekable { - return &shortIterator{ac.content, 0} -} - -func (ac *arrayContainer) getReverseIterator() shortIterable { - return &reverseIterator{ac.content, len(ac.content) - 1} -} - -func (ac *arrayContainer) getManyIterator() manyIterable { - return &shortIterator{ac.content, 0} -} - -func (ac *arrayContainer) minimum() uint16 { - return ac.content[0] // assume not empty -} - -func (ac *arrayContainer) maximum() uint16 { - return ac.content[len(ac.content)-1] // assume not empty -} - -func (ac *arrayContainer) getSizeInBytes() int { - return ac.getCardinality() * 2 -} - -func (ac *arrayContainer) serializedSizeInBytes() int { - return ac.getCardinality() * 2 -} - -func arrayContainerSizeInBytes(card int) int { - return card * 2 -} - -// add the values in the range [firstOfRange,endx) -func (ac *arrayContainer) iaddRange(firstOfRange, endx int) container { - if firstOfRange >= endx { - return ac - } - indexstart := binarySearch(ac.content, uint16(firstOfRange)) - if indexstart < 0 { - indexstart = -indexstart - 1 - } - indexend := binarySearch(ac.content, uint16(endx-1)) - if indexend < 0 { - indexend = -indexend - 1 - } else { - indexend++ - } - rangelength := endx - firstOfRange - newcardinality := indexstart + (ac.getCardinality() - indexend) + rangelength - if newcardinality > arrayDefaultMaxSize { - a := ac.toBitmapContainer() - return a.iaddRange(firstOfRange, endx) - } - if cap(ac.content) < newcardinality { - tmp := make([]uint16, newcardinality, newcardinality) - copy(tmp[:indexstart], ac.content[:indexstart]) - copy(tmp[indexstart+rangelength:], ac.content[indexend:]) - - ac.content = tmp - } else { - ac.content = ac.content[:newcardinality] - copy(ac.content[indexstart+rangelength:], ac.content[indexend:]) - - } - for k := 0; k < rangelength; k++ { - ac.content[k+indexstart] = uint16(firstOfRange + k) - } - return ac -} - -// remove the values in the range [firstOfRange,endx) -func (ac *arrayContainer) iremoveRange(firstOfRange, endx int) container { - if firstOfRange >= endx { - return ac - } - indexstart := binarySearch(ac.content, uint16(firstOfRange)) - if indexstart < 0 { - indexstart = -indexstart - 1 - } - indexend := binarySearch(ac.content, uint16(endx-1)) - if indexend < 0 { - indexend = -indexend - 1 - } else { - indexend++ - } - rangelength := indexend - indexstart - answer := ac - copy(answer.content[indexstart:], ac.content[indexstart+rangelength:]) - answer.content = answer.content[:ac.getCardinality()-rangelength] - return answer -} - -// flip the values in the range [firstOfRange,endx) -func (ac *arrayContainer) not(firstOfRange, endx int) container { - if firstOfRange >= endx { - return ac.clone() - } - return ac.notClose(firstOfRange, endx-1) // remove everything in [firstOfRange,endx-1] -} - -// flip the values in the range [firstOfRange,lastOfRange] -func (ac *arrayContainer) notClose(firstOfRange, lastOfRange int) container { - if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange] - return ac.clone() - } - - // determine the span of array indices to be affected^M - startIndex := binarySearch(ac.content, uint16(firstOfRange)) - if startIndex < 0 { - startIndex = -startIndex - 1 - } - lastIndex := binarySearch(ac.content, uint16(lastOfRange)) - if lastIndex < 0 { - lastIndex = -lastIndex - 2 - } - currentValuesInRange := lastIndex - startIndex + 1 - spanToBeFlipped := lastOfRange - firstOfRange + 1 - newValuesInRange := spanToBeFlipped - currentValuesInRange - cardinalityChange := newValuesInRange - currentValuesInRange - newCardinality := len(ac.content) + cardinalityChange - if newCardinality > arrayDefaultMaxSize { - return ac.toBitmapContainer().not(firstOfRange, lastOfRange+1) - } - answer := newArrayContainer() - answer.content = make([]uint16, newCardinality, newCardinality) //a hack for sure - - copy(answer.content, ac.content[:startIndex]) - outPos := startIndex - inPos := startIndex - valInRange := firstOfRange - for ; valInRange <= lastOfRange && inPos <= lastIndex; valInRange++ { - if uint16(valInRange) != ac.content[inPos] { - answer.content[outPos] = uint16(valInRange) - outPos++ - } else { - inPos++ - } - } - - for ; valInRange <= lastOfRange; valInRange++ { - answer.content[outPos] = uint16(valInRange) - outPos++ - } - - for i := lastIndex + 1; i < len(ac.content); i++ { - answer.content[outPos] = ac.content[i] - outPos++ - } - answer.content = answer.content[:newCardinality] - return answer - -} - -func (ac *arrayContainer) equals(o container) bool { - - srb, ok := o.(*arrayContainer) - if ok { - // Check if the containers are the same object. - if ac == srb { - return true - } - - if len(srb.content) != len(ac.content) { - return false - } - - for i, v := range ac.content { - if v != srb.content[i] { - return false - } - } - return true - } - - // use generic comparison - bCard := o.getCardinality() - aCard := ac.getCardinality() - if bCard != aCard { - return false - } - - ait := ac.getShortIterator() - bit := o.getShortIterator() - for ait.hasNext() { - if bit.next() != ait.next() { - return false - } - } - return true -} - -func (ac *arrayContainer) toBitmapContainer() *bitmapContainer { - bc := newBitmapContainer() - bc.loadData(ac) - return bc - -} -func (ac *arrayContainer) iadd(x uint16) (wasNew bool) { - // Special case adding to the end of the container. - l := len(ac.content) - if l > 0 && l < arrayDefaultMaxSize && ac.content[l-1] < x { - ac.content = append(ac.content, x) - return true - } - - loc := binarySearch(ac.content, x) - - if loc < 0 { - s := ac.content - i := -loc - 1 - s = append(s, 0) - copy(s[i+1:], s[i:]) - s[i] = x - ac.content = s - return true - } - return false -} - -func (ac *arrayContainer) iaddReturnMinimized(x uint16) container { - // Special case adding to the end of the container. - l := len(ac.content) - if l > 0 && l < arrayDefaultMaxSize && ac.content[l-1] < x { - ac.content = append(ac.content, x) - return ac - } - - loc := binarySearch(ac.content, x) - - if loc < 0 { - if len(ac.content) >= arrayDefaultMaxSize { - a := ac.toBitmapContainer() - a.iadd(x) - return a - } - s := ac.content - i := -loc - 1 - s = append(s, 0) - copy(s[i+1:], s[i:]) - s[i] = x - ac.content = s - } - return ac -} - -// iremoveReturnMinimized is allowed to change the return type to minimize storage. -func (ac *arrayContainer) iremoveReturnMinimized(x uint16) container { - ac.iremove(x) - return ac -} - -func (ac *arrayContainer) iremove(x uint16) bool { - loc := binarySearch(ac.content, x) - if loc >= 0 { - s := ac.content - s = append(s[:loc], s[loc+1:]...) - ac.content = s - return true - } - return false -} - -func (ac *arrayContainer) remove(x uint16) container { - out := &arrayContainer{make([]uint16, len(ac.content))} - copy(out.content, ac.content[:]) - - loc := binarySearch(out.content, x) - if loc >= 0 { - s := out.content - s = append(s[:loc], s[loc+1:]...) - out.content = s - } - return out -} - -func (ac *arrayContainer) or(a container) container { - switch x := a.(type) { - case *arrayContainer: - return ac.orArray(x) - case *bitmapContainer: - return x.orArray(ac) - case *runContainer16: - if x.isFull() { - return x.clone() - } - return x.orArray(ac) - } - panic("unsupported container type") -} - -func (ac *arrayContainer) orCardinality(a container) int { - switch x := a.(type) { - case *arrayContainer: - return ac.orArrayCardinality(x) - case *bitmapContainer: - return x.orArrayCardinality(ac) - case *runContainer16: - return x.orArrayCardinality(ac) - } - panic("unsupported container type") -} - -func (ac *arrayContainer) ior(a container) container { - switch x := a.(type) { - case *arrayContainer: - return ac.iorArray(x) - case *bitmapContainer: - return a.(*bitmapContainer).orArray(ac) - //return ac.iorBitmap(x) // note: this does not make sense - case *runContainer16: - if x.isFull() { - return x.clone() - } - return ac.iorRun16(x) - } - panic("unsupported container type") -} - -func (ac *arrayContainer) iorArray(value2 *arrayContainer) container { - value1 := ac - len1 := value1.getCardinality() - len2 := value2.getCardinality() - maxPossibleCardinality := len1 + len2 - if maxPossibleCardinality > cap(value1.content) { - // doubling the capacity reduces new slice allocations in the case of - // repeated calls to iorArray(). - newSize := 2 * maxPossibleCardinality - // the second check is to handle overly large array containers - // and should not occur in normal usage, - // as all array containers should be at most arrayDefaultMaxSize - if newSize > 2*arrayDefaultMaxSize && maxPossibleCardinality <= 2*arrayDefaultMaxSize { - newSize = 2 * arrayDefaultMaxSize - } - newcontent := make([]uint16, 0, newSize) - copy(newcontent[len2:maxPossibleCardinality], ac.content[0:len1]) - ac.content = newcontent - } else { - copy(ac.content[len2:maxPossibleCardinality], ac.content[0:len1]) - } - nl := union2by2(value1.content[len2:maxPossibleCardinality], value2.content, ac.content) - ac.content = ac.content[:nl] // reslice to match actual used capacity - - if nl > arrayDefaultMaxSize { - // Only converting to a bitmap when arrayDefaultMaxSize - // is actually exceeded minimizes conversions in the case of repeated - // calls to iorArray(). - return ac.toBitmapContainer() - } - return ac -} - -// Note: such code does not make practical sense, except for lazy evaluations -func (ac *arrayContainer) iorBitmap(bc2 *bitmapContainer) container { - bc1 := ac.toBitmapContainer() - bc1.iorBitmap(bc2) - *ac = *newArrayContainerFromBitmap(bc1) - return ac -} - -func (ac *arrayContainer) iorRun16(rc *runContainer16) container { - runCardinality := rc.getCardinality() - // heuristic for if the container should maybe be an - // array container. - if runCardinality < ac.getCardinality() && - runCardinality+ac.getCardinality() < arrayDefaultMaxSize { - var result container - result = ac - for _, run := range rc.iv { - result = result.iaddRange(int(run.start), int(run.start)+int(run.length)+1) - } - return result - } - return rc.orArray(ac) -} - -func (ac *arrayContainer) lazyIOR(a container) container { - switch x := a.(type) { - case *arrayContainer: - return ac.lazyIorArray(x) - case *bitmapContainer: - return ac.lazyIorBitmap(x) - case *runContainer16: - if x.isFull() { - return x.clone() - } - return ac.lazyIorRun16(x) - - } - panic("unsupported container type") -} - -func (ac *arrayContainer) lazyIorArray(ac2 *arrayContainer) container { - // TODO actually make this lazy - return ac.iorArray(ac2) -} - -func (ac *arrayContainer) lazyIorBitmap(bc *bitmapContainer) container { - // TODO actually make this lazy - return ac.iorBitmap(bc) -} - -func (ac *arrayContainer) lazyIorRun16(rc *runContainer16) container { - // TODO actually make this lazy - return ac.iorRun16(rc) -} - -func (ac *arrayContainer) lazyOR(a container) container { - switch x := a.(type) { - case *arrayContainer: - return ac.lazyorArray(x) - case *bitmapContainer: - return a.lazyOR(ac) - case *runContainer16: - if x.isFull() { - return x.clone() - } - return x.orArray(ac) - } - panic("unsupported container type") -} - -func (ac *arrayContainer) orArray(value2 *arrayContainer) container { - value1 := ac - maxPossibleCardinality := value1.getCardinality() + value2.getCardinality() - if maxPossibleCardinality > arrayDefaultMaxSize { // it could be a bitmap! - bc := newBitmapContainer() - for k := 0; k < len(value2.content); k++ { - v := value2.content[k] - i := uint(v) >> 6 - mask := uint64(1) << (v % 64) - bc.bitmap[i] |= mask - } - for k := 0; k < len(ac.content); k++ { - v := ac.content[k] - i := uint(v) >> 6 - mask := uint64(1) << (v % 64) - bc.bitmap[i] |= mask - } - bc.cardinality = int(popcntSlice(bc.bitmap)) - if bc.cardinality <= arrayDefaultMaxSize { - return bc.toArrayContainer() - } - return bc - } - answer := newArrayContainerCapacity(maxPossibleCardinality) - nl := union2by2(value1.content, value2.content, answer.content) - answer.content = answer.content[:nl] // reslice to match actual used capacity - return answer -} - -func (ac *arrayContainer) orArrayCardinality(value2 *arrayContainer) int { - return union2by2Cardinality(ac.content, value2.content) -} - -func (ac *arrayContainer) lazyorArray(value2 *arrayContainer) container { - value1 := ac - maxPossibleCardinality := value1.getCardinality() + value2.getCardinality() - if maxPossibleCardinality > arrayLazyLowerBound { // it could be a bitmap! - bc := newBitmapContainer() - for k := 0; k < len(value2.content); k++ { - v := value2.content[k] - i := uint(v) >> 6 - mask := uint64(1) << (v % 64) - bc.bitmap[i] |= mask - } - for k := 0; k < len(ac.content); k++ { - v := ac.content[k] - i := uint(v) >> 6 - mask := uint64(1) << (v % 64) - bc.bitmap[i] |= mask - } - bc.cardinality = invalidCardinality - return bc - } - answer := newArrayContainerCapacity(maxPossibleCardinality) - nl := union2by2(value1.content, value2.content, answer.content) - answer.content = answer.content[:nl] // reslice to match actual used capacity - return answer -} - -func (ac *arrayContainer) and(a container) container { - switch x := a.(type) { - case *arrayContainer: - return ac.andArray(x) - case *bitmapContainer: - return x.and(ac) - case *runContainer16: - if x.isFull() { - return ac.clone() - } - return x.andArray(ac) - } - panic("unsupported container type") -} - -func (ac *arrayContainer) andCardinality(a container) int { - switch x := a.(type) { - case *arrayContainer: - return ac.andArrayCardinality(x) - case *bitmapContainer: - return x.andCardinality(ac) - case *runContainer16: - return x.andArrayCardinality(ac) - } - panic("unsupported container type") -} - -func (ac *arrayContainer) intersects(a container) bool { - switch x := a.(type) { - case *arrayContainer: - return ac.intersectsArray(x) - case *bitmapContainer: - return x.intersects(ac) - case *runContainer16: - return x.intersects(ac) - } - panic("unsupported container type") -} - -func (ac *arrayContainer) iand(a container) container { - switch x := a.(type) { - case *arrayContainer: - return ac.iandArray(x) - case *bitmapContainer: - return ac.iandBitmap(x) - case *runContainer16: - if x.isFull() { - return ac - } - return x.andArray(ac) - } - panic("unsupported container type") -} - -func (ac *arrayContainer) iandBitmap(bc *bitmapContainer) container { - pos := 0 - c := ac.getCardinality() - for k := 0; k < c; k++ { - // branchless - v := ac.content[k] - ac.content[pos] = v - pos += int(bc.bitValue(v)) - } - ac.content = ac.content[:pos] - return ac - -} - -func (ac *arrayContainer) xor(a container) container { - switch x := a.(type) { - case *arrayContainer: - return ac.xorArray(x) - case *bitmapContainer: - return a.xor(ac) - case *runContainer16: - return x.xorArray(ac) - } - panic("unsupported container type") -} - -func (ac *arrayContainer) xorArray(value2 *arrayContainer) container { - value1 := ac - totalCardinality := value1.getCardinality() + value2.getCardinality() - if totalCardinality > arrayDefaultMaxSize { // it could be a bitmap! - bc := newBitmapContainer() - for k := 0; k < len(value2.content); k++ { - v := value2.content[k] - i := uint(v) >> 6 - bc.bitmap[i] ^= (uint64(1) << (v % 64)) - } - for k := 0; k < len(ac.content); k++ { - v := ac.content[k] - i := uint(v) >> 6 - bc.bitmap[i] ^= (uint64(1) << (v % 64)) - } - bc.computeCardinality() - if bc.cardinality <= arrayDefaultMaxSize { - return bc.toArrayContainer() - } - return bc - } - desiredCapacity := totalCardinality - answer := newArrayContainerCapacity(desiredCapacity) - length := exclusiveUnion2by2(value1.content, value2.content, answer.content) - answer.content = answer.content[:length] - return answer - -} - -func (ac *arrayContainer) andNot(a container) container { - switch x := a.(type) { - case *arrayContainer: - return ac.andNotArray(x) - case *bitmapContainer: - return ac.andNotBitmap(x) - case *runContainer16: - return ac.andNotRun16(x) - } - panic("unsupported container type") -} - -func (ac *arrayContainer) andNotRun16(rc *runContainer16) container { - acb := ac.toBitmapContainer() - rcb := rc.toBitmapContainer() - return acb.andNotBitmap(rcb) -} - -func (ac *arrayContainer) iandNot(a container) container { - switch x := a.(type) { - case *arrayContainer: - return ac.iandNotArray(x) - case *bitmapContainer: - return ac.iandNotBitmap(x) - case *runContainer16: - return ac.iandNotRun16(x) - } - panic("unsupported container type") -} - -func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container { - // Fast path: if either the array container or the run container is empty, the result is the array. - if ac.isEmpty() || rc.isEmpty() { - // Empty - return ac - } - // Fast path: if the run container is full, the result is empty. - if rc.isFull() { - ac.content = ac.content[:0] - return ac - } - current_run := 0 - // All values in [start_run, end_end] are part of the run - start_run := rc.iv[current_run].start - end_end := start_run + rc.iv[current_run].length - // We are going to read values in the array at index i, and we are - // going to write them at index pos. So we do in-place processing. - // We always have that pos <= i by construction. So we can either - // overwrite a value just read, or a value that was previous read. - pos := 0 - i := 0 - for ; i < len(ac.content); i++ { - if ac.content[i] < start_run { - // the value in the array appears before the run [start_run, end_end] - ac.content[pos] = ac.content[i] - pos++ - } else if ac.content[i] <= end_end { - // nothing to do, the value is in the array but also in the run. - } else { - // We have the value in the array after the run. We cannot tell - // whether we need to keep it or not. So let us move to another run. - if current_run+1 < len(rc.iv) { - current_run++ - start_run = rc.iv[current_run].start - end_end = start_run + rc.iv[current_run].length - i-- // retry with the same i - } else { - // We have exhausted the number of runs. We can keep the rest of the values - // from i to len(ac.content) - 1 inclusively. - break // We are done, the rest of the array will be kept - } - } - } - for ; i < len(ac.content); i++ { - ac.content[pos] = ac.content[i] - pos++ - } - // We 'shink' the slice. - ac.content = ac.content[:pos] - return ac -} - -func (ac *arrayContainer) andNotArray(value2 *arrayContainer) container { - value1 := ac - desiredcapacity := value1.getCardinality() - answer := newArrayContainerCapacity(desiredcapacity) - length := difference(value1.content, value2.content, answer.content) - answer.content = answer.content[:length] - return answer -} - -func (ac *arrayContainer) iandNotArray(value2 *arrayContainer) container { - length := difference(ac.content, value2.content, ac.content) - ac.content = ac.content[:length] - return ac -} - -func (ac *arrayContainer) andNotBitmap(value2 *bitmapContainer) container { - desiredcapacity := ac.getCardinality() - answer := newArrayContainerCapacity(desiredcapacity) - answer.content = answer.content[:desiredcapacity] - pos := 0 - for _, v := range ac.content { - answer.content[pos] = v - pos += 1 - int(value2.bitValue(v)) - } - answer.content = answer.content[:pos] - return answer -} - -func (ac *arrayContainer) andBitmap(value2 *bitmapContainer) container { - desiredcapacity := ac.getCardinality() - answer := newArrayContainerCapacity(desiredcapacity) - answer.content = answer.content[:desiredcapacity] - pos := 0 - for _, v := range ac.content { - answer.content[pos] = v - pos += int(value2.bitValue(v)) - } - answer.content = answer.content[:pos] - return answer -} - -func (ac *arrayContainer) iandNotBitmap(value2 *bitmapContainer) container { - pos := 0 - for _, v := range ac.content { - ac.content[pos] = v - pos += 1 - int(value2.bitValue(v)) - } - ac.content = ac.content[:pos] - return ac -} - -func copyOf(array []uint16, size int) []uint16 { - result := make([]uint16, size) - for i, x := range array { - if i == size { - break - } - result[i] = x - } - return result -} - -// flip the values in the range [firstOfRange,endx) -func (ac *arrayContainer) inot(firstOfRange, endx int) container { - if firstOfRange >= endx { - return ac - } - return ac.inotClose(firstOfRange, endx-1) // remove everything in [firstOfRange,endx-1] -} - -// flip the values in the range [firstOfRange,lastOfRange] -func (ac *arrayContainer) inotClose(firstOfRange, lastOfRange int) container { - if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange] - return ac - } - // determine the span of array indices to be affected - startIndex := binarySearch(ac.content, uint16(firstOfRange)) - if startIndex < 0 { - startIndex = -startIndex - 1 - } - lastIndex := binarySearch(ac.content, uint16(lastOfRange)) - if lastIndex < 0 { - lastIndex = -lastIndex - 1 - 1 - } - currentValuesInRange := lastIndex - startIndex + 1 - spanToBeFlipped := lastOfRange - firstOfRange + 1 - - newValuesInRange := spanToBeFlipped - currentValuesInRange - buffer := make([]uint16, newValuesInRange) - cardinalityChange := newValuesInRange - currentValuesInRange - newCardinality := len(ac.content) + cardinalityChange - if cardinalityChange > 0 { - if newCardinality > len(ac.content) { - if newCardinality > arrayDefaultMaxSize { - bcRet := ac.toBitmapContainer() - bcRet.inot(firstOfRange, lastOfRange+1) - *ac = *bcRet.toArrayContainer() - return bcRet - } - ac.content = copyOf(ac.content, newCardinality) - } - base := lastIndex + 1 - copy(ac.content[lastIndex+1+cardinalityChange:], ac.content[base:base+len(ac.content)-1-lastIndex]) - ac.negateRange(buffer, startIndex, lastIndex, firstOfRange, lastOfRange+1) - } else { // no expansion needed - ac.negateRange(buffer, startIndex, lastIndex, firstOfRange, lastOfRange+1) - if cardinalityChange < 0 { - - for i := startIndex + newValuesInRange; i < newCardinality; i++ { - ac.content[i] = ac.content[i-cardinalityChange] - } - } - } - ac.content = ac.content[:newCardinality] - return ac -} - -func (ac *arrayContainer) negateRange(buffer []uint16, startIndex, lastIndex, startRange, lastRange int) { - // compute the negation into buffer - outPos := 0 - inPos := startIndex // value here always >= valInRange, - // until it is exhausted - // n.b., we can start initially exhausted. - - valInRange := startRange - for ; valInRange < lastRange && inPos <= lastIndex; valInRange++ { - if uint16(valInRange) != ac.content[inPos] { - buffer[outPos] = uint16(valInRange) - outPos++ - } else { - inPos++ - } - } - - // if there are extra items (greater than the biggest - // pre-existing one in range), buffer them - for ; valInRange < lastRange; valInRange++ { - buffer[outPos] = uint16(valInRange) - outPos++ - } - - if outPos != len(buffer) { - panic("negateRange: internal bug") - } - - for i, item := range buffer { - ac.content[i+startIndex] = item - } -} - -func (ac *arrayContainer) isFull() bool { - return false -} - -func (ac *arrayContainer) andArray(value2 *arrayContainer) container { - desiredcapacity := minOfInt(ac.getCardinality(), value2.getCardinality()) - answer := newArrayContainerCapacity(desiredcapacity) - length := intersection2by2( - ac.content, - value2.content, - answer.content) - answer.content = answer.content[:length] - return answer -} - -func (ac *arrayContainer) andArrayCardinality(value2 *arrayContainer) int { - return intersection2by2Cardinality( - ac.content, - value2.content) -} - -func (ac *arrayContainer) intersectsArray(value2 *arrayContainer) bool { - return intersects2by2( - ac.content, - value2.content) -} - -func (ac *arrayContainer) iandArray(value2 *arrayContainer) container { - length := intersection2by2( - ac.content, - value2.content, - ac.content) - ac.content = ac.content[:length] - return ac -} - -func (ac *arrayContainer) getCardinality() int { - return len(ac.content) -} - -func (ac *arrayContainer) isEmpty() bool { - return len(ac.content) == 0 -} - -func (ac *arrayContainer) rank(x uint16) int { - answer := binarySearch(ac.content, x) - if answer >= 0 { - return answer + 1 - } - return -answer - 1 - -} - -func (ac *arrayContainer) selectInt(x uint16) int { - return int(ac.content[x]) -} - -func (ac *arrayContainer) clone() container { - ptr := arrayContainer{make([]uint16, len(ac.content))} - copy(ptr.content, ac.content[:]) - return &ptr -} - -func (ac *arrayContainer) contains(x uint16) bool { - return binarySearch(ac.content, x) >= 0 -} - -func (ac *arrayContainer) loadData(bitmapContainer *bitmapContainer) { - ac.content = make([]uint16, bitmapContainer.cardinality, bitmapContainer.cardinality) - bitmapContainer.fillArray(ac.content) -} - -func (ac *arrayContainer) resetTo(a container) { - switch x := a.(type) { - case *arrayContainer: - ac.realloc(len(x.content)) - copy(ac.content, x.content) - - case *bitmapContainer: - ac.realloc(x.cardinality) - x.fillArray(ac.content) - - case *runContainer16: - card := int(x.getCardinality()) - ac.realloc(card) - cur := 0 - for _, r := range x.iv { - for val := r.start; val <= r.last(); val++ { - ac.content[cur] = val - cur++ - } - } - - default: - panic("unsupported container type") - } -} - -func (ac *arrayContainer) realloc(size int) { - if cap(ac.content) < size { - ac.content = make([]uint16, size) - } else { - ac.content = ac.content[:size] - } -} - -func newArrayContainer() *arrayContainer { - p := new(arrayContainer) - return p -} - -func newArrayContainerFromBitmap(bc *bitmapContainer) *arrayContainer { - ac := &arrayContainer{} - ac.loadData(bc) - return ac -} - -func newArrayContainerCapacity(size int) *arrayContainer { - p := new(arrayContainer) - p.content = make([]uint16, 0, size) - return p -} - -func newArrayContainerSize(size int) *arrayContainer { - p := new(arrayContainer) - p.content = make([]uint16, size, size) - return p -} - -func newArrayContainerRange(firstOfRun, lastOfRun int) *arrayContainer { - valuesInRange := lastOfRun - firstOfRun + 1 - this := newArrayContainerCapacity(valuesInRange) - for i := 0; i < valuesInRange; i++ { - this.content = append(this.content, uint16(firstOfRun+i)) - } - return this -} - -func (ac *arrayContainer) numberOfRuns() (nr int) { - n := len(ac.content) - var runlen uint16 - var cur, prev uint16 - - switch n { - case 0: - return 0 - case 1: - return 1 - default: - for i := 1; i < n; i++ { - prev = ac.content[i-1] - cur = ac.content[i] - - if cur == prev+1 { - runlen++ - } else { - if cur < prev { - panic("the fundamental arrayContainer assumption of sorted ac.content was broken") - } - if cur == prev { - panic("the fundamental arrayContainer assumption of deduplicated content was broken") - } else { - nr++ - runlen = 0 - } - } - } - nr++ - } - return -} - -// convert to run or array *if needed* -func (ac *arrayContainer) toEfficientContainer() container { - - numRuns := ac.numberOfRuns() - - sizeAsRunContainer := runContainer16SerializedSizeInBytes(numRuns) - sizeAsBitmapContainer := bitmapContainerSizeInBytes() - card := ac.getCardinality() - sizeAsArrayContainer := arrayContainerSizeInBytes(card) - - if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) { - return newRunContainer16FromArray(ac) - } - if card <= arrayDefaultMaxSize { - return ac - } - return ac.toBitmapContainer() -} - -func (ac *arrayContainer) containerType() contype { - return arrayContype -} - -func (ac *arrayContainer) addOffset(x uint16) (container, container) { - var low, high *arrayContainer - - if len(ac.content) == 0 { - return nil, nil - } - - if y := uint32(ac.content[0]) + uint32(x); highbits(y) == 0 { - // Some elements will fall into low part, allocate a container. - // Checking the first one is enough because they are ordered. - low = &arrayContainer{} - } - if y := uint32(ac.content[len(ac.content)-1]) + uint32(x); highbits(y) > 0 { - // Some elements will fall into high part, allocate a container. - // Checking the last one is enough because they are ordered. - high = &arrayContainer{} - } - - for _, val := range ac.content { - y := uint32(val) + uint32(x) - if highbits(y) > 0 { - // OK, if high == nil then highbits(y) == 0 for all y. - high.content = append(high.content, lowbits(y)) - } else { - // OK, if low == nil then highbits(y) > 0 for all y. - low.content = append(low.content, lowbits(y)) - } - } - - // Ensure proper nil interface. - if low == nil { - return nil, high - } - if high == nil { - return low, nil - } - - return low, high -} diff --git a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go deleted file mode 100644 index bf08bfca3a..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go +++ /dev/null @@ -1,1236 +0,0 @@ -package roaring - -import ( - "fmt" - "unsafe" -) - -type bitmapContainer struct { - cardinality int - bitmap []uint64 -} - -func (bc bitmapContainer) String() string { - var s string - for it := bc.getShortIterator(); it.hasNext(); { - s += fmt.Sprintf("%v, ", it.next()) - } - return s -} - -func newBitmapContainer() *bitmapContainer { - p := new(bitmapContainer) - size := (1 << 16) / 64 - p.bitmap = make([]uint64, size, size) - return p -} - -func newBitmapContainerwithRange(firstOfRun, lastOfRun int) *bitmapContainer { - bc := newBitmapContainer() - bc.cardinality = lastOfRun - firstOfRun + 1 - if bc.cardinality == maxCapacity { - fill(bc.bitmap, uint64(0xffffffffffffffff)) - } else { - firstWord := firstOfRun / 64 - lastWord := lastOfRun / 64 - zeroPrefixLength := uint64(firstOfRun & 63) - zeroSuffixLength := uint64(63 - (lastOfRun & 63)) - - fillRange(bc.bitmap, firstWord, lastWord+1, uint64(0xffffffffffffffff)) - bc.bitmap[firstWord] ^= ((uint64(1) << zeroPrefixLength) - 1) - blockOfOnes := (uint64(1) << zeroSuffixLength) - 1 - maskOnLeft := blockOfOnes << (uint64(64) - zeroSuffixLength) - bc.bitmap[lastWord] ^= maskOnLeft - } - return bc -} - -func (bc *bitmapContainer) minimum() uint16 { - for i := 0; i < len(bc.bitmap); i++ { - w := bc.bitmap[i] - if w != 0 { - r := countTrailingZeros(w) - return uint16(r + i*64) - } - } - return MaxUint16 -} - -// i should be non-zero -func clz(i uint64) int { - n := 1 - x := uint32(i >> 32) - if x == 0 { - n += 32 - x = uint32(i) - } - if x>>16 == 0 { - n += 16 - x = x << 16 - } - if x>>24 == 0 { - n += 8 - x = x << 8 - } - if x>>28 == 0 { - n += 4 - x = x << 4 - } - if x>>30 == 0 { - n += 2 - x = x << 2 - } - return n - int(x>>31) -} - -func (bc *bitmapContainer) maximum() uint16 { - for i := len(bc.bitmap); i > 0; i-- { - w := bc.bitmap[i-1] - if w != 0 { - r := clz(w) - return uint16((i-1)*64 + 63 - r) - } - } - return uint16(0) -} - -func (bc *bitmapContainer) iterate(cb func(x uint16) bool) bool { - iterator := bitmapContainerShortIterator{bc, bc.NextSetBit(0)} - - for iterator.hasNext() { - if !cb(iterator.next()) { - return false - } - } - - return true -} - -type bitmapContainerShortIterator struct { - ptr *bitmapContainer - i int -} - -func (bcsi *bitmapContainerShortIterator) next() uint16 { - j := bcsi.i - bcsi.i = bcsi.ptr.NextSetBit(uint(bcsi.i) + 1) - return uint16(j) -} -func (bcsi *bitmapContainerShortIterator) hasNext() bool { - return bcsi.i >= 0 -} - -func (bcsi *bitmapContainerShortIterator) peekNext() uint16 { - return uint16(bcsi.i) -} - -func (bcsi *bitmapContainerShortIterator) advanceIfNeeded(minval uint16) { - if bcsi.hasNext() && bcsi.peekNext() < minval { - bcsi.i = bcsi.ptr.NextSetBit(uint(minval)) - } -} - -func newBitmapContainerShortIterator(a *bitmapContainer) *bitmapContainerShortIterator { - return &bitmapContainerShortIterator{a, a.NextSetBit(0)} -} - -func (bc *bitmapContainer) getShortIterator() shortPeekable { - return newBitmapContainerShortIterator(bc) -} - -type reverseBitmapContainerShortIterator struct { - ptr *bitmapContainer - i int -} - -func (bcsi *reverseBitmapContainerShortIterator) next() uint16 { - if bcsi.i == -1 { - panic("reverseBitmapContainerShortIterator.next() going beyond what is available") - } - - j := bcsi.i - bcsi.i = bcsi.ptr.PrevSetBit(bcsi.i - 1) - return uint16(j) -} - -func (bcsi *reverseBitmapContainerShortIterator) hasNext() bool { - return bcsi.i >= 0 -} - -func newReverseBitmapContainerShortIterator(a *bitmapContainer) *reverseBitmapContainerShortIterator { - if a.cardinality == 0 { - return &reverseBitmapContainerShortIterator{a, -1} - } - return &reverseBitmapContainerShortIterator{a, int(a.maximum())} -} - -func (bc *bitmapContainer) getReverseIterator() shortIterable { - return newReverseBitmapContainerShortIterator(bc) -} - -type bitmapContainerManyIterator struct { - ptr *bitmapContainer - base int - bitset uint64 -} - -func (bcmi *bitmapContainerManyIterator) nextMany(hs uint32, buf []uint32) int { - n := 0 - base := bcmi.base - bitset := bcmi.bitset - - for n < len(buf) { - if bitset == 0 { - base++ - if base >= len(bcmi.ptr.bitmap) { - bcmi.base = base - bcmi.bitset = bitset - return n - } - bitset = bcmi.ptr.bitmap[base] - continue - } - t := bitset & -bitset - buf[n] = uint32(((base * 64) + int(popcount(t-1)))) | hs - n = n + 1 - bitset ^= t - } - - bcmi.base = base - bcmi.bitset = bitset - return n -} - -func (bcmi *bitmapContainerManyIterator) nextMany64(hs uint64, buf []uint64) int { - n := 0 - base := bcmi.base - bitset := bcmi.bitset - - for n < len(buf) { - if bitset == 0 { - base++ - if base >= len(bcmi.ptr.bitmap) { - bcmi.base = base - bcmi.bitset = bitset - return n - } - bitset = bcmi.ptr.bitmap[base] - continue - } - t := bitset & -bitset - buf[n] = uint64(((base * 64) + int(popcount(t-1)))) | hs - n = n + 1 - bitset ^= t - } - - bcmi.base = base - bcmi.bitset = bitset - return n -} - -func newBitmapContainerManyIterator(a *bitmapContainer) *bitmapContainerManyIterator { - return &bitmapContainerManyIterator{a, -1, 0} -} - -func (bc *bitmapContainer) getManyIterator() manyIterable { - return newBitmapContainerManyIterator(bc) -} - -func (bc *bitmapContainer) getSizeInBytes() int { - return len(bc.bitmap) * 8 // + bcBaseBytes -} - -func (bc *bitmapContainer) serializedSizeInBytes() int { - //return bc.Msgsize()// NOO! This breaks GetSerializedSizeInBytes - return len(bc.bitmap) * 8 -} - -const bcBaseBytes = int(unsafe.Sizeof(bitmapContainer{})) - -// bitmapContainer doesn't depend on card, always fully allocated -func bitmapContainerSizeInBytes() int { - return bcBaseBytes + (1<<16)/8 -} - -func bitmapEquals(a, b []uint64) bool { - if len(a) != len(b) { - return false - } - for i, v := range a { - if v != b[i] { - return false - } - } - return true -} - -func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int { - // TODO: should be written as optimized assembly - pos := i - base := mask - for k := 0; k < len(bc.bitmap); k++ { - bitset := bc.bitmap[k] - for bitset != 0 { - t := bitset & -bitset - x[pos] = base + uint32(popcount(t-1)) - pos++ - bitset ^= t - } - base += 64 - } - return pos -} - -func (bc *bitmapContainer) equals(o container) bool { - srb, ok := o.(*bitmapContainer) - if ok { - if srb.cardinality != bc.cardinality { - return false - } - return bitmapEquals(bc.bitmap, srb.bitmap) - } - - // use generic comparison - if bc.getCardinality() != o.getCardinality() { - return false - } - ait := o.getShortIterator() - bit := bc.getShortIterator() - - for ait.hasNext() { - if bit.next() != ait.next() { - return false - } - } - return true -} - -func (bc *bitmapContainer) iaddReturnMinimized(i uint16) container { - bc.iadd(i) - if bc.isFull() { - return newRunContainer16Range(0, MaxUint16) - } - return bc -} - -func (bc *bitmapContainer) iadd(i uint16) bool { - x := int(i) - previous := bc.bitmap[x/64] - mask := uint64(1) << (uint(x) % 64) - newb := previous | mask - bc.bitmap[x/64] = newb - bc.cardinality += int((previous ^ newb) >> (uint(x) % 64)) - return newb != previous -} - -func (bc *bitmapContainer) iremoveReturnMinimized(i uint16) container { - if bc.iremove(i) { - if bc.cardinality == arrayDefaultMaxSize { - return bc.toArrayContainer() - } - } - return bc -} - -// iremove returns true if i was found. -func (bc *bitmapContainer) iremove(i uint16) bool { - if bc.contains(i) { - bc.cardinality-- - bc.bitmap[i/64] &^= (uint64(1) << (i % 64)) - return true - } - return false -} - -func (bc *bitmapContainer) isFull() bool { - return bc.cardinality == int(MaxUint16)+1 -} - -func (bc *bitmapContainer) getCardinality() int { - return bc.cardinality -} - -func (bc *bitmapContainer) isEmpty() bool { - return bc.cardinality == 0 -} - -func (bc *bitmapContainer) clone() container { - ptr := bitmapContainer{bc.cardinality, make([]uint64, len(bc.bitmap))} - copy(ptr.bitmap, bc.bitmap[:]) - return &ptr -} - -// add all values in range [firstOfRange,lastOfRange) -func (bc *bitmapContainer) iaddRange(firstOfRange, lastOfRange int) container { - bc.cardinality += setBitmapRangeAndCardinalityChange(bc.bitmap, firstOfRange, lastOfRange) - return bc -} - -// remove all values in range [firstOfRange,lastOfRange) -func (bc *bitmapContainer) iremoveRange(firstOfRange, lastOfRange int) container { - bc.cardinality += resetBitmapRangeAndCardinalityChange(bc.bitmap, firstOfRange, lastOfRange) - if bc.getCardinality() <= arrayDefaultMaxSize { - return bc.toArrayContainer() - } - return bc -} - -// flip all values in range [firstOfRange,endx) -func (bc *bitmapContainer) inot(firstOfRange, endx int) container { - if endx-firstOfRange == maxCapacity { - flipBitmapRange(bc.bitmap, firstOfRange, endx) - bc.cardinality = maxCapacity - bc.cardinality - } else if endx-firstOfRange > maxCapacity/2 { - flipBitmapRange(bc.bitmap, firstOfRange, endx) - bc.computeCardinality() - } else { - bc.cardinality += flipBitmapRangeAndCardinalityChange(bc.bitmap, firstOfRange, endx) - } - if bc.getCardinality() <= arrayDefaultMaxSize { - return bc.toArrayContainer() - } - return bc -} - -// flip all values in range [firstOfRange,endx) -func (bc *bitmapContainer) not(firstOfRange, endx int) container { - answer := bc.clone() - return answer.inot(firstOfRange, endx) -} - -func (bc *bitmapContainer) or(a container) container { - switch x := a.(type) { - case *arrayContainer: - return bc.orArray(x) - case *bitmapContainer: - return bc.orBitmap(x) - case *runContainer16: - if x.isFull() { - return x.clone() - } - return x.orBitmapContainer(bc) - } - panic("unsupported container type") -} - -func (bc *bitmapContainer) orCardinality(a container) int { - switch x := a.(type) { - case *arrayContainer: - return bc.orArrayCardinality(x) - case *bitmapContainer: - return bc.orBitmapCardinality(x) - case *runContainer16: - return x.orBitmapContainerCardinality(bc) - } - panic("unsupported container type") -} - -func (bc *bitmapContainer) ior(a container) container { - switch x := a.(type) { - case *arrayContainer: - return bc.iorArray(x) - case *bitmapContainer: - return bc.iorBitmap(x) - case *runContainer16: - if x.isFull() { - return x.clone() - } - for i := range x.iv { - bc.iaddRange(int(x.iv[i].start), int(x.iv[i].last())+1) - } - if bc.isFull() { - return newRunContainer16Range(0, MaxUint16) - } - //bc.computeCardinality() - return bc - } - panic(fmt.Errorf("unsupported container type %T", a)) -} - -func (bc *bitmapContainer) lazyIOR(a container) container { - switch x := a.(type) { - case *arrayContainer: - return bc.lazyIORArray(x) - case *bitmapContainer: - return bc.lazyIORBitmap(x) - case *runContainer16: - if x.isFull() { - return x.clone() - } - - // Manually inlined setBitmapRange function - bitmap := bc.bitmap - for _, iv := range x.iv { - start := int(iv.start) - end := int(iv.last()) + 1 - if start >= end { - continue - } - firstword := start / 64 - endword := (end - 1) / 64 - if firstword == endword { - bitmap[firstword] |= (^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64)) - continue - } - bitmap[firstword] |= ^uint64(0) << uint(start%64) - for i := firstword + 1; i < endword; i++ { - bitmap[i] = ^uint64(0) - } - bitmap[endword] |= ^uint64(0) >> (uint(-end) % 64) - } - bc.cardinality = invalidCardinality - return bc - } - panic("unsupported container type") -} - -func (bc *bitmapContainer) lazyOR(a container) container { - switch x := a.(type) { - case *arrayContainer: - return bc.lazyORArray(x) - case *bitmapContainer: - return bc.lazyORBitmap(x) - case *runContainer16: - if x.isFull() { - return x.clone() - } - // TODO: implement lazy OR - return x.orBitmapContainer(bc) - - } - panic("unsupported container type") -} - -func (bc *bitmapContainer) orArray(value2 *arrayContainer) container { - answer := bc.clone().(*bitmapContainer) - c := value2.getCardinality() - for k := 0; k < c; k++ { - v := value2.content[k] - i := uint(v) >> 6 - bef := answer.bitmap[i] - aft := bef | (uint64(1) << (v % 64)) - answer.bitmap[i] = aft - answer.cardinality += int((bef - aft) >> 63) - } - return answer -} - -func (bc *bitmapContainer) orArrayCardinality(value2 *arrayContainer) int { - answer := 0 - c := value2.getCardinality() - for k := 0; k < c; k++ { - // branchless: - v := value2.content[k] - i := uint(v) >> 6 - bef := bc.bitmap[i] - aft := bef | (uint64(1) << (v % 64)) - answer += int((bef - aft) >> 63) - } - return answer -} - -func (bc *bitmapContainer) orBitmap(value2 *bitmapContainer) container { - answer := newBitmapContainer() - for k := 0; k < len(answer.bitmap); k++ { - answer.bitmap[k] = bc.bitmap[k] | value2.bitmap[k] - } - answer.computeCardinality() - if answer.isFull() { - return newRunContainer16Range(0, MaxUint16) - } - return answer -} - -func (bc *bitmapContainer) orBitmapCardinality(value2 *bitmapContainer) int { - return int(popcntOrSlice(bc.bitmap, value2.bitmap)) -} - -func (bc *bitmapContainer) andBitmapCardinality(value2 *bitmapContainer) int { - return int(popcntAndSlice(bc.bitmap, value2.bitmap)) -} - -func (bc *bitmapContainer) computeCardinality() { - bc.cardinality = int(popcntSlice(bc.bitmap)) -} - -func (bc *bitmapContainer) iorArray(ac *arrayContainer) container { - for k := range ac.content { - vc := ac.content[k] - i := uint(vc) >> 6 - bef := bc.bitmap[i] - aft := bef | (uint64(1) << (vc % 64)) - bc.bitmap[i] = aft - bc.cardinality += int((bef - aft) >> 63) - } - if bc.isFull() { - return newRunContainer16Range(0, MaxUint16) - } - return bc -} - -func (bc *bitmapContainer) iorBitmap(value2 *bitmapContainer) container { - answer := bc - answer.cardinality = 0 - for k := 0; k < len(answer.bitmap); k++ { - answer.bitmap[k] = bc.bitmap[k] | value2.bitmap[k] - } - answer.computeCardinality() - if bc.isFull() { - return newRunContainer16Range(0, MaxUint16) - } - return answer -} - -func (bc *bitmapContainer) lazyIORArray(value2 *arrayContainer) container { - answer := bc - c := value2.getCardinality() - for k := 0; k+3 < c; k += 4 { - content := (*[4]uint16)(unsafe.Pointer(&value2.content[k])) - vc0 := content[0] - i0 := uint(vc0) >> 6 - answer.bitmap[i0] = answer.bitmap[i0] | (uint64(1) << (vc0 % 64)) - - vc1 := content[1] - i1 := uint(vc1) >> 6 - answer.bitmap[i1] = answer.bitmap[i1] | (uint64(1) << (vc1 % 64)) - - vc2 := content[2] - i2 := uint(vc2) >> 6 - answer.bitmap[i2] = answer.bitmap[i2] | (uint64(1) << (vc2 % 64)) - - vc3 := content[3] - i3 := uint(vc3) >> 6 - answer.bitmap[i3] = answer.bitmap[i3] | (uint64(1) << (vc3 % 64)) - } - - for k := c &^ 3; k < c; k++ { - vc := value2.content[k] - i := uint(vc) >> 6 - answer.bitmap[i] = answer.bitmap[i] | (uint64(1) << (vc % 64)) - } - - answer.cardinality = invalidCardinality - return answer -} - -func (bc *bitmapContainer) lazyORArray(value2 *arrayContainer) container { - answer := bc.clone().(*bitmapContainer) - return answer.lazyIORArray(value2) -} - -func (bc *bitmapContainer) lazyIORBitmap(value2 *bitmapContainer) container { - answer := bc - for k := 0; k < len(answer.bitmap); k++ { - answer.bitmap[k] = bc.bitmap[k] | value2.bitmap[k] - } - bc.cardinality = invalidCardinality - return answer -} - -func (bc *bitmapContainer) lazyORBitmap(value2 *bitmapContainer) container { - answer := bc.clone().(*bitmapContainer) - return answer.lazyIORBitmap(value2) -} - -func (bc *bitmapContainer) xor(a container) container { - switch x := a.(type) { - case *arrayContainer: - return bc.xorArray(x) - case *bitmapContainer: - return bc.xorBitmap(x) - case *runContainer16: - return x.xorBitmap(bc) - } - panic("unsupported container type") -} - -func (bc *bitmapContainer) xorArray(value2 *arrayContainer) container { - answer := bc.clone().(*bitmapContainer) - c := value2.getCardinality() - for k := 0; k < c; k++ { - vc := value2.content[k] - index := uint(vc) >> 6 - abi := answer.bitmap[index] - mask := uint64(1) << (vc % 64) - answer.cardinality += 1 - 2*int((abi&mask)>>(vc%64)) - answer.bitmap[index] = abi ^ mask - } - if answer.cardinality <= arrayDefaultMaxSize { - return answer.toArrayContainer() - } - return answer -} - -func (bc *bitmapContainer) rank(x uint16) int { - // TODO: rewrite in assembly - leftover := (uint(x) + 1) & 63 - if leftover == 0 { - return int(popcntSlice(bc.bitmap[:(uint(x)+1)/64])) - } - return int(popcntSlice(bc.bitmap[:(uint(x)+1)/64]) + popcount(bc.bitmap[(uint(x)+1)/64]<<(64-leftover))) -} - -func (bc *bitmapContainer) selectInt(x uint16) int { - remaining := x - for k := 0; k < len(bc.bitmap); k++ { - w := popcount(bc.bitmap[k]) - if uint16(w) > remaining { - return k*64 + selectBitPosition(bc.bitmap[k], int(remaining)) - } - remaining -= uint16(w) - } - return -1 -} - -func (bc *bitmapContainer) xorBitmap(value2 *bitmapContainer) container { - newCardinality := int(popcntXorSlice(bc.bitmap, value2.bitmap)) - - if newCardinality > arrayDefaultMaxSize { - answer := newBitmapContainer() - for k := 0; k < len(answer.bitmap); k++ { - answer.bitmap[k] = bc.bitmap[k] ^ value2.bitmap[k] - } - answer.cardinality = newCardinality - if answer.isFull() { - return newRunContainer16Range(0, MaxUint16) - } - return answer - } - ac := newArrayContainerSize(newCardinality) - fillArrayXOR(ac.content, bc.bitmap, value2.bitmap) - ac.content = ac.content[:newCardinality] - return ac -} - -func (bc *bitmapContainer) and(a container) container { - switch x := a.(type) { - case *arrayContainer: - return bc.andArray(x) - case *bitmapContainer: - return bc.andBitmap(x) - case *runContainer16: - if x.isFull() { - return bc.clone() - } - return x.andBitmapContainer(bc) - } - panic("unsupported container type") -} - -func (bc *bitmapContainer) andCardinality(a container) int { - switch x := a.(type) { - case *arrayContainer: - return bc.andArrayCardinality(x) - case *bitmapContainer: - return bc.andBitmapCardinality(x) - case *runContainer16: - return x.andBitmapContainerCardinality(bc) - } - panic("unsupported container type") -} - -func (bc *bitmapContainer) intersects(a container) bool { - switch x := a.(type) { - case *arrayContainer: - return bc.intersectsArray(x) - case *bitmapContainer: - return bc.intersectsBitmap(x) - case *runContainer16: - return x.intersects(bc) - - } - panic("unsupported container type") -} - -func (bc *bitmapContainer) iand(a container) container { - switch x := a.(type) { - case *arrayContainer: - return bc.iandArray(x) - case *bitmapContainer: - return bc.iandBitmap(x) - case *runContainer16: - if x.isFull() { - return bc.clone() - } - return bc.iandRun16(x) - } - panic("unsupported container type") -} - -func (bc *bitmapContainer) iandRun16(rc *runContainer16) container { - rcb := newBitmapContainerFromRun(rc) - return bc.iandBitmap(rcb) -} - -func (bc *bitmapContainer) iandArray(ac *arrayContainer) container { - acb := ac.toBitmapContainer() - return bc.iandBitmap(acb) -} - -func (bc *bitmapContainer) andArray(value2 *arrayContainer) *arrayContainer { - answer := newArrayContainerCapacity(len(value2.content)) - answer.content = answer.content[:cap(answer.content)] - c := value2.getCardinality() - pos := 0 - for k := 0; k < c; k++ { - v := value2.content[k] - answer.content[pos] = v - pos += int(bc.bitValue(v)) - } - answer.content = answer.content[:pos] - return answer -} - -func (bc *bitmapContainer) andArrayCardinality(value2 *arrayContainer) int { - c := value2.getCardinality() - pos := 0 - for k := 0; k < c; k++ { - v := value2.content[k] - pos += int(bc.bitValue(v)) - } - return pos -} - -func (bc *bitmapContainer) getCardinalityInRange(start, end uint) int { - if start >= end { - return 0 - } - firstword := start / 64 - endword := (end - 1) / 64 - const allones = ^uint64(0) - if firstword == endword { - return int(popcount(bc.bitmap[firstword] & ((allones << (start % 64)) & (allones >> ((64 - end) & 63))))) - } - answer := popcount(bc.bitmap[firstword] & (allones << (start % 64))) - answer += popcntSlice(bc.bitmap[firstword+1 : endword]) - answer += popcount(bc.bitmap[endword] & (allones >> ((64 - end) & 63))) - return int(answer) -} - -func (bc *bitmapContainer) andBitmap(value2 *bitmapContainer) container { - newcardinality := int(popcntAndSlice(bc.bitmap, value2.bitmap)) - if newcardinality > arrayDefaultMaxSize { - answer := newBitmapContainer() - for k := 0; k < len(answer.bitmap); k++ { - answer.bitmap[k] = bc.bitmap[k] & value2.bitmap[k] - } - answer.cardinality = newcardinality - return answer - } - ac := newArrayContainerSize(newcardinality) - fillArrayAND(ac.content, bc.bitmap, value2.bitmap) - ac.content = ac.content[:newcardinality] //not sure why i need this - return ac - -} - -func (bc *bitmapContainer) intersectsArray(value2 *arrayContainer) bool { - c := value2.getCardinality() - for k := 0; k < c; k++ { - v := value2.content[k] - if bc.contains(v) { - return true - } - } - return false -} - -func (bc *bitmapContainer) intersectsBitmap(value2 *bitmapContainer) bool { - for k := 0; k < len(bc.bitmap); k++ { - if (bc.bitmap[k] & value2.bitmap[k]) != 0 { - return true - } - } - return false - -} - -func (bc *bitmapContainer) iandBitmap(value2 *bitmapContainer) container { - newcardinality := int(popcntAndSlice(bc.bitmap, value2.bitmap)) - for k := 0; k < len(bc.bitmap); k++ { - bc.bitmap[k] = bc.bitmap[k] & value2.bitmap[k] - } - bc.cardinality = newcardinality - - if newcardinality <= arrayDefaultMaxSize { - return newArrayContainerFromBitmap(bc) - } - return bc -} - -func (bc *bitmapContainer) andNot(a container) container { - switch x := a.(type) { - case *arrayContainer: - return bc.andNotArray(x) - case *bitmapContainer: - return bc.andNotBitmap(x) - case *runContainer16: - return bc.andNotRun16(x) - } - panic("unsupported container type") -} - -func (bc *bitmapContainer) andNotRun16(rc *runContainer16) container { - rcb := rc.toBitmapContainer() - return bc.andNotBitmap(rcb) -} - -func (bc *bitmapContainer) iandNot(a container) container { - switch x := a.(type) { - case *arrayContainer: - return bc.iandNotArray(x) - case *bitmapContainer: - return bc.iandNotBitmapSurely(x) - case *runContainer16: - return bc.iandNotRun16(x) - } - panic("unsupported container type") -} - -func (bc *bitmapContainer) iandNotArray(ac *arrayContainer) container { - if ac.isEmpty() || bc.isEmpty() { - // Nothing to do. - return bc - } - - // Word by word, we remove the elements in ac from bc. The approach is to build - // a mask of the elements to remove, and then apply it to the bitmap. - wordIdx := uint16(0) - mask := uint64(0) - for i, v := range ac.content { - if v/64 != wordIdx { - // Flush the current word. - if i != 0 { - // We're removing bits that are set in the mask and in the current word. - // To figure out the cardinality change, we count the number of bits that - // are set in the mask and in the current word. - mask &= bc.bitmap[wordIdx] - bc.bitmap[wordIdx] &= ^mask - bc.cardinality -= int(popcount(mask)) - } - - wordIdx = v / 64 - mask = 0 - } - mask |= 1 << (v % 64) - } - - // Flush the last word. - mask &= bc.bitmap[wordIdx] - bc.bitmap[wordIdx] &= ^mask - bc.cardinality -= int(popcount(mask)) - - if bc.getCardinality() <= arrayDefaultMaxSize { - return bc.toArrayContainer() - } - return bc -} - -func (bc *bitmapContainer) iandNotRun16(rc *runContainer16) container { - if rc.isEmpty() || bc.isEmpty() { - // Nothing to do. - return bc - } - - wordRangeStart := rc.iv[0].start / 64 - wordRangeEnd := (rc.iv[len(rc.iv)-1].last()) / 64 // inclusive - - cardinalityChange := popcntSlice(bc.bitmap[wordRangeStart : wordRangeEnd+1]) // before cardinality - after cardinality (for word range) - - for _, iv := range rc.iv { - resetBitmapRange(bc.bitmap, int(iv.start), int(iv.last())+1) - } - - cardinalityChange -= popcntSlice(bc.bitmap[wordRangeStart : wordRangeEnd+1]) - - bc.cardinality -= int(cardinalityChange) - - if bc.getCardinality() <= arrayDefaultMaxSize { - return bc.toArrayContainer() - } - return bc -} - -func (bc *bitmapContainer) andNotArray(value2 *arrayContainer) container { - answer := bc.clone().(*bitmapContainer) - c := value2.getCardinality() - for k := 0; k < c; k++ { - vc := value2.content[k] - i := uint(vc) >> 6 - oldv := answer.bitmap[i] - newv := oldv &^ (uint64(1) << (vc % 64)) - answer.bitmap[i] = newv - answer.cardinality -= int((oldv ^ newv) >> (vc % 64)) - } - if answer.cardinality <= arrayDefaultMaxSize { - return answer.toArrayContainer() - } - return answer -} - -func (bc *bitmapContainer) andNotBitmap(value2 *bitmapContainer) container { - newCardinality := int(popcntMaskSlice(bc.bitmap, value2.bitmap)) - if newCardinality > arrayDefaultMaxSize { - answer := newBitmapContainer() - for k := 0; k < len(answer.bitmap); k++ { - answer.bitmap[k] = bc.bitmap[k] &^ value2.bitmap[k] - } - answer.cardinality = newCardinality - return answer - } - ac := newArrayContainerSize(newCardinality) - fillArrayANDNOT(ac.content, bc.bitmap, value2.bitmap) - return ac -} - -func (bc *bitmapContainer) iandNotBitmapSurely(value2 *bitmapContainer) container { - newCardinality := int(popcntMaskSlice(bc.bitmap, value2.bitmap)) - for k := 0; k < len(bc.bitmap); k++ { - bc.bitmap[k] = bc.bitmap[k] &^ value2.bitmap[k] - } - bc.cardinality = newCardinality - if bc.getCardinality() <= arrayDefaultMaxSize { - return bc.toArrayContainer() - } - return bc -} - -func (bc *bitmapContainer) contains(i uint16) bool { //testbit - x := uint(i) - w := bc.bitmap[x>>6] - mask := uint64(1) << (x & 63) - return (w & mask) != 0 -} - -func (bc *bitmapContainer) bitValue(i uint16) uint64 { - x := uint(i) - w := bc.bitmap[x>>6] - return (w >> (x & 63)) & 1 -} - -func (bc *bitmapContainer) loadData(arrayContainer *arrayContainer) { - bc.cardinality = arrayContainer.getCardinality() - c := arrayContainer.getCardinality() - for k := 0; k < c; k++ { - x := arrayContainer.content[k] - i := int(x) / 64 - bc.bitmap[i] |= (uint64(1) << uint(x%64)) - } -} - -func (bc *bitmapContainer) resetTo(a container) { - switch x := a.(type) { - case *arrayContainer: - fill(bc.bitmap, 0) - bc.loadData(x) - - case *bitmapContainer: - bc.cardinality = x.cardinality - copy(bc.bitmap, x.bitmap) - - case *runContainer16: - bc.cardinality = len(x.iv) - lastEnd := 0 - for _, r := range x.iv { - bc.cardinality += int(r.length) - resetBitmapRange(bc.bitmap, lastEnd, int(r.start)) - lastEnd = int(r.start+r.length) + 1 - setBitmapRange(bc.bitmap, int(r.start), lastEnd) - } - resetBitmapRange(bc.bitmap, lastEnd, maxCapacity) - - default: - panic("unsupported container type") - } -} - -func (bc *bitmapContainer) toArrayContainer() *arrayContainer { - ac := &arrayContainer{} - ac.loadData(bc) - return ac -} - -func (bc *bitmapContainer) fillArray(container []uint16) { - //TODO: rewrite in assembly - pos := 0 - base := 0 - for k := 0; k < len(bc.bitmap); k++ { - bitset := bc.bitmap[k] - for bitset != 0 { - t := bitset & -bitset - container[pos] = uint16((base + int(popcount(t-1)))) - pos = pos + 1 - bitset ^= t - } - base += 64 - } -} - -func (bc *bitmapContainer) NextSetBit(i uint) int { - var ( - x = i / 64 - length = uint(len(bc.bitmap)) - ) - if x >= length { - return -1 - } - w := bc.bitmap[x] - w = w >> uint(i%64) - if w != 0 { - return int(i) + countTrailingZeros(w) - } - x++ - for ; x < length; x++ { - if bc.bitmap[x] != 0 { - return int(x*64) + countTrailingZeros(bc.bitmap[x]) - } - } - return -1 -} - -func (bc *bitmapContainer) PrevSetBit(i int) int { - if i < 0 { - return -1 - } - x := i / 64 - if x >= len(bc.bitmap) { - return -1 - } - - w := bc.bitmap[x] - - b := i % 64 - - w = w << uint(63-b) - if w != 0 { - return i - countLeadingZeros(w) - } - x-- - for ; x >= 0; x-- { - if bc.bitmap[x] != 0 { - return (x * 64) + 63 - countLeadingZeros(bc.bitmap[x]) - } - } - return -1 -} - -// reference the java implementation -// https://github.com/RoaringBitmap/RoaringBitmap/blob/master/src/main/java/org/roaringbitmap/BitmapContainer.java#L875-L892 -func (bc *bitmapContainer) numberOfRuns() int { - if bc.cardinality == 0 { - return 0 - } - - var numRuns uint64 - nextWord := bc.bitmap[0] - - for i := 0; i < len(bc.bitmap)-1; i++ { - word := nextWord - nextWord = bc.bitmap[i+1] - numRuns += popcount((^word)&(word<<1)) + ((word >> 63) &^ nextWord) - } - - word := nextWord - numRuns += popcount((^word) & (word << 1)) - if (word & 0x8000000000000000) != 0 { - numRuns++ - } - - return int(numRuns) -} - -// convert to run or array *if needed* -func (bc *bitmapContainer) toEfficientContainer() container { - - numRuns := bc.numberOfRuns() - - sizeAsRunContainer := runContainer16SerializedSizeInBytes(numRuns) - sizeAsBitmapContainer := bitmapContainerSizeInBytes() - card := bc.getCardinality() - sizeAsArrayContainer := arrayContainerSizeInBytes(card) - - if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) { - return newRunContainer16FromBitmapContainer(bc) - } - if card <= arrayDefaultMaxSize { - return bc.toArrayContainer() - } - return bc -} - -func newBitmapContainerFromRun(rc *runContainer16) *bitmapContainer { - - if len(rc.iv) == 1 { - return newBitmapContainerwithRange(int(rc.iv[0].start), int(rc.iv[0].last())) - } - - bc := newBitmapContainer() - for i := range rc.iv { - setBitmapRange(bc.bitmap, int(rc.iv[i].start), int(rc.iv[i].last())+1) - bc.cardinality += int(rc.iv[i].last()) + 1 - int(rc.iv[i].start) - } - //bc.computeCardinality() - return bc -} - -func (bc *bitmapContainer) containerType() contype { - return bitmapContype -} - -func (bc *bitmapContainer) addOffset(x uint16) (container, container) { - var low, high *bitmapContainer - - if bc.cardinality == 0 { - return nil, nil - } - - b := uint32(x) >> 6 - i := uint32(x) % 64 - end := uint32(1024) - b - - low = newBitmapContainer() - if i == 0 { - copy(low.bitmap[b:], bc.bitmap[:end]) - } else { - low.bitmap[b] = bc.bitmap[0] << i - for k := uint32(1); k < end; k++ { - newval := bc.bitmap[k] << i - newval |= bc.bitmap[k-1] >> (64 - i) - low.bitmap[b+k] = newval - } - } - low.computeCardinality() - - if low.cardinality == bc.cardinality { - // All elements from bc ended up in low, meaning high will be empty. - return low, nil - } - - if low.cardinality == 0 { - // low is empty, let's reuse the container for high. - high = low - low = nil - } else { - // None of the containers will be empty, so allocate both. - high = newBitmapContainer() - } - - if i == 0 { - copy(high.bitmap[:b], bc.bitmap[end:]) - } else { - for k := end; k < 1024; k++ { - newval := bc.bitmap[k] << i - newval |= bc.bitmap[k-1] >> (64 - i) - high.bitmap[k-end] = newval - } - high.bitmap[b] = bc.bitmap[1023] >> (64 - i) - } - high.computeCardinality() - - // Ensure proper nil interface. - if low == nil { - return nil, high - } - - return low, high -} diff --git a/vendor/github.com/RoaringBitmap/roaring/clz.go b/vendor/github.com/RoaringBitmap/roaring/clz.go deleted file mode 100644 index ee0ebc6c96..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/clz.go +++ /dev/null @@ -1,13 +0,0 @@ -//go:build go1.9 -// +build go1.9 - -// "go1.9", from Go version 1.9 onward -// See https://golang.org/pkg/go/build/#hdr-Build_Constraints - -package roaring - -import "math/bits" - -func countLeadingZeros(x uint64) int { - return bits.LeadingZeros64(x) -} diff --git a/vendor/github.com/RoaringBitmap/roaring/clz_compat.go b/vendor/github.com/RoaringBitmap/roaring/clz_compat.go deleted file mode 100644 index 7ee16b4aeb..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/clz_compat.go +++ /dev/null @@ -1,37 +0,0 @@ -//go:build !go1.9 -// +build !go1.9 - -package roaring - -// LeadingZeroBits returns the number of consecutive most significant zero -// bits of x. -func countLeadingZeros(i uint64) int { - if i == 0 { - return 64 - } - n := 1 - x := uint32(i >> 32) - if x == 0 { - n += 32 - x = uint32(i) - } - if (x >> 16) == 0 { - n += 16 - x <<= 16 - } - if (x >> 24) == 0 { - n += 8 - x <<= 8 - } - if x>>28 == 0 { - n += 4 - x <<= 4 - } - if x>>30 == 0 { - n += 2 - x <<= 2 - - } - n -= int(x >> 31) - return n -} diff --git a/vendor/github.com/RoaringBitmap/roaring/ctz.go b/vendor/github.com/RoaringBitmap/roaring/ctz.go deleted file mode 100644 index fbcfe9128a..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/ctz.go +++ /dev/null @@ -1,13 +0,0 @@ -//go:build go1.9 -// +build go1.9 - -// "go1.9", from Go version 1.9 onward -// See https://golang.org/pkg/go/build/#hdr-Build_Constraints - -package roaring - -import "math/bits" - -func countTrailingZeros(x uint64) int { - return bits.TrailingZeros64(x) -} diff --git a/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go b/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go deleted file mode 100644 index d01df825a0..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go +++ /dev/null @@ -1,72 +0,0 @@ -//go:build !go1.9 -// +build !go1.9 - -package roaring - -// Reuse of portions of go/src/math/big standard lib code -// under this license: -/* -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -const deBruijn32 = 0x077CB531 - -var deBruijn32Lookup = []byte{ - 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, - 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9, -} - -const deBruijn64 = 0x03f79d71b4ca8b09 - -var deBruijn64Lookup = []byte{ - 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, - 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, - 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, - 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, -} - -// trailingZeroBits returns the number of consecutive least significant zero -// bits of x. -func countTrailingZeros(x uint64) int { - // x & -x leaves only the right-most bit set in the word. Let k be the - // index of that bit. Since only a single bit is set, the value is two - // to the power of k. Multiplying by a power of two is equivalent to - // left shifting, in this case by k bits. The de Bruijn constant is - // such that all six bit, consecutive substrings are distinct. - // Therefore, if we have a left shifted version of this constant we can - // find by how many bits it was shifted by looking at which six bit - // substring ended up at the top of the word. - // (Knuth, volume 4, section 7.3.1) - if x == 0 { - // We have to special case 0; the fomula - // below doesn't work for 0. - return 64 - } - return int(deBruijn64Lookup[((x&-x)*(deBruijn64))>>58]) -} diff --git a/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go b/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go deleted file mode 100644 index 7d0a92fe0e..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go +++ /dev/null @@ -1,313 +0,0 @@ -package roaring - -import ( - "container/heap" -) - -// Or function that requires repairAfterLazy -func lazyOR(x1, x2 *Bitmap) *Bitmap { - answer := NewBitmap() - pos1 := 0 - pos2 := 0 - length1 := x1.highlowcontainer.size() - length2 := x2.highlowcontainer.size() -main: - for (pos1 < length1) && (pos2 < length2) { - s1 := x1.highlowcontainer.getKeyAtIndex(pos1) - s2 := x2.highlowcontainer.getKeyAtIndex(pos2) - - for { - if s1 < s2 { - answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1) - pos1++ - if pos1 == length1 { - break main - } - s1 = x1.highlowcontainer.getKeyAtIndex(pos1) - } else if s1 > s2 { - answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2) - pos2++ - if pos2 == length2 { - break main - } - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } else { - c1 := x1.highlowcontainer.getContainerAtIndex(pos1) - answer.highlowcontainer.appendContainer(s1, c1.lazyOR(x2.highlowcontainer.getContainerAtIndex(pos2)), false) - pos1++ - pos2++ - if (pos1 == length1) || (pos2 == length2) { - break main - } - s1 = x1.highlowcontainer.getKeyAtIndex(pos1) - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } - } - } - if pos1 == length1 { - answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) - } else if pos2 == length2 { - answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1) - } - return answer -} - -// In-place Or function that requires repairAfterLazy -func (x1 *Bitmap) lazyOR(x2 *Bitmap) *Bitmap { - pos1 := 0 - pos2 := 0 - length1 := x1.highlowcontainer.size() - length2 := x2.highlowcontainer.size() -main: - for (pos1 < length1) && (pos2 < length2) { - s1 := x1.highlowcontainer.getKeyAtIndex(pos1) - s2 := x2.highlowcontainer.getKeyAtIndex(pos2) - - for { - if s1 < s2 { - pos1++ - if pos1 == length1 { - break main - } - s1 = x1.highlowcontainer.getKeyAtIndex(pos1) - } else if s1 > s2 { - x1.highlowcontainer.insertNewKeyValueAt(pos1, s2, x2.highlowcontainer.getContainerAtIndex(pos2).clone()) - pos2++ - pos1++ - length1++ - if pos2 == length2 { - break main - } - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } else { - c1 := x1.highlowcontainer.getWritableContainerAtIndex(pos1) - x1.highlowcontainer.containers[pos1] = c1.lazyIOR(x2.highlowcontainer.getContainerAtIndex(pos2)) - x1.highlowcontainer.needCopyOnWrite[pos1] = false - pos1++ - pos2++ - if (pos1 == length1) || (pos2 == length2) { - break main - } - s1 = x1.highlowcontainer.getKeyAtIndex(pos1) - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } - } - } - if pos1 == length1 { - x1.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) - } - return x1 -} - -// to be called after lazy aggregates -func (x1 *Bitmap) repairAfterLazy() { - for pos := 0; pos < x1.highlowcontainer.size(); pos++ { - c := x1.highlowcontainer.getContainerAtIndex(pos) - switch c.(type) { - case *bitmapContainer: - if c.(*bitmapContainer).cardinality == invalidCardinality { - c = x1.highlowcontainer.getWritableContainerAtIndex(pos) - c.(*bitmapContainer).computeCardinality() - if c.(*bitmapContainer).getCardinality() <= arrayDefaultMaxSize { - x1.highlowcontainer.setContainerAtIndex(pos, c.(*bitmapContainer).toArrayContainer()) - } else if c.(*bitmapContainer).isFull() { - x1.highlowcontainer.setContainerAtIndex(pos, newRunContainer16Range(0, MaxUint16)) - } - } - } - } -} - -// FastAnd computes the intersection between many bitmaps quickly -// Compared to the And function, it can take many bitmaps as input, thus saving the trouble -// of manually calling "And" many times. -// -// Performance hints: if you have very large and tiny bitmaps, -// it may be beneficial performance-wise to put a tiny bitmap -// in first position. -func FastAnd(bitmaps ...*Bitmap) *Bitmap { - if len(bitmaps) == 0 { - return NewBitmap() - } else if len(bitmaps) == 1 { - return bitmaps[0].Clone() - } - answer := And(bitmaps[0], bitmaps[1]) - for _, bm := range bitmaps[2:] { - answer.And(bm) - } - return answer -} - -// FastOr computes the union between many bitmaps quickly, as opposed to having to call Or repeatedly. -// It might also be faster than calling Or repeatedly. -func FastOr(bitmaps ...*Bitmap) *Bitmap { - if len(bitmaps) == 0 { - return NewBitmap() - } else if len(bitmaps) == 1 { - return bitmaps[0].Clone() - } - answer := lazyOR(bitmaps[0], bitmaps[1]) - for _, bm := range bitmaps[2:] { - answer = answer.lazyOR(bm) - } - // here is where repairAfterLazy is called. - answer.repairAfterLazy() - return answer -} - -// HeapOr computes the union between many bitmaps quickly using a heap. -// It might be faster than calling Or repeatedly. -func HeapOr(bitmaps ...*Bitmap) *Bitmap { - if len(bitmaps) == 0 { - return NewBitmap() - } - // TODO: for better speed, we could do the operation lazily, see Java implementation - pq := make(priorityQueue, len(bitmaps)) - for i, bm := range bitmaps { - pq[i] = &item{bm, i} - } - heap.Init(&pq) - - for pq.Len() > 1 { - x1 := heap.Pop(&pq).(*item) - x2 := heap.Pop(&pq).(*item) - heap.Push(&pq, &item{Or(x1.value, x2.value), 0}) - } - return heap.Pop(&pq).(*item).value -} - -// HeapXor computes the symmetric difference between many bitmaps quickly (as opposed to calling Xor repeated). -// Internally, this function uses a heap. -// It might be faster than calling Xor repeatedly. -func HeapXor(bitmaps ...*Bitmap) *Bitmap { - if len(bitmaps) == 0 { - return NewBitmap() - } - - pq := make(priorityQueue, len(bitmaps)) - for i, bm := range bitmaps { - pq[i] = &item{bm, i} - } - heap.Init(&pq) - - for pq.Len() > 1 { - x1 := heap.Pop(&pq).(*item) - x2 := heap.Pop(&pq).(*item) - heap.Push(&pq, &item{Xor(x1.value, x2.value), 0}) - } - return heap.Pop(&pq).(*item).value -} - -// AndAny provides a result equivalent to x1.And(FastOr(bitmaps)). -// It's optimized to minimize allocations. It also might be faster than separate calls. -func (x1 *Bitmap) AndAny(bitmaps ...*Bitmap) { - if len(bitmaps) == 0 { - return - } else if len(bitmaps) == 1 { - x1.And(bitmaps[0]) - return - } - - type withPos struct { - bitmap *roaringArray - pos int - key uint16 - } - filters := make([]withPos, 0, len(bitmaps)) - - for _, b := range bitmaps { - if b.highlowcontainer.size() > 0 { - filters = append(filters, withPos{ - bitmap: &b.highlowcontainer, - pos: 0, - key: b.highlowcontainer.getKeyAtIndex(0), - }) - } - } - - basePos := 0 - intersections := 0 - keyContainers := make([]container, 0, len(filters)) - var ( - tmpArray *arrayContainer - tmpBitmap *bitmapContainer - minNextKey uint16 - ) - - for basePos < x1.highlowcontainer.size() && len(filters) > 0 { - baseKey := x1.highlowcontainer.getKeyAtIndex(basePos) - - // accumulate containers for current key, find next minimal key in filters - // and exclude filters that do not have related values anymore - i := 0 - maxPossibleOr := 0 - minNextKey = MaxUint16 - for _, f := range filters { - if f.key < baseKey { - f.pos = f.bitmap.advanceUntil(baseKey, f.pos) - if f.pos == f.bitmap.size() { - continue - } - f.key = f.bitmap.getKeyAtIndex(f.pos) - } - - if f.key == baseKey { - cont := f.bitmap.getContainerAtIndex(f.pos) - keyContainers = append(keyContainers, cont) - maxPossibleOr += cont.getCardinality() - - f.pos++ - if f.pos == f.bitmap.size() { - continue - } - f.key = f.bitmap.getKeyAtIndex(f.pos) - } - - minNextKey = minOfUint16(minNextKey, f.key) - filters[i] = f - i++ - } - filters = filters[:i] - - if len(keyContainers) == 0 { - basePos = x1.highlowcontainer.advanceUntil(minNextKey, basePos) - continue - } - - var ored container - - if len(keyContainers) == 1 { - ored = keyContainers[0] - } else { - //TODO: special case for run containers? - if maxPossibleOr > arrayDefaultMaxSize { - if tmpBitmap == nil { - tmpBitmap = newBitmapContainer() - } - tmpBitmap.resetTo(keyContainers[0]) - ored = tmpBitmap - } else { - if tmpArray == nil { - tmpArray = newArrayContainerCapacity(maxPossibleOr) - } - tmpArray.realloc(maxPossibleOr) - tmpArray.resetTo(keyContainers[0]) - ored = tmpArray - } - for _, c := range keyContainers[1:] { - ored = ored.ior(c) - } - } - - result := x1.highlowcontainer.getWritableContainerAtIndex(basePos).iand(ored) - if !result.isEmpty() { - x1.highlowcontainer.replaceKeyAndContainerAtIndex(intersections, baseKey, result, false) - intersections++ - } - - keyContainers = keyContainers[:0] - basePos = x1.highlowcontainer.advanceUntil(minNextKey, basePos) - } - - x1.highlowcontainer.resize(intersections) -} diff --git a/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go b/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go deleted file mode 100644 index d5ebb91ab7..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go +++ /dev/null @@ -1,215 +0,0 @@ -package internal - -import ( - "encoding/binary" - "io" -) - -// ByteInput typed interface around io.Reader or raw bytes -type ByteInput interface { - // Next returns a slice containing the next n bytes from the buffer, - // advancing the buffer as if the bytes had been returned by Read. - Next(n int) ([]byte, error) - // NextReturnsSafeSlice returns true if Next() returns a safe slice as opposed - // to a slice that points to an underlying buffer possibly owned by another system. - // When NextReturnsSafeSlice returns false, the result from Next() should be copied - // before it is modified (i.e., it is immutable). - NextReturnsSafeSlice() bool - // ReadUInt32 reads uint32 with LittleEndian order - ReadUInt32() (uint32, error) - // ReadUInt16 reads uint16 with LittleEndian order - ReadUInt16() (uint16, error) - // GetReadBytes returns read bytes - GetReadBytes() int64 - // SkipBytes skips exactly n bytes - SkipBytes(n int) error -} - -// NewByteInputFromReader creates reader wrapper -func NewByteInputFromReader(reader io.Reader) ByteInput { - return &ByteInputAdapter{ - r: reader, - readBytes: 0, - } -} - -// NewByteInput creates raw bytes wrapper -func NewByteInput(buf []byte) ByteInput { - return &ByteBuffer{ - buf: buf, - off: 0, - } -} - -// ByteBuffer raw bytes wrapper -type ByteBuffer struct { - buf []byte - off int -} - -// NewByteBuffer creates a new ByteBuffer. -func NewByteBuffer(buf []byte) *ByteBuffer { - return &ByteBuffer{ - buf: buf, - } -} - -var _ io.Reader = (*ByteBuffer)(nil) - -// Read implements io.Reader. -func (b *ByteBuffer) Read(p []byte) (int, error) { - data, err := b.Next(len(p)) - if err != nil { - return 0, err - } - copy(p, data) - return len(data), nil -} - -// Next returns a slice containing the next n bytes from the reader -// If there are fewer bytes than the given n, io.ErrUnexpectedEOF will be returned -func (b *ByteBuffer) Next(n int) ([]byte, error) { - m := len(b.buf) - b.off - - if n > m { - return nil, io.ErrUnexpectedEOF - } - - data := b.buf[b.off : b.off+n] - b.off += n - - return data, nil -} - -// NextReturnsSafeSlice returns false since ByteBuffer might hold -// an array owned by some other systems. -func (b *ByteBuffer) NextReturnsSafeSlice() bool { - return false -} - -// ReadUInt32 reads uint32 with LittleEndian order -func (b *ByteBuffer) ReadUInt32() (uint32, error) { - if len(b.buf)-b.off < 4 { - return 0, io.ErrUnexpectedEOF - } - - v := binary.LittleEndian.Uint32(b.buf[b.off:]) - b.off += 4 - - return v, nil -} - -// ReadUInt16 reads uint16 with LittleEndian order -func (b *ByteBuffer) ReadUInt16() (uint16, error) { - if len(b.buf)-b.off < 2 { - return 0, io.ErrUnexpectedEOF - } - - v := binary.LittleEndian.Uint16(b.buf[b.off:]) - b.off += 2 - - return v, nil -} - -// GetReadBytes returns read bytes -func (b *ByteBuffer) GetReadBytes() int64 { - return int64(b.off) -} - -// SkipBytes skips exactly n bytes -func (b *ByteBuffer) SkipBytes(n int) error { - m := len(b.buf) - b.off - - if n > m { - return io.ErrUnexpectedEOF - } - - b.off += n - - return nil -} - -// Reset resets the given buffer with a new byte slice -func (b *ByteBuffer) Reset(buf []byte) { - b.buf = buf - b.off = 0 -} - -// ByteInputAdapter reader wrapper -type ByteInputAdapter struct { - r io.Reader - readBytes int - buf [4]byte -} - -var _ io.Reader = (*ByteInputAdapter)(nil) - -// Read implements io.Reader. -func (b *ByteInputAdapter) Read(buf []byte) (int, error) { - m, err := io.ReadAtLeast(b.r, buf, len(buf)) - b.readBytes += m - - if err != nil { - return 0, err - } - - return m, nil -} - -// Next returns a slice containing the next n bytes from the buffer, -// advancing the buffer as if the bytes had been returned by Read. -func (b *ByteInputAdapter) Next(n int) ([]byte, error) { - buf := make([]byte, n) - _, err := b.Read(buf) - - if err != nil { - return nil, err - } - return buf, nil -} - -// NextReturnsSafeSlice returns true since ByteInputAdapter always returns a slice -// allocated with make([]byte, ...) -func (b *ByteInputAdapter) NextReturnsSafeSlice() bool { - return true -} - -// ReadUInt32 reads uint32 with LittleEndian order -func (b *ByteInputAdapter) ReadUInt32() (uint32, error) { - buf := b.buf[:4] - _, err := b.Read(buf) - if err != nil { - return 0, err - } - - return binary.LittleEndian.Uint32(buf), nil -} - -// ReadUInt16 reads uint16 with LittleEndian order -func (b *ByteInputAdapter) ReadUInt16() (uint16, error) { - buf := b.buf[:2] - _, err := b.Read(buf) - if err != nil { - return 0, err - } - - return binary.LittleEndian.Uint16(buf), nil -} - -// GetReadBytes returns read bytes -func (b *ByteInputAdapter) GetReadBytes() int64 { - return int64(b.readBytes) -} - -// SkipBytes skips exactly n bytes -func (b *ByteInputAdapter) SkipBytes(n int) error { - _, err := b.Next(n) - - return err -} - -// Reset resets the given buffer with a new stream -func (b *ByteInputAdapter) Reset(stream io.Reader) { - b.r = stream - b.readBytes = 0 -} diff --git a/vendor/github.com/RoaringBitmap/roaring/internal/pools.go b/vendor/github.com/RoaringBitmap/roaring/internal/pools.go deleted file mode 100644 index d2583568d0..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/internal/pools.go +++ /dev/null @@ -1,21 +0,0 @@ -package internal - -import ( - "sync" -) - -var ( - // ByteInputAdapterPool shared pool - ByteInputAdapterPool = sync.Pool{ - New: func() interface{} { - return &ByteInputAdapter{} - }, - } - - // ByteBufferPool shared pool - ByteBufferPool = sync.Pool{ - New: func() interface{} { - return &ByteBuffer{} - }, - } -) diff --git a/vendor/github.com/RoaringBitmap/roaring/manyiterator.go b/vendor/github.com/RoaringBitmap/roaring/manyiterator.go deleted file mode 100644 index eaa5b79507..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/manyiterator.go +++ /dev/null @@ -1,32 +0,0 @@ -package roaring - -type manyIterable interface { - nextMany(hs uint32, buf []uint32) int - nextMany64(hs uint64, buf []uint64) int -} - -func (si *shortIterator) nextMany(hs uint32, buf []uint32) int { - n := 0 - l := si.loc - s := si.slice - for n < len(buf) && l < len(s) { - buf[n] = uint32(s[l]) | hs - l++ - n++ - } - si.loc = l - return n -} - -func (si *shortIterator) nextMany64(hs uint64, buf []uint64) int { - n := 0 - l := si.loc - s := si.slice - for n < len(buf) && l < len(s) { - buf[n] = uint64(s[l]) | hs - l++ - n++ - } - si.loc = l - return n -} diff --git a/vendor/github.com/RoaringBitmap/roaring/parallel.go b/vendor/github.com/RoaringBitmap/roaring/parallel.go deleted file mode 100644 index 9208e3e380..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/parallel.go +++ /dev/null @@ -1,612 +0,0 @@ -package roaring - -import ( - "container/heap" - "fmt" - "runtime" - "sync" -) - -var defaultWorkerCount = runtime.NumCPU() - -type bitmapContainerKey struct { - key uint16 - idx int - bitmap *Bitmap -} - -type multipleContainers struct { - key uint16 - containers []container - idx int -} - -type keyedContainer struct { - key uint16 - container container - idx int -} - -type bitmapContainerHeap []bitmapContainerKey - -func (h bitmapContainerHeap) Len() int { return len(h) } -func (h bitmapContainerHeap) Less(i, j int) bool { return h[i].key < h[j].key } -func (h bitmapContainerHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } - -func (h *bitmapContainerHeap) Push(x interface{}) { - // Push and Pop use pointer receivers because they modify the slice's length, - // not just its contents. - *h = append(*h, x.(bitmapContainerKey)) -} - -func (h *bitmapContainerHeap) Pop() interface{} { - old := *h - n := len(old) - x := old[n-1] - *h = old[0 : n-1] - return x -} - -func (h bitmapContainerHeap) Peek() bitmapContainerKey { - return h[0] -} - -func (h *bitmapContainerHeap) popIncrementing() (key uint16, container container) { - k := h.Peek() - key = k.key - container = k.bitmap.highlowcontainer.containers[k.idx] - - newIdx := k.idx + 1 - if newIdx < k.bitmap.highlowcontainer.size() { - k = bitmapContainerKey{ - k.bitmap.highlowcontainer.keys[newIdx], - newIdx, - k.bitmap, - } - (*h)[0] = k - heap.Fix(h, 0) - } else { - heap.Pop(h) - } - - return -} - -func (h *bitmapContainerHeap) Next(containers []container) multipleContainers { - if h.Len() == 0 { - return multipleContainers{} - } - - key, container := h.popIncrementing() - containers = append(containers, container) - - for h.Len() > 0 && key == h.Peek().key { - _, container = h.popIncrementing() - containers = append(containers, container) - } - - return multipleContainers{ - key, - containers, - -1, - } -} - -func newBitmapContainerHeap(bitmaps ...*Bitmap) bitmapContainerHeap { - // Initialize heap - var h bitmapContainerHeap = make([]bitmapContainerKey, 0, len(bitmaps)) - for _, bitmap := range bitmaps { - if !bitmap.IsEmpty() { - key := bitmapContainerKey{ - bitmap.highlowcontainer.keys[0], - 0, - bitmap, - } - h = append(h, key) - } - } - - heap.Init(&h) - - return h -} - -func repairAfterLazy(c container) container { - switch t := c.(type) { - case *bitmapContainer: - if t.cardinality == invalidCardinality { - t.computeCardinality() - } - - if t.getCardinality() <= arrayDefaultMaxSize { - return t.toArrayContainer() - } else if c.(*bitmapContainer).isFull() { - return newRunContainer16Range(0, MaxUint16) - } - } - - return c -} - -func toBitmapContainer(c container) container { - switch t := c.(type) { - case *arrayContainer: - return t.toBitmapContainer() - case *runContainer16: - if !t.isFull() { - return t.toBitmapContainer() - } - } - return c -} - -func appenderRoutine(bitmapChan chan<- *Bitmap, resultChan <-chan keyedContainer, expectedKeysChan <-chan int) { - expectedKeys := -1 - appendedKeys := 0 - var keys []uint16 - var containers []container - for appendedKeys != expectedKeys { - select { - case item := <-resultChan: - if len(keys) <= item.idx { - keys = append(keys, make([]uint16, item.idx-len(keys)+1)...) - containers = append(containers, make([]container, item.idx-len(containers)+1)...) - } - keys[item.idx] = item.key - containers[item.idx] = item.container - - appendedKeys++ - case msg := <-expectedKeysChan: - expectedKeys = msg - } - } - answer := &Bitmap{ - roaringArray{ - make([]uint16, 0, expectedKeys), - make([]container, 0, expectedKeys), - make([]bool, 0, expectedKeys), - false, - }, - } - for i := range keys { - if containers[i] != nil { // in case a resulting container was empty, see ParAnd function - answer.highlowcontainer.appendContainer(keys[i], containers[i], false) - } - } - - bitmapChan <- answer -} - -// ParHeapOr computes the union (OR) of all provided bitmaps in parallel, -// where the parameter "parallelism" determines how many workers are to be used -// (if it is set to 0, a default number of workers is chosen) -// ParHeapOr uses a heap to compute the union. For rare cases it might be faster than ParOr -func ParHeapOr(parallelism int, bitmaps ...*Bitmap) *Bitmap { - - bitmapCount := len(bitmaps) - if bitmapCount == 0 { - return NewBitmap() - } else if bitmapCount == 1 { - return bitmaps[0].Clone() - } - - if parallelism == 0 { - parallelism = defaultWorkerCount - } - - h := newBitmapContainerHeap(bitmaps...) - - bitmapChan := make(chan *Bitmap) - inputChan := make(chan multipleContainers, 128) - resultChan := make(chan keyedContainer, 32) - expectedKeysChan := make(chan int) - - pool := sync.Pool{ - New: func() interface{} { - return make([]container, 0, len(bitmaps)) - }, - } - - orFunc := func() { - // Assumes only structs with >=2 containers are passed - for input := range inputChan { - c := toBitmapContainer(input.containers[0]).lazyOR(input.containers[1]) - for _, next := range input.containers[2:] { - c = c.lazyIOR(next) - } - c = repairAfterLazy(c) - kx := keyedContainer{ - input.key, - c, - input.idx, - } - resultChan <- kx - pool.Put(input.containers[:0]) - } - } - - go appenderRoutine(bitmapChan, resultChan, expectedKeysChan) - - for i := 0; i < parallelism; i++ { - go orFunc() - } - - idx := 0 - for h.Len() > 0 { - ck := h.Next(pool.Get().([]container)) - if len(ck.containers) == 1 { - resultChan <- keyedContainer{ - ck.key, - ck.containers[0], - idx, - } - pool.Put(ck.containers[:0]) - } else { - ck.idx = idx - inputChan <- ck - } - idx++ - } - expectedKeysChan <- idx - - bitmap := <-bitmapChan - - close(inputChan) - close(resultChan) - close(expectedKeysChan) - - return bitmap -} - -// ParAnd computes the intersection (AND) of all provided bitmaps in parallel, -// where the parameter "parallelism" determines how many workers are to be used -// (if it is set to 0, a default number of workers is chosen) -func ParAnd(parallelism int, bitmaps ...*Bitmap) *Bitmap { - bitmapCount := len(bitmaps) - if bitmapCount == 0 { - return NewBitmap() - } else if bitmapCount == 1 { - return bitmaps[0].Clone() - } - - if parallelism == 0 { - parallelism = defaultWorkerCount - } - - h := newBitmapContainerHeap(bitmaps...) - - bitmapChan := make(chan *Bitmap) - inputChan := make(chan multipleContainers, 128) - resultChan := make(chan keyedContainer, 32) - expectedKeysChan := make(chan int) - - andFunc := func() { - // Assumes only structs with >=2 containers are passed - for input := range inputChan { - c := input.containers[0].and(input.containers[1]) - for _, next := range input.containers[2:] { - if c.isEmpty() { - break - } - c = c.iand(next) - } - - // Send a nil explicitly if the result of the intersection is an empty container - if c.isEmpty() { - c = nil - } - - kx := keyedContainer{ - input.key, - c, - input.idx, - } - resultChan <- kx - } - } - - go appenderRoutine(bitmapChan, resultChan, expectedKeysChan) - - for i := 0; i < parallelism; i++ { - go andFunc() - } - - idx := 0 - for h.Len() > 0 { - ck := h.Next(make([]container, 0, 4)) - if len(ck.containers) == bitmapCount { - ck.idx = idx - inputChan <- ck - idx++ - } - } - expectedKeysChan <- idx - - bitmap := <-bitmapChan - - close(inputChan) - close(resultChan) - close(expectedKeysChan) - - return bitmap -} - -// ParOr computes the union (OR) of all provided bitmaps in parallel, -// where the parameter "parallelism" determines how many workers are to be used -// (if it is set to 0, a default number of workers is chosen) -func ParOr(parallelism int, bitmaps ...*Bitmap) *Bitmap { - var lKey uint16 = MaxUint16 - var hKey uint16 - - bitmapsFiltered := bitmaps[:0] - for _, b := range bitmaps { - if !b.IsEmpty() { - bitmapsFiltered = append(bitmapsFiltered, b) - } - } - bitmaps = bitmapsFiltered - - for _, b := range bitmaps { - lKey = minOfUint16(lKey, b.highlowcontainer.keys[0]) - hKey = maxOfUint16(hKey, b.highlowcontainer.keys[b.highlowcontainer.size()-1]) - } - - if lKey == MaxUint16 && hKey == 0 { - return New() - } else if len(bitmaps) == 1 { - return bitmaps[0].Clone() - } - - keyRange := int(hKey) - int(lKey) + 1 - if keyRange == 1 { - // revert to FastOr. Since the key range is 0 - // no container-level aggregation parallelism is achievable - return FastOr(bitmaps...) - } - - if parallelism == 0 { - parallelism = defaultWorkerCount - } - - var chunkSize int - var chunkCount int - if parallelism*4 > int(keyRange) { - chunkSize = 1 - chunkCount = int(keyRange) - } else { - chunkCount = parallelism * 4 - chunkSize = (int(keyRange) + chunkCount - 1) / chunkCount - } - - if chunkCount*chunkSize < int(keyRange) { - // it's fine to panic to indicate an implementation error - panic(fmt.Sprintf("invariant check failed: chunkCount * chunkSize < keyRange, %d * %d < %d", chunkCount, chunkSize, keyRange)) - } - - chunks := make([]*roaringArray, chunkCount) - - chunkSpecChan := make(chan parChunkSpec, minOfInt(maxOfInt(64, 2*parallelism), int(chunkCount))) - chunkChan := make(chan parChunk, minOfInt(32, int(chunkCount))) - - orFunc := func() { - for spec := range chunkSpecChan { - ra := lazyOrOnRange(&bitmaps[0].highlowcontainer, &bitmaps[1].highlowcontainer, spec.start, spec.end) - for _, b := range bitmaps[2:] { - ra = lazyIOrOnRange(ra, &b.highlowcontainer, spec.start, spec.end) - } - - for i, c := range ra.containers { - ra.containers[i] = repairAfterLazy(c) - } - - chunkChan <- parChunk{ra, spec.idx} - } - } - - for i := 0; i < parallelism; i++ { - go orFunc() - } - - go func() { - for i := 0; i < chunkCount; i++ { - spec := parChunkSpec{ - start: uint16(int(lKey) + i*chunkSize), - end: uint16(minOfInt(int(lKey)+(i+1)*chunkSize-1, int(hKey))), - idx: int(i), - } - chunkSpecChan <- spec - } - }() - - chunksRemaining := chunkCount - for chunk := range chunkChan { - chunks[chunk.idx] = chunk.ra - chunksRemaining-- - if chunksRemaining == 0 { - break - } - } - close(chunkChan) - close(chunkSpecChan) - - containerCount := 0 - for _, chunk := range chunks { - containerCount += chunk.size() - } - - result := Bitmap{ - roaringArray{ - containers: make([]container, containerCount), - keys: make([]uint16, containerCount), - needCopyOnWrite: make([]bool, containerCount), - }, - } - - resultOffset := 0 - for _, chunk := range chunks { - copy(result.highlowcontainer.containers[resultOffset:], chunk.containers) - copy(result.highlowcontainer.keys[resultOffset:], chunk.keys) - copy(result.highlowcontainer.needCopyOnWrite[resultOffset:], chunk.needCopyOnWrite) - resultOffset += chunk.size() - } - - return &result -} - -type parChunkSpec struct { - start uint16 - end uint16 - idx int -} - -type parChunk struct { - ra *roaringArray - idx int -} - -func (c parChunk) size() int { - return c.ra.size() -} - -func parNaiveStartAt(ra *roaringArray, start uint16, last uint16) int { - for idx, key := range ra.keys { - if key >= start && key <= last { - return idx - } else if key > last { - break - } - } - return ra.size() -} - -func lazyOrOnRange(ra1, ra2 *roaringArray, start, last uint16) *roaringArray { - answer := newRoaringArray() - length1 := ra1.size() - length2 := ra2.size() - - idx1 := parNaiveStartAt(ra1, start, last) - idx2 := parNaiveStartAt(ra2, start, last) - - var key1 uint16 - var key2 uint16 - if idx1 < length1 && idx2 < length2 { - key1 = ra1.getKeyAtIndex(idx1) - key2 = ra2.getKeyAtIndex(idx2) - - for key1 <= last && key2 <= last { - - if key1 < key2 { - answer.appendCopy(*ra1, idx1) - idx1++ - if idx1 == length1 { - break - } - key1 = ra1.getKeyAtIndex(idx1) - } else if key1 > key2 { - answer.appendCopy(*ra2, idx2) - idx2++ - if idx2 == length2 { - break - } - key2 = ra2.getKeyAtIndex(idx2) - } else { - c1 := ra1.getFastContainerAtIndex(idx1, false) - - answer.appendContainer(key1, c1.lazyOR(ra2.getContainerAtIndex(idx2)), false) - idx1++ - idx2++ - if idx1 == length1 || idx2 == length2 { - break - } - - key1 = ra1.getKeyAtIndex(idx1) - key2 = ra2.getKeyAtIndex(idx2) - } - } - } - - if idx2 < length2 { - key2 = ra2.getKeyAtIndex(idx2) - for key2 <= last { - answer.appendCopy(*ra2, idx2) - idx2++ - if idx2 == length2 { - break - } - key2 = ra2.getKeyAtIndex(idx2) - } - } - - if idx1 < length1 { - key1 = ra1.getKeyAtIndex(idx1) - for key1 <= last { - answer.appendCopy(*ra1, idx1) - idx1++ - if idx1 == length1 { - break - } - key1 = ra1.getKeyAtIndex(idx1) - } - } - return answer -} - -func lazyIOrOnRange(ra1, ra2 *roaringArray, start, last uint16) *roaringArray { - length1 := ra1.size() - length2 := ra2.size() - - idx1 := 0 - idx2 := parNaiveStartAt(ra2, start, last) - - var key1 uint16 - var key2 uint16 - if idx1 < length1 && idx2 < length2 { - key1 = ra1.getKeyAtIndex(idx1) - key2 = ra2.getKeyAtIndex(idx2) - - for key1 <= last && key2 <= last { - if key1 < key2 { - idx1++ - if idx1 >= length1 { - break - } - key1 = ra1.getKeyAtIndex(idx1) - } else if key1 > key2 { - ra1.insertNewKeyValueAt(idx1, key2, ra2.getContainerAtIndex(idx2)) - ra1.needCopyOnWrite[idx1] = true - idx2++ - idx1++ - length1++ - if idx2 >= length2 { - break - } - key2 = ra2.getKeyAtIndex(idx2) - } else { - c1 := ra1.getFastContainerAtIndex(idx1, true) - - ra1.containers[idx1] = c1.lazyIOR(ra2.getContainerAtIndex(idx2)) - ra1.needCopyOnWrite[idx1] = false - idx1++ - idx2++ - if idx1 >= length1 || idx2 >= length2 { - break - } - - key1 = ra1.getKeyAtIndex(idx1) - key2 = ra2.getKeyAtIndex(idx2) - } - } - } - if idx2 < length2 { - key2 = ra2.getKeyAtIndex(idx2) - for key2 <= last { - ra1.appendCopy(*ra2, idx2) - idx2++ - if idx2 >= length2 { - break - } - key2 = ra2.getKeyAtIndex(idx2) - } - } - return ra1 -} diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt.go b/vendor/github.com/RoaringBitmap/roaring/popcnt.go deleted file mode 100644 index b4980aadb8..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/popcnt.go +++ /dev/null @@ -1,13 +0,0 @@ -//go:build go1.9 -// +build go1.9 - -// "go1.9", from Go version 1.9 onward -// See https://golang.org/pkg/go/build/#hdr-Build_Constraints - -package roaring - -import "math/bits" - -func popcount(x uint64) uint64 { - return uint64(bits.OnesCount64(x)) -} diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_amd64.s b/vendor/github.com/RoaringBitmap/roaring/popcnt_amd64.s deleted file mode 100644 index 1f13fa2eca..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/popcnt_amd64.s +++ /dev/null @@ -1,103 +0,0 @@ -// +build amd64,!appengine,!go1.9 - -TEXT ·hasAsm(SB),4,$0-1 -MOVQ $1, AX -CPUID -SHRQ $23, CX -ANDQ $1, CX -MOVB CX, ret+0(FP) -RET - -#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 - -TEXT ·popcntSliceAsm(SB),4,$0-32 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntSliceEnd -popcntSliceLoop: -BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX -ADDQ DX, AX -ADDQ $8, SI -LOOP popcntSliceLoop -popcntSliceEnd: -MOVQ AX, ret+24(FP) -RET - -TEXT ·popcntMaskSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntMaskSliceEnd -MOVQ m+24(FP), DI -popcntMaskSliceLoop: -MOVQ (DI), DX -NOTQ DX -ANDQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntMaskSliceLoop -popcntMaskSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntAndSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntAndSliceEnd -MOVQ m+24(FP), DI -popcntAndSliceLoop: -MOVQ (DI), DX -ANDQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntAndSliceLoop -popcntAndSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntOrSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntOrSliceEnd -MOVQ m+24(FP), DI -popcntOrSliceLoop: -MOVQ (DI), DX -ORQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntOrSliceLoop -popcntOrSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntXorSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntXorSliceEnd -MOVQ m+24(FP), DI -popcntXorSliceLoop: -MOVQ (DI), DX -XORQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntXorSliceLoop -popcntXorSliceEnd: -MOVQ AX, ret+48(FP) -RET diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go deleted file mode 100644 index ba2dac91ef..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go +++ /dev/null @@ -1,68 +0,0 @@ -//go:build amd64 && !appengine && !go1.9 -// +build amd64,!appengine,!go1.9 - -package roaring - -// *** the following functions are defined in popcnt_amd64.s - -//go:noescape - -func hasAsm() bool - -// useAsm is a flag used to select the GO or ASM implementation of the popcnt function -var useAsm = hasAsm() - -//go:noescape - -func popcntSliceAsm(s []uint64) uint64 - -//go:noescape - -func popcntMaskSliceAsm(s, m []uint64) uint64 - -//go:noescape - -func popcntAndSliceAsm(s, m []uint64) uint64 - -//go:noescape - -func popcntOrSliceAsm(s, m []uint64) uint64 - -//go:noescape - -func popcntXorSliceAsm(s, m []uint64) uint64 - -func popcntSlice(s []uint64) uint64 { - if useAsm { - return popcntSliceAsm(s) - } - return popcntSliceGo(s) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - if useAsm { - return popcntMaskSliceAsm(s, m) - } - return popcntMaskSliceGo(s, m) -} - -func popcntAndSlice(s, m []uint64) uint64 { - if useAsm { - return popcntAndSliceAsm(s, m) - } - return popcntAndSliceGo(s, m) -} - -func popcntOrSlice(s, m []uint64) uint64 { - if useAsm { - return popcntOrSliceAsm(s, m) - } - return popcntOrSliceGo(s, m) -} - -func popcntXorSlice(s, m []uint64) uint64 { - if useAsm { - return popcntXorSliceAsm(s, m) - } - return popcntXorSliceGo(s, m) -} diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go deleted file mode 100644 index 5933e52fc9..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go +++ /dev/null @@ -1,18 +0,0 @@ -//go:build !go1.9 -// +build !go1.9 - -package roaring - -// bit population count, take from -// https://code.google.com/p/go/issues/detail?id=4988#c11 -// credit: https://code.google.com/u/arnehormann/ -// credit: https://play.golang.org/p/U7SogJ7psJ -// credit: http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel -func popcount(x uint64) uint64 { - x -= (x >> 1) & 0x5555555555555555 - x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 - x += x >> 4 - x &= 0x0f0f0f0f0f0f0f0f - x *= 0x0101010101010101 - return x >> 56 -} diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go deleted file mode 100644 index 4ae6d5afa4..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build !amd64 || appengine || go1.9 -// +build !amd64 appengine go1.9 - -package roaring - -func popcntSlice(s []uint64) uint64 { - return popcntSliceGo(s) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - return popcntMaskSliceGo(s, m) -} - -func popcntAndSlice(s, m []uint64) uint64 { - return popcntAndSliceGo(s, m) -} - -func popcntOrSlice(s, m []uint64) uint64 { - return popcntOrSliceGo(s, m) -} - -func popcntXorSlice(s, m []uint64) uint64 { - return popcntXorSliceGo(s, m) -} diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_slices.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_slices.go deleted file mode 100644 index d27c5f383d..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/popcnt_slices.go +++ /dev/null @@ -1,41 +0,0 @@ -package roaring - -func popcntSliceGo(s []uint64) uint64 { - cnt := uint64(0) - for _, x := range s { - cnt += popcount(x) - } - return cnt -} - -func popcntMaskSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcount(s[i] &^ m[i]) - } - return cnt -} - -func popcntAndSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcount(s[i] & m[i]) - } - return cnt -} - -func popcntOrSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcount(s[i] | m[i]) - } - return cnt -} - -func popcntXorSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcount(s[i] ^ m[i]) - } - return cnt -} diff --git a/vendor/github.com/RoaringBitmap/roaring/priorityqueue.go b/vendor/github.com/RoaringBitmap/roaring/priorityqueue.go deleted file mode 100644 index 9259a68163..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/priorityqueue.go +++ /dev/null @@ -1,101 +0,0 @@ -package roaring - -import "container/heap" - -///////////// -// The priorityQueue is used to keep Bitmaps sorted. -//////////// - -type item struct { - value *Bitmap - index int -} - -type priorityQueue []*item - -func (pq priorityQueue) Len() int { return len(pq) } - -func (pq priorityQueue) Less(i, j int) bool { - return pq[i].value.GetSizeInBytes() < pq[j].value.GetSizeInBytes() -} - -func (pq priorityQueue) Swap(i, j int) { - pq[i], pq[j] = pq[j], pq[i] - pq[i].index = i - pq[j].index = j -} - -func (pq *priorityQueue) Push(x interface{}) { - n := len(*pq) - item := x.(*item) - item.index = n - *pq = append(*pq, item) -} - -func (pq *priorityQueue) Pop() interface{} { - old := *pq - n := len(old) - item := old[n-1] - item.index = -1 // for safety - *pq = old[0 : n-1] - return item -} - -func (pq *priorityQueue) update(item *item, value *Bitmap) { - item.value = value - heap.Fix(pq, item.index) -} - -///////////// -// The containerPriorityQueue is used to keep the containers of various Bitmaps sorted. -//////////// - -type containeritem struct { - value *Bitmap - keyindex int - index int -} - -type containerPriorityQueue []*containeritem - -func (pq containerPriorityQueue) Len() int { return len(pq) } - -func (pq containerPriorityQueue) Less(i, j int) bool { - k1 := pq[i].value.highlowcontainer.getKeyAtIndex(pq[i].keyindex) - k2 := pq[j].value.highlowcontainer.getKeyAtIndex(pq[j].keyindex) - if k1 != k2 { - return k1 < k2 - } - c1 := pq[i].value.highlowcontainer.getContainerAtIndex(pq[i].keyindex) - c2 := pq[j].value.highlowcontainer.getContainerAtIndex(pq[j].keyindex) - - return c1.getCardinality() > c2.getCardinality() -} - -func (pq containerPriorityQueue) Swap(i, j int) { - pq[i], pq[j] = pq[j], pq[i] - pq[i].index = i - pq[j].index = j -} - -func (pq *containerPriorityQueue) Push(x interface{}) { - n := len(*pq) - item := x.(*containeritem) - item.index = n - *pq = append(*pq, item) -} - -func (pq *containerPriorityQueue) Pop() interface{} { - old := *pq - n := len(old) - item := old[n-1] - item.index = -1 // for safety - *pq = old[0 : n-1] - return item -} - -//func (pq *containerPriorityQueue) update(item *containeritem, value *Bitmap, keyindex int) { -// item.value = value -// item.keyindex = keyindex -// heap.Fix(pq, item.index) -//} diff --git a/vendor/github.com/RoaringBitmap/roaring/roaring.go b/vendor/github.com/RoaringBitmap/roaring/roaring.go deleted file mode 100644 index a31cdbd9e8..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/roaring.go +++ /dev/null @@ -1,1918 +0,0 @@ -// Package roaring is an implementation of Roaring Bitmaps in Go. -// They provide fast compressed bitmap data structures (also called bitset). -// They are ideally suited to represent sets of integers over -// relatively small ranges. -// See http://roaringbitmap.org for details. -package roaring - -import ( - "bytes" - "encoding/base64" - "fmt" - "io" - "strconv" - - "github.com/RoaringBitmap/roaring/internal" - "github.com/bits-and-blooms/bitset" -) - -// Bitmap represents a compressed bitmap where you can add integers. -type Bitmap struct { - highlowcontainer roaringArray -} - -// ToBase64 serializes a bitmap as Base64 -func (rb *Bitmap) ToBase64() (string, error) { - buf := new(bytes.Buffer) - _, err := rb.WriteTo(buf) - return base64.StdEncoding.EncodeToString(buf.Bytes()), err - -} - -// FromBase64 deserializes a bitmap from Base64 -func (rb *Bitmap) FromBase64(str string) (int64, error) { - data, err := base64.StdEncoding.DecodeString(str) - if err != nil { - return 0, err - } - buf := bytes.NewBuffer(data) - - return rb.ReadFrom(buf) -} - -// WriteTo writes a serialized version of this bitmap to stream. -// The format is compatible with other RoaringBitmap -// implementations (Java, C) and is documented here: -// https://github.com/RoaringBitmap/RoaringFormatSpec -func (rb *Bitmap) WriteTo(stream io.Writer) (int64, error) { - return rb.highlowcontainer.writeTo(stream) -} - -// ToBytes returns an array of bytes corresponding to what is written -// when calling WriteTo -func (rb *Bitmap) ToBytes() ([]byte, error) { - return rb.highlowcontainer.toBytes() -} - -const wordSize = uint64(64) -const log2WordSize = uint64(6) -const capacity = ^uint64(0) -const bitmapContainerSize = (1 << 16) / 64 // bitmap size in words - -// DenseSize returns the size of the bitmap when stored as a dense bitmap. -func (rb *Bitmap) DenseSize() uint64 { - if rb.highlowcontainer.size() == 0 { - return 0 - } - - maximum := 1 + uint64(rb.Maximum()) - if maximum > (capacity - wordSize + 1) { - return uint64(capacity >> log2WordSize) - } - - return uint64((maximum + (wordSize - 1)) >> log2WordSize) -} - -// ToDense returns a slice of uint64s representing the bitmap as a dense bitmap. -// Useful to convert a roaring bitmap to a format that can be used by other libraries -// like https://github.com/bits-and-blooms/bitset or https://github.com/kelindar/bitmap -func (rb *Bitmap) ToDense() []uint64 { - sz := rb.DenseSize() - if sz == 0 { - return nil - } - - bitmap := make([]uint64, sz) - rb.WriteDenseTo(bitmap) - return bitmap -} - -// FromDense creates a bitmap from a slice of uint64s representing the bitmap as a dense bitmap. -// Useful to convert bitmaps from libraries like https://github.com/bits-and-blooms/bitset or -// https://github.com/kelindar/bitmap into roaring bitmaps fast and with convenience. -// -// This function will not create any run containers, only array and bitmap containers. It's up to -// the caller to call RunOptimize if they want to further compress the runs of consecutive values. -// -// When doCopy is true, the bitmap is copied into a new slice for each bitmap container. -// This is useful when the bitmap is going to be modified after this function returns or if it's -// undesirable to hold references to large bitmaps which the GC would not be able to collect. -// One copy can still happen even when doCopy is false if the bitmap length is not divisible -// by bitmapContainerSize. -// -// See also FromBitSet. -func FromDense(bitmap []uint64, doCopy bool) *Bitmap { - sz := (len(bitmap) + bitmapContainerSize - 1) / bitmapContainerSize // round up - rb := &Bitmap{ - highlowcontainer: roaringArray{ - containers: make([]container, 0, sz), - keys: make([]uint16, 0, sz), - needCopyOnWrite: make([]bool, 0, sz), - }, - } - rb.FromDense(bitmap, doCopy) - return rb -} - -// FromDense unmarshalls from a slice of uint64s representing the bitmap as a dense bitmap. -// Useful to convert bitmaps from libraries like https://github.com/bits-and-blooms/bitset or -// https://github.com/kelindar/bitmap into roaring bitmaps fast and with convenience. -// Callers are responsible for ensuring that the bitmap is empty before calling this function. -// -// This function will not create any run containers, only array and bitmap containers. It is up to -// the caller to call RunOptimize if they want to further compress the runs of consecutive values. -// -// When doCopy is true, the bitmap is copied into a new slice for each bitmap container. -// This is useful when the bitmap is going to be modified after this function returns or if it's -// undesirable to hold references to large bitmaps which the GC would not be able to collect. -// One copy can still happen even when doCopy is false if the bitmap length is not divisible -// by bitmapContainerSize. -// -// See FromBitSet. -func (rb *Bitmap) FromDense(bitmap []uint64, doCopy bool) { - if len(bitmap) == 0 { - return - } - - var k uint16 - const size = bitmapContainerSize - - for len(bitmap) > 0 { - hi := size - if len(bitmap) < size { - hi = len(bitmap) - } - - words := bitmap[:hi] - count := int(popcntSlice(words)) - - switch { - case count > arrayDefaultMaxSize: - c := &bitmapContainer{cardinality: count, bitmap: words} - cow := true - - if doCopy || len(words) < size { - c.bitmap = make([]uint64, size) - copy(c.bitmap, words) - cow = false - } - - rb.highlowcontainer.appendContainer(k, c, cow) - - case count > 0: - c := &arrayContainer{content: make([]uint16, count)} - var pos, base int - for _, w := range words { - for w != 0 { - t := w & -w - c.content[pos] = uint16(base + int(popcount(t-1))) - pos++ - w ^= t - } - base += 64 - } - rb.highlowcontainer.appendContainer(k, c, false) - } - - bitmap = bitmap[hi:] - k++ - } -} - -// WriteDenseTo writes to a slice of uint64s representing the bitmap as a dense bitmap. -// Callers are responsible for allocating enough space in the bitmap using DenseSize. -// Useful to convert a roaring bitmap to a format that can be used by other libraries -// like https://github.com/bits-and-blooms/bitset or https://github.com/kelindar/bitmap -func (rb *Bitmap) WriteDenseTo(bitmap []uint64) { - for i, ct := range rb.highlowcontainer.containers { - hb := uint32(rb.highlowcontainer.keys[i]) << 16 - - switch c := ct.(type) { - case *arrayContainer: - for _, x := range c.content { - n := int(hb | uint32(x)) - bitmap[n>>log2WordSize] |= uint64(1) << uint(x%64) - } - - case *bitmapContainer: - copy(bitmap[int(hb)>>log2WordSize:], c.bitmap) - - case *runContainer16: - for j := range c.iv { - start := uint32(c.iv[j].start) - end := start + uint32(c.iv[j].length) + 1 - lo := int(hb|start) >> log2WordSize - hi := int(hb|(end-1)) >> log2WordSize - - if lo == hi { - bitmap[lo] |= (^uint64(0) << uint(start%64)) & - (^uint64(0) >> (uint(-end) % 64)) - continue - } - - bitmap[lo] |= ^uint64(0) << uint(start%64) - for n := lo + 1; n < hi; n++ { - bitmap[n] = ^uint64(0) - } - bitmap[hi] |= ^uint64(0) >> (uint(-end) % 64) - } - default: - panic("unsupported container type") - } - } -} - -// Checksum computes a hash (currently FNV-1a) for a bitmap that is suitable for -// using bitmaps as elements in hash sets or as keys in hash maps, as well as -// generally quicker comparisons. -// The implementation is biased towards efficiency in little endian machines, so -// expect some extra CPU cycles and memory to be used if your machine is big endian. -// Likewise, do not use this to verify integrity unless you are certain you will load -// the bitmap on a machine with the same endianess used to create it. (Thankfully -// very few people use big endian machines these days.) -func (rb *Bitmap) Checksum() uint64 { - const ( - offset = 14695981039346656037 - prime = 1099511628211 - ) - - var bytes []byte - - hash := uint64(offset) - - bytes = uint16SliceAsByteSlice(rb.highlowcontainer.keys) - - for _, b := range bytes { - hash ^= uint64(b) - hash *= prime - } - - for _, c := range rb.highlowcontainer.containers { - // 0 separator - hash ^= 0 - hash *= prime - - switch c := c.(type) { - case *bitmapContainer: - bytes = uint64SliceAsByteSlice(c.bitmap) - case *arrayContainer: - bytes = uint16SliceAsByteSlice(c.content) - case *runContainer16: - bytes = interval16SliceAsByteSlice(c.iv) - default: - panic("invalid container type") - } - - if len(bytes) == 0 { - panic("empty containers are not supported") - } - - for _, b := range bytes { - hash ^= uint64(b) - hash *= prime - } - } - - return hash -} - -// FromUnsafeBytes reads a serialized version of this bitmap from the byte buffer without copy. -// It is the caller's responsibility to ensure that the input data is not modified and remains valid for the entire lifetime of this bitmap. -// This method avoids small allocations but holds references to the input data buffer. It is GC-friendly, but it may consume more memory eventually. -// The containers in the resulting bitmap are immutable containers tied to the provided byte array and they rely on -// copy-on-write which means that modifying them creates copies. Thus FromUnsafeBytes is more likely to be appropriate for read-only use cases, -// when the resulting bitmap can be considered immutable. -// -// See also the FromBuffer function. -// See https://github.com/RoaringBitmap/roaring/pull/395 for more details. -func (rb *Bitmap) FromUnsafeBytes(data []byte, cookieHeader ...byte) (p int64, err error) { - stream := internal.NewByteBuffer(data) - return rb.ReadFrom(stream) -} - -// ReadFrom reads a serialized version of this bitmap from stream. -// The format is compatible with other RoaringBitmap -// implementations (Java, C) and is documented here: -// https://github.com/RoaringBitmap/RoaringFormatSpec -// Since io.Reader is regarded as a stream and cannot be read twice. -// So add cookieHeader to accept the 4-byte data that has been read in roaring64.ReadFrom. -// It is not necessary to pass cookieHeader when call roaring.ReadFrom to read the roaring32 data directly. -func (rb *Bitmap) ReadFrom(reader io.Reader, cookieHeader ...byte) (p int64, err error) { - stream, ok := reader.(internal.ByteInput) - if !ok { - byteInputAdapter := internal.ByteInputAdapterPool.Get().(*internal.ByteInputAdapter) - byteInputAdapter.Reset(reader) - stream = byteInputAdapter - } - - p, err = rb.highlowcontainer.readFrom(stream, cookieHeader...) - - if !ok { - internal.ByteInputAdapterPool.Put(stream.(*internal.ByteInputAdapter)) - } - return -} - -// FromBuffer creates a bitmap from its serialized version stored in buffer -// -// The format specification is available here: -// https://github.com/RoaringBitmap/RoaringFormatSpec -// -// The provided byte array (buf) is expected to be a constant. -// The function makes the best effort attempt not to copy data. -// You should take care not to modify buff as it will -// likely result in unexpected program behavior. -// -// Resulting bitmaps are effectively immutable in the following sense: -// a copy-on-write marker is used so that when you modify the resulting -// bitmap, copies of selected data (containers) are made. -// You should *not* change the copy-on-write status of the resulting -// bitmaps (SetCopyOnWrite). -// -// Thus FromBuffer is more likely to be appropriate for read-only use cases, -// when the resulting bitmap can be considered immutable. -// -// If buf becomes unavailable, then a bitmap created with -// FromBuffer would be effectively broken. Furthermore, any -// bitmap derived from this bitmap (e.g., via Or, And) might -// also be broken. Thus, before making buf unavailable, you should -// call CloneCopyOnWriteContainers on all such bitmaps. -// -// See also the FromUnsafeBytes function which can have better performance -// in some cases. -func (rb *Bitmap) FromBuffer(buf []byte) (p int64, err error) { - stream := internal.ByteBufferPool.Get().(*internal.ByteBuffer) - stream.Reset(buf) - - p, err = rb.highlowcontainer.readFrom(stream) - internal.ByteBufferPool.Put(stream) - - return -} - -// RunOptimize attempts to further compress the runs of consecutive values found in the bitmap -func (rb *Bitmap) RunOptimize() { - rb.highlowcontainer.runOptimize() -} - -// HasRunCompression returns true if the bitmap benefits from run compression -func (rb *Bitmap) HasRunCompression() bool { - return rb.highlowcontainer.hasRunCompression() -} - -// MarshalBinary implements the encoding.BinaryMarshaler interface for the bitmap -// (same as ToBytes) -func (rb *Bitmap) MarshalBinary() ([]byte, error) { - return rb.ToBytes() -} - -// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface for the bitmap -func (rb *Bitmap) UnmarshalBinary(data []byte) error { - r := bytes.NewReader(data) - _, err := rb.ReadFrom(r) - return err -} - -// NewBitmap creates a new empty Bitmap (see also New) -func NewBitmap() *Bitmap { - return &Bitmap{} -} - -// New creates a new empty Bitmap (same as NewBitmap) -func New() *Bitmap { - return &Bitmap{} -} - -// Clear resets the Bitmap to be logically empty, but may retain -// some memory allocations that may speed up future operations -func (rb *Bitmap) Clear() { - rb.highlowcontainer.clear() -} - -// ToBitSet copies the content of the RoaringBitmap into a bitset.BitSet instance -func (rb *Bitmap) ToBitSet() *bitset.BitSet { - return bitset.From(rb.ToDense()) -} - -// FromBitSet creates a new RoaringBitmap from a bitset.BitSet instance -func FromBitSet(bitset *bitset.BitSet) *Bitmap { - return FromDense(bitset.Bytes(), false) -} - -// ToArray creates a new slice containing all of the integers stored in the Bitmap in sorted order -func (rb *Bitmap) ToArray() []uint32 { - array := make([]uint32, rb.GetCardinality()) - pos := 0 - pos2 := 0 - - for pos < rb.highlowcontainer.size() { - hs := uint32(rb.highlowcontainer.getKeyAtIndex(pos)) << 16 - c := rb.highlowcontainer.getContainerAtIndex(pos) - pos++ - pos2 = c.fillLeastSignificant16bits(array, pos2, hs) - } - return array -} - -// GetSizeInBytes estimates the memory usage of the Bitmap. Note that this -// might differ slightly from the amount of bytes required for persistent storage -func (rb *Bitmap) GetSizeInBytes() uint64 { - size := uint64(8) - for _, c := range rb.highlowcontainer.containers { - size += uint64(2) + uint64(c.getSizeInBytes()) - } - return size -} - -// GetSerializedSizeInBytes computes the serialized size in bytes -// of the Bitmap. It should correspond to the -// number of bytes written when invoking WriteTo. You can expect -// that this function is much cheaper computationally than WriteTo. -func (rb *Bitmap) GetSerializedSizeInBytes() uint64 { - return rb.highlowcontainer.serializedSizeInBytes() -} - -// BoundSerializedSizeInBytes returns an upper bound on the serialized size in bytes -// assuming that one wants to store "cardinality" integers in [0, universe_size) -func BoundSerializedSizeInBytes(cardinality uint64, universeSize uint64) uint64 { - contnbr := (universeSize + uint64(65535)) / uint64(65536) - if contnbr > cardinality { - contnbr = cardinality - // we cannot have more containers than we have values - } - headermax := 8*contnbr + 4 - if 4 > (contnbr+7)/8 { - headermax += 4 - } else { - headermax += (contnbr + 7) / 8 - } - valsarray := uint64(arrayContainerSizeInBytes(int(cardinality))) - valsbitmap := contnbr * uint64(bitmapContainerSizeInBytes()) - valsbest := valsarray - if valsbest > valsbitmap { - valsbest = valsbitmap - } - return valsbest + headermax -} - -// IntIterable allows you to iterate over the values in a Bitmap -type IntIterable interface { - HasNext() bool - Next() uint32 -} - -// IntPeekable allows you to look at the next value without advancing and -// advance as long as the next value is smaller than minval -type IntPeekable interface { - IntIterable - // PeekNext peeks the next value without advancing the iterator - PeekNext() uint32 - // AdvanceIfNeeded advances as long as the next value is smaller than minval - AdvanceIfNeeded(minval uint32) -} - -type intIterator struct { - pos int - hs uint32 - iter shortPeekable - highlowcontainer *roaringArray - - // These embedded iterators per container type help reduce load in the GC. - // This way, instead of making up-to 64k allocations per full iteration - // we get a single allocation and simply reinitialize the appropriate - // iterator and point to it in the generic `iter` member on each key bound. - shortIter shortIterator - runIter runIterator16 - bitmapIter bitmapContainerShortIterator -} - -// HasNext returns true if there are more integers to iterate over -func (ii *intIterator) HasNext() bool { - return ii.pos < ii.highlowcontainer.size() -} - -func (ii *intIterator) init() { - if ii.highlowcontainer.size() > ii.pos { - ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16 - c := ii.highlowcontainer.getContainerAtIndex(ii.pos) - switch t := c.(type) { - case *arrayContainer: - ii.shortIter = shortIterator{t.content, 0} - ii.iter = &ii.shortIter - case *runContainer16: - ii.runIter = runIterator16{rc: t, curIndex: 0, curPosInIndex: 0} - ii.iter = &ii.runIter - case *bitmapContainer: - ii.bitmapIter = bitmapContainerShortIterator{t, t.NextSetBit(0)} - ii.iter = &ii.bitmapIter - } - } -} - -// Next returns the next integer -func (ii *intIterator) Next() uint32 { - x := uint32(ii.iter.next()) | ii.hs - if !ii.iter.hasNext() { - ii.pos = ii.pos + 1 - ii.init() - } - return x -} - -// PeekNext peeks the next value without advancing the iterator -func (ii *intIterator) PeekNext() uint32 { - return uint32(ii.iter.peekNext()&maxLowBit) | ii.hs -} - -// AdvanceIfNeeded advances as long as the next value is smaller than minval -func (ii *intIterator) AdvanceIfNeeded(minval uint32) { - to := minval & 0xffff0000 - - for ii.HasNext() && ii.hs < to { - ii.pos++ - ii.init() - } - - if ii.HasNext() && ii.hs == to { - ii.iter.advanceIfNeeded(lowbits(minval)) - - if !ii.iter.hasNext() { - ii.pos++ - ii.init() - } - } -} - -// IntIterator is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap) -type IntIterator = intIterator - -// Initialize configures the existing iterator so that it can iterate through the values of -// the provided bitmap. -// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove). -func (ii *intIterator) Initialize(a *Bitmap) { - ii.pos = 0 - ii.highlowcontainer = &a.highlowcontainer - ii.init() -} - -type intReverseIterator struct { - pos int - hs uint32 - iter shortIterable - highlowcontainer *roaringArray - - shortIter reverseIterator - runIter runReverseIterator16 - bitmapIter reverseBitmapContainerShortIterator -} - -// HasNext returns true if there are more integers to iterate over -func (ii *intReverseIterator) HasNext() bool { - return ii.pos >= 0 -} - -func (ii *intReverseIterator) init() { - if ii.pos >= 0 { - ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16 - c := ii.highlowcontainer.getContainerAtIndex(ii.pos) - switch t := c.(type) { - case *arrayContainer: - ii.shortIter = reverseIterator{t.content, len(t.content) - 1} - ii.iter = &ii.shortIter - case *runContainer16: - index := int(len(t.iv)) - 1 - pos := uint16(0) - - if index >= 0 { - pos = t.iv[index].length - } - - ii.runIter = runReverseIterator16{rc: t, curIndex: index, curPosInIndex: pos} - ii.iter = &ii.runIter - case *bitmapContainer: - pos := -1 - if t.cardinality > 0 { - pos = int(t.maximum()) - } - ii.bitmapIter = reverseBitmapContainerShortIterator{t, pos} - ii.iter = &ii.bitmapIter - } - } else { - ii.iter = nil - } -} - -// Next returns the next integer -func (ii *intReverseIterator) Next() uint32 { - x := uint32(ii.iter.next()) | ii.hs - if !ii.iter.hasNext() { - ii.pos = ii.pos - 1 - ii.init() - } - return x -} - -// IntReverseIterator is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap) -type IntReverseIterator = intReverseIterator - -// Initialize configures the existing iterator so that it can iterate through the values of -// the provided bitmap. -// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove). -func (ii *intReverseIterator) Initialize(a *Bitmap) { - ii.highlowcontainer = &a.highlowcontainer - ii.pos = a.highlowcontainer.size() - 1 - ii.init() -} - -// ManyIntIterable allows you to iterate over the values in a Bitmap -type ManyIntIterable interface { - // NextMany fills buf up with values, returns how many values were returned - NextMany(buf []uint32) int - // NextMany64 fills up buf with 64 bit values, uses hs as a mask (OR), returns how many values were returned - NextMany64(hs uint64, buf []uint64) int -} - -type manyIntIterator struct { - pos int - hs uint32 - iter manyIterable - highlowcontainer *roaringArray - - shortIter shortIterator - runIter runIterator16 - bitmapIter bitmapContainerManyIterator -} - -func (ii *manyIntIterator) init() { - if ii.highlowcontainer.size() > ii.pos { - ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16 - c := ii.highlowcontainer.getContainerAtIndex(ii.pos) - switch t := c.(type) { - case *arrayContainer: - ii.shortIter = shortIterator{t.content, 0} - ii.iter = &ii.shortIter - case *runContainer16: - ii.runIter = runIterator16{rc: t, curIndex: 0, curPosInIndex: 0} - ii.iter = &ii.runIter - case *bitmapContainer: - ii.bitmapIter = bitmapContainerManyIterator{t, -1, 0} - ii.iter = &ii.bitmapIter - } - } else { - ii.iter = nil - } -} - -func (ii *manyIntIterator) NextMany(buf []uint32) int { - n := 0 - for n < len(buf) { - if ii.iter == nil { - break - } - moreN := ii.iter.nextMany(ii.hs, buf[n:]) - n += moreN - if moreN == 0 { - ii.pos = ii.pos + 1 - ii.init() - } - } - - return n -} - -func (ii *manyIntIterator) NextMany64(hs64 uint64, buf []uint64) int { - n := 0 - for n < len(buf) { - if ii.iter == nil { - break - } - - hs := uint64(ii.hs) | hs64 - moreN := ii.iter.nextMany64(hs, buf[n:]) - n += moreN - if moreN == 0 { - ii.pos = ii.pos + 1 - ii.init() - } - } - - return n -} - -// ManyIntIterator is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap) -type ManyIntIterator = manyIntIterator - -// Initialize configures the existing iterator so that it can iterate through the values of -// the provided bitmap. -// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove). -func (ii *manyIntIterator) Initialize(a *Bitmap) { - ii.pos = 0 - ii.highlowcontainer = &a.highlowcontainer - ii.init() -} - -// String creates a string representation of the Bitmap -func (rb *Bitmap) String() string { - // inspired by https://github.com/fzandona/goroar/ - var buffer bytes.Buffer - start := []byte("{") - buffer.Write(start) - i := rb.Iterator() - counter := 0 - if i.HasNext() { - counter = counter + 1 - buffer.WriteString(strconv.FormatInt(int64(i.Next()), 10)) - } - for i.HasNext() { - buffer.WriteString(",") - counter = counter + 1 - // to avoid exhausting the memory - if counter > 0x40000 { - buffer.WriteString("...") - break - } - buffer.WriteString(strconv.FormatInt(int64(i.Next()), 10)) - } - buffer.WriteString("}") - return buffer.String() -} - -// Iterate iterates over the bitmap, calling the given callback with each value in the bitmap. If the callback returns -// false, the iteration is halted. -// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove). -// There is no guarantee as to what order the values will be iterated. -func (rb *Bitmap) Iterate(cb func(x uint32) bool) { - for i := 0; i < rb.highlowcontainer.size(); i++ { - hs := uint32(rb.highlowcontainer.getKeyAtIndex(i)) << 16 - c := rb.highlowcontainer.getContainerAtIndex(i) - - var shouldContinue bool - // This is hacky but it avoids allocations from invoking an interface method with a closure - switch t := c.(type) { - case *arrayContainer: - shouldContinue = t.iterate(func(x uint16) bool { - return cb(uint32(x) | hs) - }) - case *runContainer16: - shouldContinue = t.iterate(func(x uint16) bool { - return cb(uint32(x) | hs) - }) - case *bitmapContainer: - shouldContinue = t.iterate(func(x uint16) bool { - return cb(uint32(x) | hs) - }) - } - - if !shouldContinue { - break - } - } -} - -// Iterator creates a new IntPeekable to iterate over the integers contained in the bitmap, in sorted order; -// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove). -func (rb *Bitmap) Iterator() IntPeekable { - p := new(intIterator) - p.Initialize(rb) - return p -} - -// ReverseIterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order; -// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove). -func (rb *Bitmap) ReverseIterator() IntIterable { - p := new(intReverseIterator) - p.Initialize(rb) - return p -} - -// ManyIterator creates a new ManyIntIterable to iterate over the integers contained in the bitmap, in sorted order; -// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove). -func (rb *Bitmap) ManyIterator() ManyIntIterable { - p := new(manyIntIterator) - p.Initialize(rb) - return p -} - -// Clone creates a copy of the Bitmap -func (rb *Bitmap) Clone() *Bitmap { - ptr := new(Bitmap) - ptr.highlowcontainer = *rb.highlowcontainer.clone() - return ptr -} - -// Minimum get the smallest value stored in this roaring bitmap, assumes that it is not empty -func (rb *Bitmap) Minimum() uint32 { - if len(rb.highlowcontainer.containers) == 0 { - panic("Empty bitmap") - } - return uint32(rb.highlowcontainer.containers[0].minimum()) | (uint32(rb.highlowcontainer.keys[0]) << 16) -} - -// Maximum get the largest value stored in this roaring bitmap, assumes that it is not empty -func (rb *Bitmap) Maximum() uint32 { - if len(rb.highlowcontainer.containers) == 0 { - panic("Empty bitmap") - } - lastindex := len(rb.highlowcontainer.containers) - 1 - return uint32(rb.highlowcontainer.containers[lastindex].maximum()) | (uint32(rb.highlowcontainer.keys[lastindex]) << 16) -} - -// Contains returns true if the integer is contained in the bitmap -func (rb *Bitmap) Contains(x uint32) bool { - hb := highbits(x) - c := rb.highlowcontainer.getContainer(hb) - return c != nil && c.contains(lowbits(x)) -} - -// ContainsInt returns true if the integer is contained in the bitmap (this is a convenience method, the parameter is casted to uint32 and Contains is called) -func (rb *Bitmap) ContainsInt(x int) bool { - return rb.Contains(uint32(x)) -} - -// Equals returns true if the two bitmaps contain the same integers -func (rb *Bitmap) Equals(o interface{}) bool { - srb, ok := o.(*Bitmap) - if ok { - return srb.highlowcontainer.equals(rb.highlowcontainer) - } - return false -} - -// AddOffset adds the value 'offset' to each and every value in a bitmap, generating a new bitmap in the process -func AddOffset(x *Bitmap, offset uint32) (answer *Bitmap) { - return AddOffset64(x, int64(offset)) -} - -// AddOffset64 adds the value 'offset' to each and every value in a bitmap, generating a new bitmap in the process -// If offset + element is outside of the range [0,2^32), that the element will be dropped -func AddOffset64(x *Bitmap, offset int64) (answer *Bitmap) { - // we need "offset" to be a long because we want to support values - // between -0xFFFFFFFF up to +-0xFFFFFFFF - var containerOffset64 int64 - - if offset < 0 { - containerOffset64 = (offset - (1 << 16) + 1) / (1 << 16) - } else { - containerOffset64 = offset >> 16 - } - - answer = New() - - if containerOffset64 >= (1<<16) || containerOffset64 < -(1<<16) { - return answer - } - - containerOffset := int32(containerOffset64) - inOffset := (uint16)(offset - containerOffset64*(1<<16)) - - if inOffset == 0 { - for pos := 0; pos < x.highlowcontainer.size(); pos++ { - key := int32(x.highlowcontainer.getKeyAtIndex(pos)) - key += containerOffset - - if key >= 0 && key <= MaxUint16 { - c := x.highlowcontainer.getContainerAtIndex(pos).clone() - answer.highlowcontainer.appendContainer(uint16(key), c, false) - } - } - } else { - for pos := 0; pos < x.highlowcontainer.size(); pos++ { - key := int32(x.highlowcontainer.getKeyAtIndex(pos)) - key += containerOffset - - if key+1 < 0 || key > MaxUint16 { - continue - } - - c := x.highlowcontainer.getContainerAtIndex(pos) - lo, hi := c.addOffset(inOffset) - - if lo != nil && key >= 0 { - curSize := answer.highlowcontainer.size() - lastkey := int32(0) - - if curSize > 0 { - lastkey = int32(answer.highlowcontainer.getKeyAtIndex(curSize - 1)) - } - - if curSize > 0 && lastkey == key { - prev := answer.highlowcontainer.getContainerAtIndex(curSize - 1) - orresult := prev.ior(lo) - answer.highlowcontainer.setContainerAtIndex(curSize-1, orresult) - } else { - answer.highlowcontainer.appendContainer(uint16(key), lo, false) - } - } - - if hi != nil && key+1 <= MaxUint16 { - answer.highlowcontainer.appendContainer(uint16(key+1), hi, false) - } - } - } - - return answer -} - -// Add the integer x to the bitmap -func (rb *Bitmap) Add(x uint32) { - hb := highbits(x) - ra := &rb.highlowcontainer - i := ra.getIndex(hb) - if i >= 0 { - var c container - c = ra.getWritableContainerAtIndex(i).iaddReturnMinimized(lowbits(x)) - rb.highlowcontainer.setContainerAtIndex(i, c) - } else { - newac := newArrayContainer() - rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, newac.iaddReturnMinimized(lowbits(x))) - } -} - -// add the integer x to the bitmap, return the container and its index -func (rb *Bitmap) addwithptr(x uint32) (int, container) { - hb := highbits(x) - ra := &rb.highlowcontainer - i := ra.getIndex(hb) - var c container - if i >= 0 { - c = ra.getWritableContainerAtIndex(i).iaddReturnMinimized(lowbits(x)) - rb.highlowcontainer.setContainerAtIndex(i, c) - return i, c - } - newac := newArrayContainer() - c = newac.iaddReturnMinimized(lowbits(x)) - rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, c) - return -i - 1, c -} - -// CheckedAdd adds the integer x to the bitmap and return true if it was added (false if the integer was already present) -func (rb *Bitmap) CheckedAdd(x uint32) bool { - // TODO: add unit tests for this method - hb := highbits(x) - i := rb.highlowcontainer.getIndex(hb) - if i >= 0 { - C := rb.highlowcontainer.getWritableContainerAtIndex(i) - oldcard := C.getCardinality() - C = C.iaddReturnMinimized(lowbits(x)) - rb.highlowcontainer.setContainerAtIndex(i, C) - return C.getCardinality() > oldcard - } - newac := newArrayContainer() - rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, newac.iaddReturnMinimized(lowbits(x))) - return true - -} - -// AddInt adds the integer x to the bitmap (convenience method: the parameter is casted to uint32 and we call Add) -func (rb *Bitmap) AddInt(x int) { - rb.Add(uint32(x)) -} - -// Remove the integer x from the bitmap -func (rb *Bitmap) Remove(x uint32) { - hb := highbits(x) - i := rb.highlowcontainer.getIndex(hb) - if i >= 0 { - c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveReturnMinimized(lowbits(x)) - rb.highlowcontainer.setContainerAtIndex(i, c) - if rb.highlowcontainer.getContainerAtIndex(i).isEmpty() { - rb.highlowcontainer.removeAtIndex(i) - } - } -} - -// CheckedRemove removes the integer x from the bitmap and return true if the integer was effectively removed (and false if the integer was not present) -func (rb *Bitmap) CheckedRemove(x uint32) bool { - // TODO: add unit tests for this method - hb := highbits(x) - i := rb.highlowcontainer.getIndex(hb) - if i >= 0 { - C := rb.highlowcontainer.getWritableContainerAtIndex(i) - oldcard := C.getCardinality() - C = C.iremoveReturnMinimized(lowbits(x)) - rb.highlowcontainer.setContainerAtIndex(i, C) - if rb.highlowcontainer.getContainerAtIndex(i).isEmpty() { - rb.highlowcontainer.removeAtIndex(i) - return true - } - return C.getCardinality() < oldcard - } - return false - -} - -// IsEmpty returns true if the Bitmap is empty (it is faster than doing (GetCardinality() == 0)) -func (rb *Bitmap) IsEmpty() bool { - return rb.highlowcontainer.size() == 0 -} - -// GetCardinality returns the number of integers contained in the bitmap -func (rb *Bitmap) GetCardinality() uint64 { - size := uint64(0) - for _, c := range rb.highlowcontainer.containers { - size += uint64(c.getCardinality()) - } - return size -} - -// Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be GetCardinality()). -// If you pass the smallest value, you get the value 1. If you pass a value that is smaller than the smallest -// value, you get 0. Note that this function differs in convention from the Select function since it -// return 1 and not 0 on the smallest value. -func (rb *Bitmap) Rank(x uint32) uint64 { - size := uint64(0) - for i := 0; i < rb.highlowcontainer.size(); i++ { - key := rb.highlowcontainer.getKeyAtIndex(i) - if key > highbits(x) { - return size - } - if key < highbits(x) { - size += uint64(rb.highlowcontainer.getContainerAtIndex(i).getCardinality()) - } else { - return size + uint64(rb.highlowcontainer.getContainerAtIndex(i).rank(lowbits(x))) - } - } - return size -} - -// Select returns the xth integer in the bitmap. If you pass 0, you get -// the smallest element. Note that this function differs in convention from -// the Rank function which returns 1 on the smallest value. -func (rb *Bitmap) Select(x uint32) (uint32, error) { - remaining := x - for i := 0; i < rb.highlowcontainer.size(); i++ { - c := rb.highlowcontainer.getContainerAtIndex(i) - card := uint32(c.getCardinality()) - if remaining >= card { - remaining -= card - } else { - key := rb.highlowcontainer.getKeyAtIndex(i) - return uint32(key)<<16 + uint32(c.selectInt(uint16(remaining))), nil - } - } - return 0, fmt.Errorf("cannot find %dth integer in a bitmap with only %d items", x, rb.GetCardinality()) -} - -// And computes the intersection between two bitmaps and stores the result in the current bitmap -func (rb *Bitmap) And(x2 *Bitmap) { - pos1 := 0 - pos2 := 0 - intersectionsize := 0 - length1 := rb.highlowcontainer.size() - length2 := x2.highlowcontainer.size() - -main: - for { - if pos1 < length1 && pos2 < length2 { - s1 := rb.highlowcontainer.getKeyAtIndex(pos1) - s2 := x2.highlowcontainer.getKeyAtIndex(pos2) - for { - if s1 == s2 { - c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1) - c2 := x2.highlowcontainer.getContainerAtIndex(pos2) - diff := c1.iand(c2) - if !diff.isEmpty() { - rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false) - intersectionsize++ - } - pos1++ - pos2++ - if (pos1 == length1) || (pos2 == length2) { - break main - } - s1 = rb.highlowcontainer.getKeyAtIndex(pos1) - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } else if s1 < s2 { - pos1 = rb.highlowcontainer.advanceUntil(s2, pos1) - if pos1 == length1 { - break main - } - s1 = rb.highlowcontainer.getKeyAtIndex(pos1) - } else { //s1 > s2 - pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) - if pos2 == length2 { - break main - } - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } - } - } else { - break - } - } - rb.highlowcontainer.resize(intersectionsize) -} - -// OrCardinality returns the cardinality of the union between two bitmaps, bitmaps are not modified -func (rb *Bitmap) OrCardinality(x2 *Bitmap) uint64 { - pos1 := 0 - pos2 := 0 - length1 := rb.highlowcontainer.size() - length2 := x2.highlowcontainer.size() - answer := uint64(0) -main: - for { - if (pos1 < length1) && (pos2 < length2) { - s1 := rb.highlowcontainer.getKeyAtIndex(pos1) - s2 := x2.highlowcontainer.getKeyAtIndex(pos2) - - for { - if s1 < s2 { - answer += uint64(rb.highlowcontainer.getContainerAtIndex(pos1).getCardinality()) - pos1++ - if pos1 == length1 { - break main - } - s1 = rb.highlowcontainer.getKeyAtIndex(pos1) - } else if s1 > s2 { - answer += uint64(x2.highlowcontainer.getContainerAtIndex(pos2).getCardinality()) - pos2++ - if pos2 == length2 { - break main - } - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } else { - // TODO: could be faster if we did not have to materialize the container - answer += uint64(rb.highlowcontainer.getContainerAtIndex(pos1).or(x2.highlowcontainer.getContainerAtIndex(pos2)).getCardinality()) - pos1++ - pos2++ - if (pos1 == length1) || (pos2 == length2) { - break main - } - s1 = rb.highlowcontainer.getKeyAtIndex(pos1) - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } - } - } else { - break - } - } - for ; pos1 < length1; pos1++ { - answer += uint64(rb.highlowcontainer.getContainerAtIndex(pos1).getCardinality()) - } - for ; pos2 < length2; pos2++ { - answer += uint64(x2.highlowcontainer.getContainerAtIndex(pos2).getCardinality()) - } - return answer -} - -// AndCardinality returns the cardinality of the intersection between two bitmaps, bitmaps are not modified -func (rb *Bitmap) AndCardinality(x2 *Bitmap) uint64 { - pos1 := 0 - pos2 := 0 - answer := uint64(0) - length1 := rb.highlowcontainer.size() - length2 := x2.highlowcontainer.size() - -main: - for { - if pos1 < length1 && pos2 < length2 { - s1 := rb.highlowcontainer.getKeyAtIndex(pos1) - s2 := x2.highlowcontainer.getKeyAtIndex(pos2) - for { - if s1 == s2 { - c1 := rb.highlowcontainer.getContainerAtIndex(pos1) - c2 := x2.highlowcontainer.getContainerAtIndex(pos2) - answer += uint64(c1.andCardinality(c2)) - pos1++ - pos2++ - if (pos1 == length1) || (pos2 == length2) { - break main - } - s1 = rb.highlowcontainer.getKeyAtIndex(pos1) - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } else if s1 < s2 { - pos1 = rb.highlowcontainer.advanceUntil(s2, pos1) - if pos1 == length1 { - break main - } - s1 = rb.highlowcontainer.getKeyAtIndex(pos1) - } else { //s1 > s2 - pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) - if pos2 == length2 { - break main - } - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } - } - } else { - break - } - } - return answer -} - -// IntersectsWithInterval checks whether a bitmap 'rb' and an open interval '[x,y)' intersect. -func (rb *Bitmap) IntersectsWithInterval(x, y uint64) bool { - if x >= y { - return false - } - if x > MaxUint32 { - return false - } - - it := intIterator{} - it.Initialize(rb) - it.AdvanceIfNeeded(uint32(x)) - if !it.HasNext() { - return false - } - if uint64(it.Next()) >= y { - return false - } - - return true -} - -// Intersects checks whether two bitmap intersects, bitmaps are not modified -func (rb *Bitmap) Intersects(x2 *Bitmap) bool { - pos1 := 0 - pos2 := 0 - length1 := rb.highlowcontainer.size() - length2 := x2.highlowcontainer.size() - -main: - for { - if pos1 < length1 && pos2 < length2 { - s1 := rb.highlowcontainer.getKeyAtIndex(pos1) - s2 := x2.highlowcontainer.getKeyAtIndex(pos2) - for { - if s1 == s2 { - c1 := rb.highlowcontainer.getContainerAtIndex(pos1) - c2 := x2.highlowcontainer.getContainerAtIndex(pos2) - if c1.intersects(c2) { - return true - } - pos1++ - pos2++ - if (pos1 == length1) || (pos2 == length2) { - break main - } - s1 = rb.highlowcontainer.getKeyAtIndex(pos1) - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } else if s1 < s2 { - pos1 = rb.highlowcontainer.advanceUntil(s2, pos1) - if pos1 == length1 { - break main - } - s1 = rb.highlowcontainer.getKeyAtIndex(pos1) - } else { //s1 > s2 - pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) - if pos2 == length2 { - break main - } - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } - } - } else { - break - } - } - return false -} - -// Xor computes the symmetric difference between two bitmaps and stores the result in the current bitmap -func (rb *Bitmap) Xor(x2 *Bitmap) { - pos1 := 0 - pos2 := 0 - length1 := rb.highlowcontainer.size() - length2 := x2.highlowcontainer.size() - for { - if (pos1 < length1) && (pos2 < length2) { - s1 := rb.highlowcontainer.getKeyAtIndex(pos1) - s2 := x2.highlowcontainer.getKeyAtIndex(pos2) - if s1 < s2 { - pos1 = rb.highlowcontainer.advanceUntil(s2, pos1) - if pos1 == length1 { - break - } - } else if s1 > s2 { - c := x2.highlowcontainer.getWritableContainerAtIndex(pos2) - rb.highlowcontainer.insertNewKeyValueAt(pos1, x2.highlowcontainer.getKeyAtIndex(pos2), c) - length1++ - pos1++ - pos2++ - } else { - // TODO: couple be computed in-place for reduced memory usage - c := rb.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2)) - if !c.isEmpty() { - rb.highlowcontainer.setContainerAtIndex(pos1, c) - pos1++ - } else { - rb.highlowcontainer.removeAtIndex(pos1) - length1-- - } - pos2++ - } - } else { - break - } - } - if pos1 == length1 { - rb.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) - } -} - -// Or computes the union between two bitmaps and stores the result in the current bitmap -func (rb *Bitmap) Or(x2 *Bitmap) { - pos1 := 0 - pos2 := 0 - length1 := rb.highlowcontainer.size() - length2 := x2.highlowcontainer.size() -main: - for (pos1 < length1) && (pos2 < length2) { - s1 := rb.highlowcontainer.getKeyAtIndex(pos1) - s2 := x2.highlowcontainer.getKeyAtIndex(pos2) - - for { - if s1 < s2 { - pos1++ - if pos1 == length1 { - break main - } - s1 = rb.highlowcontainer.getKeyAtIndex(pos1) - } else if s1 > s2 { - rb.highlowcontainer.insertNewKeyValueAt(pos1, s2, x2.highlowcontainer.getContainerAtIndex(pos2).clone()) - pos1++ - length1++ - pos2++ - if pos2 == length2 { - break main - } - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } else { - rb.highlowcontainer.replaceKeyAndContainerAtIndex(pos1, s1, rb.highlowcontainer.getUnionedWritableContainer(pos1, x2.highlowcontainer.getContainerAtIndex(pos2)), false) - pos1++ - pos2++ - if (pos1 == length1) || (pos2 == length2) { - break main - } - s1 = rb.highlowcontainer.getKeyAtIndex(pos1) - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } - } - } - if pos1 == length1 { - rb.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) - } -} - -// AndNot computes the difference between two bitmaps and stores the result in the current bitmap -func (rb *Bitmap) AndNot(x2 *Bitmap) { - pos1 := 0 - pos2 := 0 - intersectionsize := 0 - length1 := rb.highlowcontainer.size() - length2 := x2.highlowcontainer.size() - -main: - for { - if pos1 < length1 && pos2 < length2 { - s1 := rb.highlowcontainer.getKeyAtIndex(pos1) - s2 := x2.highlowcontainer.getKeyAtIndex(pos2) - for { - if s1 == s2 { - c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1) - c2 := x2.highlowcontainer.getContainerAtIndex(pos2) - diff := c1.iandNot(c2) - if !diff.isEmpty() { - rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false) - intersectionsize++ - } - pos1++ - pos2++ - if (pos1 == length1) || (pos2 == length2) { - break main - } - s1 = rb.highlowcontainer.getKeyAtIndex(pos1) - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } else if s1 < s2 { - c1 := rb.highlowcontainer.getContainerAtIndex(pos1) - mustCopyOnWrite := rb.highlowcontainer.needsCopyOnWrite(pos1) - rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, c1, mustCopyOnWrite) - intersectionsize++ - pos1++ - if pos1 == length1 { - break main - } - s1 = rb.highlowcontainer.getKeyAtIndex(pos1) - } else { //s1 > s2 - pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) - if pos2 == length2 { - break main - } - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } - } - } else { - break - } - } - // TODO:implement as a copy - for pos1 < length1 { - c1 := rb.highlowcontainer.getContainerAtIndex(pos1) - s1 := rb.highlowcontainer.getKeyAtIndex(pos1) - mustCopyOnWrite := rb.highlowcontainer.needsCopyOnWrite(pos1) - rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, c1, mustCopyOnWrite) - intersectionsize++ - pos1++ - } - rb.highlowcontainer.resize(intersectionsize) -} - -// Or computes the union between two bitmaps and returns the result -func Or(x1, x2 *Bitmap) *Bitmap { - answer := NewBitmap() - pos1 := 0 - pos2 := 0 - length1 := x1.highlowcontainer.size() - length2 := x2.highlowcontainer.size() -main: - for (pos1 < length1) && (pos2 < length2) { - s1 := x1.highlowcontainer.getKeyAtIndex(pos1) - s2 := x2.highlowcontainer.getKeyAtIndex(pos2) - - for { - if s1 < s2 { - answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1) - pos1++ - if pos1 == length1 { - break main - } - s1 = x1.highlowcontainer.getKeyAtIndex(pos1) - } else if s1 > s2 { - answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2) - pos2++ - if pos2 == length2 { - break main - } - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } else { - - answer.highlowcontainer.appendContainer(s1, x1.highlowcontainer.getContainerAtIndex(pos1).or(x2.highlowcontainer.getContainerAtIndex(pos2)), false) - pos1++ - pos2++ - if (pos1 == length1) || (pos2 == length2) { - break main - } - s1 = x1.highlowcontainer.getKeyAtIndex(pos1) - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } - } - } - if pos1 == length1 { - answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) - } else if pos2 == length2 { - answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1) - } - return answer -} - -// And computes the intersection between two bitmaps and returns the result -func And(x1, x2 *Bitmap) *Bitmap { - answer := NewBitmap() - pos1 := 0 - pos2 := 0 - length1 := x1.highlowcontainer.size() - length2 := x2.highlowcontainer.size() -main: - for pos1 < length1 && pos2 < length2 { - s1 := x1.highlowcontainer.getKeyAtIndex(pos1) - s2 := x2.highlowcontainer.getKeyAtIndex(pos2) - for { - if s1 == s2 { - C := x1.highlowcontainer.getContainerAtIndex(pos1) - C = C.and(x2.highlowcontainer.getContainerAtIndex(pos2)) - - if !C.isEmpty() { - answer.highlowcontainer.appendContainer(s1, C, false) - } - pos1++ - pos2++ - if (pos1 == length1) || (pos2 == length2) { - break main - } - s1 = x1.highlowcontainer.getKeyAtIndex(pos1) - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } else if s1 < s2 { - pos1 = x1.highlowcontainer.advanceUntil(s2, pos1) - if pos1 == length1 { - break main - } - s1 = x1.highlowcontainer.getKeyAtIndex(pos1) - } else { // s1 > s2 - pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) - if pos2 == length2 { - break main - } - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } - } - } - return answer -} - -// Xor computes the symmetric difference between two bitmaps and returns the result -func Xor(x1, x2 *Bitmap) *Bitmap { - answer := NewBitmap() - pos1 := 0 - pos2 := 0 - length1 := x1.highlowcontainer.size() - length2 := x2.highlowcontainer.size() - for { - if (pos1 < length1) && (pos2 < length2) { - s1 := x1.highlowcontainer.getKeyAtIndex(pos1) - s2 := x2.highlowcontainer.getKeyAtIndex(pos2) - if s1 < s2 { - answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1) - pos1++ - } else if s1 > s2 { - answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2) - pos2++ - } else { - c := x1.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2)) - if !c.isEmpty() { - answer.highlowcontainer.appendContainer(s1, c, false) - } - pos1++ - pos2++ - } - } else { - break - } - } - if pos1 == length1 { - answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) - } else if pos2 == length2 { - answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1) - } - return answer -} - -// AndNot computes the difference between two bitmaps and returns the result -func AndNot(x1, x2 *Bitmap) *Bitmap { - answer := NewBitmap() - pos1 := 0 - pos2 := 0 - length1 := x1.highlowcontainer.size() - length2 := x2.highlowcontainer.size() - -main: - for { - if pos1 < length1 && pos2 < length2 { - s1 := x1.highlowcontainer.getKeyAtIndex(pos1) - s2 := x2.highlowcontainer.getKeyAtIndex(pos2) - for { - if s1 < s2 { - answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1) - pos1++ - if pos1 == length1 { - break main - } - s1 = x1.highlowcontainer.getKeyAtIndex(pos1) - } else if s1 == s2 { - c1 := x1.highlowcontainer.getContainerAtIndex(pos1) - c2 := x2.highlowcontainer.getContainerAtIndex(pos2) - diff := c1.andNot(c2) - if !diff.isEmpty() { - answer.highlowcontainer.appendContainer(s1, diff, false) - } - pos1++ - pos2++ - if (pos1 == length1) || (pos2 == length2) { - break main - } - s1 = x1.highlowcontainer.getKeyAtIndex(pos1) - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } else { //s1 > s2 - pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) - if pos2 == length2 { - break main - } - s2 = x2.highlowcontainer.getKeyAtIndex(pos2) - } - } - } else { - break - } - } - if pos2 == length2 { - answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1) - } - return answer -} - -// AddMany add all of the values in dat -func (rb *Bitmap) AddMany(dat []uint32) { - if len(dat) == 0 { - return - } - prev := dat[0] - idx, c := rb.addwithptr(prev) - for _, i := range dat[1:] { - if highbits(prev) == highbits(i) { - c = c.iaddReturnMinimized(lowbits(i)) - rb.highlowcontainer.setContainerAtIndex(idx, c) - } else { - idx, c = rb.addwithptr(i) - } - prev = i - } -} - -// BitmapOf generates a new bitmap filled with the specified integers -func BitmapOf(dat ...uint32) *Bitmap { - ans := NewBitmap() - ans.AddMany(dat) - return ans -} - -// Flip negates the bits in the given range (i.e., [rangeStart,rangeEnd)), any integer present in this range and in the bitmap is removed, -// and any integer present in the range and not in the bitmap is added. -// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range -// while uint64(0x100000000) cannot be represented as a 32-bit value. -func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) { - - if rangeEnd > MaxUint32+1 { - panic("rangeEnd > MaxUint32+1") - } - if rangeStart > MaxUint32+1 { - panic("rangeStart > MaxUint32+1") - } - - if rangeStart >= rangeEnd { - return - } - - hbStart := uint32(highbits(uint32(rangeStart))) - lbStart := uint32(lowbits(uint32(rangeStart))) - hbLast := uint32(highbits(uint32(rangeEnd - 1))) - lbLast := uint32(lowbits(uint32(rangeEnd - 1))) - - var max uint32 = maxLowBit - for hb := hbStart; hb <= hbLast; hb++ { - var containerStart uint32 - if hb == hbStart { - containerStart = uint32(lbStart) - } - containerLast := max - if hb == hbLast { - containerLast = uint32(lbLast) - } - - i := rb.highlowcontainer.getIndex(uint16(hb)) - - if i >= 0 { - c := rb.highlowcontainer.getWritableContainerAtIndex(i).inot(int(containerStart), int(containerLast)+1) - if !c.isEmpty() { - rb.highlowcontainer.setContainerAtIndex(i, c) - } else { - rb.highlowcontainer.removeAtIndex(i) - } - } else { // *think* the range of ones must never be - // empty. - rb.highlowcontainer.insertNewKeyValueAt(-i-1, uint16(hb), rangeOfOnes(int(containerStart), int(containerLast))) - } - } -} - -// FlipInt calls Flip after casting the parameters (convenience method) -func (rb *Bitmap) FlipInt(rangeStart, rangeEnd int) { - rb.Flip(uint64(rangeStart), uint64(rangeEnd)) -} - -// AddRange adds the integers in [rangeStart, rangeEnd) to the bitmap. -// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range -// while uint64(0x100000000) cannot be represented as a 32-bit value. -func (rb *Bitmap) AddRange(rangeStart, rangeEnd uint64) { - if rangeStart >= rangeEnd { - return - } - if rangeEnd-1 > MaxUint32 { - panic("rangeEnd-1 > MaxUint32") - } - hbStart := uint32(highbits(uint32(rangeStart))) - lbStart := uint32(lowbits(uint32(rangeStart))) - hbLast := uint32(highbits(uint32(rangeEnd - 1))) - lbLast := uint32(lowbits(uint32(rangeEnd - 1))) - - var max uint32 = maxLowBit - for hb := hbStart; hb <= hbLast; hb++ { - containerStart := uint32(0) - if hb == hbStart { - containerStart = lbStart - } - containerLast := max - if hb == hbLast { - containerLast = lbLast - } - - i := rb.highlowcontainer.getIndex(uint16(hb)) - - if i >= 0 { - c := rb.highlowcontainer.getWritableContainerAtIndex(i).iaddRange(int(containerStart), int(containerLast)+1) - rb.highlowcontainer.setContainerAtIndex(i, c) - } else { // *think* the range of ones must never be - // empty. - rb.highlowcontainer.insertNewKeyValueAt(-i-1, uint16(hb), rangeOfOnes(int(containerStart), int(containerLast))) - } - } -} - -// RemoveRange removes the integers in [rangeStart, rangeEnd) from the bitmap. -// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range -// while uint64(0x100000000) cannot be represented as a 32-bit value. -func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) { - if rangeStart >= rangeEnd { - return - } - if rangeEnd-1 > MaxUint32 { - // logically, we should assume that the user wants to - // remove all values from rangeStart to infinity - // see https://github.com/RoaringBitmap/roaring/issues/141 - rangeEnd = uint64(0x100000000) - } - hbStart := uint32(highbits(uint32(rangeStart))) - lbStart := uint32(lowbits(uint32(rangeStart))) - hbLast := uint32(highbits(uint32(rangeEnd - 1))) - lbLast := uint32(lowbits(uint32(rangeEnd - 1))) - - var max uint32 = maxLowBit - - if hbStart == hbLast { - i := rb.highlowcontainer.getIndex(uint16(hbStart)) - if i < 0 { - return - } - c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveRange(int(lbStart), int(lbLast+1)) - if !c.isEmpty() { - rb.highlowcontainer.setContainerAtIndex(i, c) - } else { - rb.highlowcontainer.removeAtIndex(i) - } - return - } - ifirst := rb.highlowcontainer.getIndex(uint16(hbStart)) - ilast := rb.highlowcontainer.getIndex(uint16(hbLast)) - - if ifirst >= 0 { - if lbStart != 0 { - c := rb.highlowcontainer.getWritableContainerAtIndex(ifirst).iremoveRange(int(lbStart), int(max+1)) - if !c.isEmpty() { - rb.highlowcontainer.setContainerAtIndex(ifirst, c) - ifirst++ - } - } - } else { - ifirst = -ifirst - 1 - } - if ilast >= 0 { - if lbLast != max { - c := rb.highlowcontainer.getWritableContainerAtIndex(ilast).iremoveRange(int(0), int(lbLast+1)) - if !c.isEmpty() { - rb.highlowcontainer.setContainerAtIndex(ilast, c) - } else { - ilast++ - } - } else { - ilast++ - } - } else { - ilast = -ilast - 1 - } - rb.highlowcontainer.removeIndexRange(ifirst, ilast) -} - -// Flip negates the bits in the given range (i.e., [rangeStart,rangeEnd)), any integer present in this range and in the bitmap is removed, -// and any integer present in the range and not in the bitmap is added, a new bitmap is returned leaving -// the current bitmap unchanged. -// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range -// while uint64(0x100000000) cannot be represented as a 32-bit value. -func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap { - if rangeStart >= rangeEnd { - return bm.Clone() - } - - if rangeStart > MaxUint32 { - panic("rangeStart > MaxUint32") - } - if rangeEnd-1 > MaxUint32 { - panic("rangeEnd-1 > MaxUint32") - } - - answer := NewBitmap() - hbStart := uint32(highbits(uint32(rangeStart))) - lbStart := uint32(lowbits(uint32(rangeStart))) - hbLast := uint32(highbits(uint32(rangeEnd - 1))) - lbLast := uint32(lowbits(uint32(rangeEnd - 1))) - - // copy the containers before the active area - answer.highlowcontainer.appendCopiesUntil(bm.highlowcontainer, uint16(hbStart)) - - var max uint32 = maxLowBit - for hb := hbStart; hb <= hbLast; hb++ { - var containerStart uint32 - if hb == hbStart { - containerStart = uint32(lbStart) - } - containerLast := max - if hb == hbLast { - containerLast = uint32(lbLast) - } - - i := bm.highlowcontainer.getIndex(uint16(hb)) - j := answer.highlowcontainer.getIndex(uint16(hb)) - - if i >= 0 { - c := bm.highlowcontainer.getContainerAtIndex(i).not(int(containerStart), int(containerLast)+1) - if !c.isEmpty() { - answer.highlowcontainer.insertNewKeyValueAt(-j-1, uint16(hb), c) - } - - } else { // *think* the range of ones must never be - // empty. - answer.highlowcontainer.insertNewKeyValueAt(-j-1, uint16(hb), - rangeOfOnes(int(containerStart), int(containerLast))) - } - } - // copy the containers after the active area. - answer.highlowcontainer.appendCopiesAfter(bm.highlowcontainer, uint16(hbLast)) - - return answer -} - -// SetCopyOnWrite sets this bitmap to use copy-on-write so that copies are fast and memory conscious -// if the parameter is true, otherwise we leave the default where hard copies are made -// (copy-on-write requires extra care in a threaded context). -// Calling SetCopyOnWrite(true) on a bitmap created with FromBuffer is unsafe. -func (rb *Bitmap) SetCopyOnWrite(val bool) { - rb.highlowcontainer.copyOnWrite = val -} - -// GetCopyOnWrite gets this bitmap's copy-on-write property -func (rb *Bitmap) GetCopyOnWrite() (val bool) { - return rb.highlowcontainer.copyOnWrite -} - -// CloneCopyOnWriteContainers clones all containers which have -// needCopyOnWrite set to true. -// This can be used to make sure it is safe to munmap a []byte -// that the roaring array may still have a reference to, after -// calling FromBuffer. -// More generally this function is useful if you call FromBuffer -// to construct a bitmap with a backing array buf -// and then later discard the buf array. Note that you should call -// CloneCopyOnWriteContainers on all bitmaps that were derived -// from the 'FromBuffer' bitmap since they map have dependencies -// on the buf array as well. -func (rb *Bitmap) CloneCopyOnWriteContainers() { - rb.highlowcontainer.cloneCopyOnWriteContainers() -} - -// FlipInt calls Flip after casting the parameters (convenience method) -func FlipInt(bm *Bitmap, rangeStart, rangeEnd int) *Bitmap { - return Flip(bm, uint64(rangeStart), uint64(rangeEnd)) -} - -// Statistics provides details on the container types in use. -type Statistics struct { - Cardinality uint64 - Containers uint64 - - ArrayContainers uint64 - ArrayContainerBytes uint64 - ArrayContainerValues uint64 - - BitmapContainers uint64 - BitmapContainerBytes uint64 - BitmapContainerValues uint64 - - RunContainers uint64 - RunContainerBytes uint64 - RunContainerValues uint64 -} - -// Stats returns details on container type usage in a Statistics struct. -func (rb *Bitmap) Stats() Statistics { - stats := Statistics{} - stats.Containers = uint64(len(rb.highlowcontainer.containers)) - for _, c := range rb.highlowcontainer.containers { - stats.Cardinality += uint64(c.getCardinality()) - - switch c.(type) { - case *arrayContainer: - stats.ArrayContainers++ - stats.ArrayContainerBytes += uint64(c.getSizeInBytes()) - stats.ArrayContainerValues += uint64(c.getCardinality()) - case *bitmapContainer: - stats.BitmapContainers++ - stats.BitmapContainerBytes += uint64(c.getSizeInBytes()) - stats.BitmapContainerValues += uint64(c.getCardinality()) - case *runContainer16: - stats.RunContainers++ - stats.RunContainerBytes += uint64(c.getSizeInBytes()) - stats.RunContainerValues += uint64(c.getCardinality()) - } - } - return stats -} diff --git a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go deleted file mode 100644 index 079195ddaf..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go +++ /dev/null @@ -1,761 +0,0 @@ -package roaring - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - - "github.com/RoaringBitmap/roaring/internal" -) - -type container interface { - // addOffset returns the (low, high) parts of the shifted container. - // Whenever one of them would be empty, nil will be returned instead to - // avoid unnecessary allocations. - addOffset(uint16) (container, container) - - clone() container - and(container) container - andCardinality(container) int - iand(container) container // i stands for inplace - andNot(container) container - iandNot(container) container // i stands for inplace - isEmpty() bool - getCardinality() int - // rank returns the number of integers that are - // smaller or equal to x. rank(infinity) would be getCardinality(). - rank(uint16) int - - iadd(x uint16) bool // inplace, returns true if x was new. - iaddReturnMinimized(uint16) container // may change return type to minimize storage. - - //addRange(start, final int) container // range is [firstOfRange,lastOfRange) (unused) - iaddRange(start, endx int) container // i stands for inplace, range is [firstOfRange,endx) - - iremove(x uint16) bool // inplace, returns true if x was present. - iremoveReturnMinimized(uint16) container // may change return type to minimize storage. - - not(start, final int) container // range is [firstOfRange,lastOfRange) - inot(firstOfRange, endx int) container // i stands for inplace, range is [firstOfRange,endx) - xor(r container) container - getShortIterator() shortPeekable - iterate(cb func(x uint16) bool) bool - getReverseIterator() shortIterable - getManyIterator() manyIterable - contains(i uint16) bool - maximum() uint16 - minimum() uint16 - - // equals is now logical equals; it does not require the - // same underlying container types, but compares across - // any of the implementations. - equals(r container) bool - - fillLeastSignificant16bits(array []uint32, i int, mask uint32) int - or(r container) container - orCardinality(r container) int - isFull() bool - ior(r container) container // i stands for inplace - intersects(r container) bool // whether the two containers intersect - lazyOR(r container) container - lazyIOR(r container) container - getSizeInBytes() int - //removeRange(start, final int) container // range is [firstOfRange,lastOfRange) (unused) - iremoveRange(start, final int) container // i stands for inplace, range is [firstOfRange,lastOfRange) - selectInt(x uint16) int // selectInt returns the xth integer in the container - serializedSizeInBytes() int - writeTo(io.Writer) (int, error) - - numberOfRuns() int - toEfficientContainer() container - String() string - containerType() contype -} - -type contype uint8 - -const ( - bitmapContype contype = iota - arrayContype - run16Contype - run32Contype -) - -// careful: range is [firstOfRange,lastOfRange] -func rangeOfOnes(start, last int) container { - if start > MaxUint16 { - panic("rangeOfOnes called with start > MaxUint16") - } - if last > MaxUint16 { - panic("rangeOfOnes called with last > MaxUint16") - } - if start < 0 { - panic("rangeOfOnes called with start < 0") - } - if last < 0 { - panic("rangeOfOnes called with last < 0") - } - return newRunContainer16Range(uint16(start), uint16(last)) -} - -type roaringArray struct { - keys []uint16 - containers []container `msg:"-"` // don't try to serialize directly. - needCopyOnWrite []bool - copyOnWrite bool -} - -func newRoaringArray() *roaringArray { - return &roaringArray{} -} - -// runOptimize compresses the element containers to minimize space consumed. -// Q: how does this interact with copyOnWrite and needCopyOnWrite? -// A: since we aren't changing the logical content, just the representation, -// -// we don't bother to check the needCopyOnWrite bits. We replace -// (possibly all) elements of ra.containers in-place with space -// optimized versions. -func (ra *roaringArray) runOptimize() { - for i := range ra.containers { - ra.containers[i] = ra.containers[i].toEfficientContainer() - } -} - -func (ra *roaringArray) appendContainer(key uint16, value container, mustCopyOnWrite bool) { - ra.keys = append(ra.keys, key) - ra.containers = append(ra.containers, value) - ra.needCopyOnWrite = append(ra.needCopyOnWrite, mustCopyOnWrite) -} - -func (ra *roaringArray) appendWithoutCopy(sa roaringArray, startingindex int) { - mustCopyOnWrite := sa.needCopyOnWrite[startingindex] - ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex], mustCopyOnWrite) -} - -func (ra *roaringArray) appendCopy(sa roaringArray, startingindex int) { - // cow only if the two request it, or if we already have a lightweight copy - copyonwrite := (ra.copyOnWrite && sa.copyOnWrite) || sa.needsCopyOnWrite(startingindex) - if !copyonwrite { - // since there is no copy-on-write, we need to clone the container (this is important) - ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex].clone(), copyonwrite) - } else { - ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex], copyonwrite) - if !sa.needsCopyOnWrite(startingindex) { - sa.setNeedsCopyOnWrite(startingindex) - } - } -} - -func (ra *roaringArray) appendWithoutCopyMany(sa roaringArray, startingindex, end int) { - for i := startingindex; i < end; i++ { - ra.appendWithoutCopy(sa, i) - } -} - -func (ra *roaringArray) appendCopyMany(sa roaringArray, startingindex, end int) { - for i := startingindex; i < end; i++ { - ra.appendCopy(sa, i) - } -} - -func (ra *roaringArray) appendCopiesUntil(sa roaringArray, stoppingKey uint16) { - // cow only if the two request it, or if we already have a lightweight copy - copyonwrite := ra.copyOnWrite && sa.copyOnWrite - - for i := 0; i < sa.size(); i++ { - if sa.keys[i] >= stoppingKey { - break - } - thiscopyonewrite := copyonwrite || sa.needsCopyOnWrite(i) - if thiscopyonewrite { - ra.appendContainer(sa.keys[i], sa.containers[i], thiscopyonewrite) - if !sa.needsCopyOnWrite(i) { - sa.setNeedsCopyOnWrite(i) - } - - } else { - // since there is no copy-on-write, we need to clone the container (this is important) - ra.appendContainer(sa.keys[i], sa.containers[i].clone(), thiscopyonewrite) - - } - } -} - -func (ra *roaringArray) appendCopiesAfter(sa roaringArray, beforeStart uint16) { - // cow only if the two request it, or if we already have a lightweight copy - copyonwrite := ra.copyOnWrite && sa.copyOnWrite - - startLocation := sa.getIndex(beforeStart) - if startLocation >= 0 { - startLocation++ - } else { - startLocation = -startLocation - 1 - } - - for i := startLocation; i < sa.size(); i++ { - thiscopyonewrite := copyonwrite || sa.needsCopyOnWrite(i) - if thiscopyonewrite { - ra.appendContainer(sa.keys[i], sa.containers[i], thiscopyonewrite) - if !sa.needsCopyOnWrite(i) { - sa.setNeedsCopyOnWrite(i) - } - } else { - // since there is no copy-on-write, we need to clone the container (this is important) - ra.appendContainer(sa.keys[i], sa.containers[i].clone(), thiscopyonewrite) - - } - } -} - -func (ra *roaringArray) removeIndexRange(begin, end int) { - if end <= begin { - return - } - - r := end - begin - - copy(ra.keys[begin:], ra.keys[end:]) - copy(ra.containers[begin:], ra.containers[end:]) - copy(ra.needCopyOnWrite[begin:], ra.needCopyOnWrite[end:]) - - ra.resize(len(ra.keys) - r) -} - -func (ra *roaringArray) resize(newsize int) { - for k := newsize; k < len(ra.containers); k++ { - ra.containers[k] = nil - } - - ra.keys = ra.keys[:newsize] - ra.containers = ra.containers[:newsize] - ra.needCopyOnWrite = ra.needCopyOnWrite[:newsize] -} - -func (ra *roaringArray) clear() { - ra.resize(0) - ra.copyOnWrite = false -} - -func (ra *roaringArray) clone() *roaringArray { - - sa := roaringArray{} - sa.copyOnWrite = ra.copyOnWrite - - // this is where copyOnWrite is used. - if ra.copyOnWrite { - sa.keys = make([]uint16, len(ra.keys)) - copy(sa.keys, ra.keys) - sa.containers = make([]container, len(ra.containers)) - copy(sa.containers, ra.containers) - sa.needCopyOnWrite = make([]bool, len(ra.needCopyOnWrite)) - - ra.markAllAsNeedingCopyOnWrite() - sa.markAllAsNeedingCopyOnWrite() - - // sa.needCopyOnWrite is shared - } else { - // make a full copy - - sa.keys = make([]uint16, len(ra.keys)) - copy(sa.keys, ra.keys) - - sa.containers = make([]container, len(ra.containers)) - for i := range sa.containers { - sa.containers[i] = ra.containers[i].clone() - } - - sa.needCopyOnWrite = make([]bool, len(ra.needCopyOnWrite)) - } - return &sa -} - -// clone all containers which have needCopyOnWrite set to true -// This can be used to make sure it is safe to munmap a []byte -// that the roaring array may still have a reference to. -func (ra *roaringArray) cloneCopyOnWriteContainers() { - for i, needCopyOnWrite := range ra.needCopyOnWrite { - if needCopyOnWrite { - ra.containers[i] = ra.containers[i].clone() - ra.needCopyOnWrite[i] = false - } - } -} - -// unused function: -//func (ra *roaringArray) containsKey(x uint16) bool { -// return (ra.binarySearch(0, int64(len(ra.keys)), x) >= 0) -//} - -func (ra *roaringArray) getContainer(x uint16) container { - i := ra.binarySearch(0, int64(len(ra.keys)), x) - if i < 0 { - return nil - } - return ra.containers[i] -} - -func (ra *roaringArray) getContainerAtIndex(i int) container { - return ra.containers[i] -} - -func (ra *roaringArray) getFastContainerAtIndex(i int, needsWriteable bool) container { - c := ra.getContainerAtIndex(i) - switch t := c.(type) { - case *arrayContainer: - c = t.toBitmapContainer() - case *runContainer16: - if !t.isFull() { - c = t.toBitmapContainer() - } - case *bitmapContainer: - if needsWriteable && ra.needCopyOnWrite[i] { - c = ra.containers[i].clone() - } - } - return c -} - -// getUnionedWritableContainer switches behavior for in-place Or -// depending on whether the container requires a copy on write. -// If it does using the non-inplace or() method leads to fewer allocations. -func (ra *roaringArray) getUnionedWritableContainer(pos int, other container) container { - if ra.needCopyOnWrite[pos] { - return ra.getContainerAtIndex(pos).or(other) - } - return ra.getContainerAtIndex(pos).ior(other) - -} - -func (ra *roaringArray) getWritableContainerAtIndex(i int) container { - if ra.needCopyOnWrite[i] { - ra.containers[i] = ra.containers[i].clone() - ra.needCopyOnWrite[i] = false - } - return ra.containers[i] -} - -func (ra *roaringArray) getIndex(x uint16) int { - // before the binary search, we optimize for frequent cases - size := len(ra.keys) - if (size == 0) || (ra.keys[size-1] == x) { - return size - 1 - } - return ra.binarySearch(0, int64(size), x) -} - -func (ra *roaringArray) getKeyAtIndex(i int) uint16 { - return ra.keys[i] -} - -func (ra *roaringArray) insertNewKeyValueAt(i int, key uint16, value container) { - ra.keys = append(ra.keys, 0) - ra.containers = append(ra.containers, nil) - - copy(ra.keys[i+1:], ra.keys[i:]) - copy(ra.containers[i+1:], ra.containers[i:]) - - ra.keys[i] = key - ra.containers[i] = value - - ra.needCopyOnWrite = append(ra.needCopyOnWrite, false) - copy(ra.needCopyOnWrite[i+1:], ra.needCopyOnWrite[i:]) - ra.needCopyOnWrite[i] = false -} - -func (ra *roaringArray) remove(key uint16) bool { - i := ra.binarySearch(0, int64(len(ra.keys)), key) - if i >= 0 { // if a new key - ra.removeAtIndex(i) - return true - } - return false -} - -func (ra *roaringArray) removeAtIndex(i int) { - copy(ra.keys[i:], ra.keys[i+1:]) - copy(ra.containers[i:], ra.containers[i+1:]) - - copy(ra.needCopyOnWrite[i:], ra.needCopyOnWrite[i+1:]) - - ra.resize(len(ra.keys) - 1) -} - -func (ra *roaringArray) setContainerAtIndex(i int, c container) { - ra.containers[i] = c -} - -func (ra *roaringArray) replaceKeyAndContainerAtIndex(i int, key uint16, c container, mustCopyOnWrite bool) { - ra.keys[i] = key - ra.containers[i] = c - ra.needCopyOnWrite[i] = mustCopyOnWrite -} - -func (ra *roaringArray) size() int { - return len(ra.keys) -} - -func (ra *roaringArray) binarySearch(begin, end int64, ikey uint16) int { - low := begin - high := end - 1 - for low+16 <= high { - middleIndex := low + (high-low)/2 // avoid overflow - middleValue := ra.keys[middleIndex] - - if middleValue < ikey { - low = middleIndex + 1 - } else if middleValue > ikey { - high = middleIndex - 1 - } else { - return int(middleIndex) - } - } - for ; low <= high; low++ { - val := ra.keys[low] - if val >= ikey { - if val == ikey { - return int(low) - } - break - } - } - return -int(low + 1) -} - -func (ra *roaringArray) equals(o interface{}) bool { - srb, ok := o.(roaringArray) - if ok { - - if srb.size() != ra.size() { - return false - } - for i, k := range ra.keys { - if k != srb.keys[i] { - return false - } - } - - for i, c := range ra.containers { - if !c.equals(srb.containers[i]) { - return false - } - } - return true - } - return false -} - -func (ra *roaringArray) headerSize() uint64 { - size := uint64(len(ra.keys)) - if ra.hasRunCompression() { - if size < noOffsetThreshold { // for small bitmaps, we omit the offsets - return 4 + (size+7)/8 + 4*size - } - return 4 + (size+7)/8 + 8*size // - 4 because we pack the size with the cookie - } - return 4 + 4 + 8*size - -} - -// should be dirt cheap -func (ra *roaringArray) serializedSizeInBytes() uint64 { - answer := ra.headerSize() - for _, c := range ra.containers { - answer += uint64(c.serializedSizeInBytes()) - } - return answer -} - -// spec: https://github.com/RoaringBitmap/RoaringFormatSpec -func (ra *roaringArray) writeTo(w io.Writer) (n int64, err error) { - hasRun := ra.hasRunCompression() - isRunSizeInBytes := 0 - cookieSize := 8 - if hasRun { - cookieSize = 4 - isRunSizeInBytes = (len(ra.keys) + 7) / 8 - } - descriptiveHeaderSize := 4 * len(ra.keys) - preambleSize := cookieSize + isRunSizeInBytes + descriptiveHeaderSize - - buf := make([]byte, preambleSize+4*len(ra.keys)) - - nw := 0 - - if hasRun { - binary.LittleEndian.PutUint16(buf[0:], uint16(serialCookie)) - nw += 2 - binary.LittleEndian.PutUint16(buf[2:], uint16(len(ra.keys)-1)) - nw += 2 - // compute isRun bitmap without temporary allocation - var runbitmapslice = buf[nw : nw+isRunSizeInBytes] - for i, c := range ra.containers { - switch c.(type) { - case *runContainer16: - runbitmapslice[i/8] |= 1 << (uint(i) % 8) - } - } - nw += isRunSizeInBytes - } else { - binary.LittleEndian.PutUint32(buf[0:], uint32(serialCookieNoRunContainer)) - nw += 4 - binary.LittleEndian.PutUint32(buf[4:], uint32(len(ra.keys))) - nw += 4 - } - - // descriptive header - for i, key := range ra.keys { - binary.LittleEndian.PutUint16(buf[nw:], key) - nw += 2 - c := ra.containers[i] - binary.LittleEndian.PutUint16(buf[nw:], uint16(c.getCardinality()-1)) - nw += 2 - } - - startOffset := int64(preambleSize + 4*len(ra.keys)) - if !hasRun || (len(ra.keys) >= noOffsetThreshold) { - // offset header - for _, c := range ra.containers { - binary.LittleEndian.PutUint32(buf[nw:], uint32(startOffset)) - nw += 4 - switch rc := c.(type) { - case *runContainer16: - startOffset += 2 + int64(len(rc.iv))*4 - default: - startOffset += int64(getSizeInBytesFromCardinality(c.getCardinality())) - } - } - } - - written, err := w.Write(buf[:nw]) - if err != nil { - return n, err - } - n += int64(written) - - for _, c := range ra.containers { - written, err := c.writeTo(w) - if err != nil { - return n, err - } - n += int64(written) - } - return n, nil -} - -// spec: https://github.com/RoaringBitmap/RoaringFormatSpec -func (ra *roaringArray) toBytes() ([]byte, error) { - var buf bytes.Buffer - _, err := ra.writeTo(&buf) - return buf.Bytes(), err -} - -// Reads a serialized roaringArray from a byte slice. -func (ra *roaringArray) readFrom(stream internal.ByteInput, cookieHeader ...byte) (int64, error) { - var cookie uint32 - var err error - if len(cookieHeader) > 0 && len(cookieHeader) != 4 { - return int64(len(cookieHeader)), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: incorrect size of cookie header") - } - if len(cookieHeader) == 4 { - cookie = binary.LittleEndian.Uint32(cookieHeader) - } else { - cookie, err = stream.ReadUInt32() - if err != nil { - return stream.GetReadBytes(), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err) - } - } - // If NextReturnsSafeSlice is false, then willNeedCopyOnWrite should be true - willNeedCopyOnWrite := !stream.NextReturnsSafeSlice() - - var size uint32 - var isRunBitmap []byte - - if cookie&0x0000FFFF == serialCookie { - size = uint32(cookie>>16 + 1) - // create is-run-container bitmap - isRunBitmapSize := (int(size) + 7) / 8 - isRunBitmap, err = stream.Next(isRunBitmapSize) - - if err != nil { - return stream.GetReadBytes(), fmt.Errorf("malformed bitmap, failed to read is-run bitmap, got: %s", err) - } - } else if cookie == serialCookieNoRunContainer { - size, err = stream.ReadUInt32() - if err != nil { - return stream.GetReadBytes(), fmt.Errorf("malformed bitmap, failed to read a bitmap size: %s", err) - } - } else { - return stream.GetReadBytes(), fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header") - } - - if size > (1 << 16) { - return stream.GetReadBytes(), fmt.Errorf("it is logically impossible to have more than (1<<16) containers") - } - - // descriptive header - buf, err := stream.Next(2 * 2 * int(size)) - - if err != nil { - return stream.GetReadBytes(), fmt.Errorf("failed to read descriptive header: %s", err) - } - - keycard := byteSliceAsUint16Slice(buf) - - if isRunBitmap == nil || size >= noOffsetThreshold { - if err := stream.SkipBytes(int(size) * 4); err != nil { - return stream.GetReadBytes(), fmt.Errorf("failed to skip bytes: %s", err) - } - } - - // Allocate slices upfront as number of containers is known - if cap(ra.containers) >= int(size) { - ra.containers = ra.containers[:size] - } else { - ra.containers = make([]container, size) - } - - if cap(ra.keys) >= int(size) { - ra.keys = ra.keys[:size] - } else { - ra.keys = make([]uint16, size) - } - - if cap(ra.needCopyOnWrite) >= int(size) { - ra.needCopyOnWrite = ra.needCopyOnWrite[:size] - } else { - ra.needCopyOnWrite = make([]bool, size) - } - - for i := uint32(0); i < size; i++ { - key := keycard[2*i] - card := int(keycard[2*i+1]) + 1 - ra.keys[i] = key - ra.needCopyOnWrite[i] = willNeedCopyOnWrite - - if isRunBitmap != nil && isRunBitmap[i/8]&(1<<(i%8)) != 0 { - // run container - nr, err := stream.ReadUInt16() - - if err != nil { - return 0, fmt.Errorf("failed to read runtime container size: %s", err) - } - - buf, err := stream.Next(int(nr) * 4) - - if err != nil { - return stream.GetReadBytes(), fmt.Errorf("failed to read runtime container content: %s", err) - } - - nb := runContainer16{ - iv: byteSliceAsInterval16Slice(buf), - } - - ra.containers[i] = &nb - } else if card > arrayDefaultMaxSize { - // bitmap container - buf, err := stream.Next(arrayDefaultMaxSize * 2) - - if err != nil { - return stream.GetReadBytes(), fmt.Errorf("failed to read bitmap container: %s", err) - } - - nb := bitmapContainer{ - cardinality: card, - bitmap: byteSliceAsUint64Slice(buf), - } - - ra.containers[i] = &nb - } else { - // array container - buf, err := stream.Next(card * 2) - - if err != nil { - return stream.GetReadBytes(), fmt.Errorf("failed to read array container: %s", err) - } - - nb := arrayContainer{ - byteSliceAsUint16Slice(buf), - } - - ra.containers[i] = &nb - } - } - - return stream.GetReadBytes(), nil -} - -func (ra *roaringArray) hasRunCompression() bool { - for _, c := range ra.containers { - switch c.(type) { - case *runContainer16: - return true - } - } - return false -} - -func (ra *roaringArray) advanceUntil(min uint16, pos int) int { - lower := pos + 1 - - if lower >= len(ra.keys) || ra.keys[lower] >= min { - return lower - } - - spansize := 1 - - for lower+spansize < len(ra.keys) && ra.keys[lower+spansize] < min { - spansize *= 2 - } - var upper int - if lower+spansize < len(ra.keys) { - upper = lower + spansize - } else { - upper = len(ra.keys) - 1 - } - - if ra.keys[upper] == min { - return upper - } - - if ra.keys[upper] < min { - // means - // array - // has no - // item - // >= min - // pos = array.length; - return len(ra.keys) - } - - // we know that the next-smallest span was too small - lower += (spansize >> 1) - - mid := 0 - for lower+1 != upper { - mid = (lower + upper) >> 1 - if ra.keys[mid] == min { - return mid - } else if ra.keys[mid] < min { - lower = mid - } else { - upper = mid - } - } - return upper -} - -func (ra *roaringArray) markAllAsNeedingCopyOnWrite() { - for i := range ra.needCopyOnWrite { - ra.needCopyOnWrite[i] = true - } -} - -func (ra *roaringArray) needsCopyOnWrite(i int) bool { - return ra.needCopyOnWrite[i] -} - -func (ra *roaringArray) setNeedsCopyOnWrite(i int) { - ra.needCopyOnWrite[i] = true -} diff --git a/vendor/github.com/RoaringBitmap/roaring/runcontainer.go b/vendor/github.com/RoaringBitmap/roaring/runcontainer.go deleted file mode 100644 index 7098ba28fc..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/runcontainer.go +++ /dev/null @@ -1,2624 +0,0 @@ -package roaring - -// -// Copyright (c) 2016 by the roaring authors. -// Licensed under the Apache License, Version 2.0. -// -// We derive a few lines of code from the sort.Search -// function in the golang standard library. That function -// is Copyright 2009 The Go Authors, and licensed -// under the following BSD-style license. -/* -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -import ( - "fmt" - "sort" - "unsafe" -) - -// runContainer16 does run-length encoding of sets of -// uint16 integers. -type runContainer16 struct { - // iv is a slice of sorted, non-overlapping, non-adjacent intervals. - iv []interval16 -} - -// interval16 is the internal to runContainer16 -// structure that maintains the individual [start, last] -// closed intervals. -type interval16 struct { - start uint16 - length uint16 // length minus 1 -} - -func newInterval16Range(start, last uint16) interval16 { - if last < start { - panic(fmt.Sprintf("last (%d) cannot be smaller than start (%d)", last, start)) - } - - return interval16{ - start, - last - start, - } -} - -// runlen returns the count of integers in the interval. -func (iv interval16) runlen() int { - return int(iv.length) + 1 -} - -func (iv interval16) last() uint16 { - return iv.start + iv.length -} - -// String produces a human viewable string of the contents. -func (iv interval16) String() string { - return fmt.Sprintf("[%d, %d]", iv.start, iv.length) -} - -func ivalString16(iv []interval16) string { - var s string - var j int - var p interval16 - for j, p = range iv { - s += fmt.Sprintf("%v:[%d, %d], ", j, p.start, p.last()) - } - return s -} - -// String produces a human viewable string of the contents. -func (rc *runContainer16) String() string { - if len(rc.iv) == 0 { - return "runContainer16{}" - } - is := ivalString16(rc.iv) - return `runContainer16{` + is + `}` -} - -// uint16Slice is a sort.Sort convenience method -type uint16Slice []uint16 - -// Len returns the length of p. -func (p uint16Slice) Len() int { return len(p) } - -// Less returns p[i] < p[j] -func (p uint16Slice) Less(i, j int) bool { return p[i] < p[j] } - -// Swap swaps elements i and j. -func (p uint16Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } - -// addHelper helps build a runContainer16. -type addHelper16 struct { - runstart uint16 - runlen uint16 - actuallyAdded uint16 - m []interval16 - rc *runContainer16 -} - -func (ah *addHelper16) storeIval(runstart, runlen uint16) { - mi := interval16{start: runstart, length: runlen} - ah.m = append(ah.m, mi) -} - -func (ah *addHelper16) add(cur, prev uint16, i int) { - if cur == prev+1 { - ah.runlen++ - ah.actuallyAdded++ - } else { - if cur < prev { - panic(fmt.Sprintf("newRunContainer16FromVals sees "+ - "unsorted vals; vals[%v]=cur=%v < prev=%v. Sort your vals"+ - " before calling us with alreadySorted == true.", i, cur, prev)) - } - if cur == prev { - // ignore duplicates - } else { - ah.actuallyAdded++ - ah.storeIval(ah.runstart, ah.runlen) - ah.runstart = cur - ah.runlen = 0 - } - } -} - -// newRunContainerRange makes a new container made of just the specified closed interval [rangestart,rangelast] -func newRunContainer16Range(rangestart uint16, rangelast uint16) *runContainer16 { - rc := &runContainer16{} - rc.iv = append(rc.iv, newInterval16Range(rangestart, rangelast)) - return rc -} - -// newRunContainer16FromVals makes a new container from vals. -// -// For efficiency, vals should be sorted in ascending order. -// Ideally vals should not contain duplicates, but we detect and -// ignore them. If vals is already sorted in ascending order, then -// pass alreadySorted = true. Otherwise, for !alreadySorted, -// we will sort vals before creating a runContainer16 of them. -// We sort the original vals, so this will change what the -// caller sees in vals as a side effect. -func newRunContainer16FromVals(alreadySorted bool, vals ...uint16) *runContainer16 { - // keep this in sync with newRunContainer16FromArray below - - rc := &runContainer16{} - ah := addHelper16{rc: rc} - - if !alreadySorted { - sort.Sort(uint16Slice(vals)) - } - n := len(vals) - var cur, prev uint16 - switch { - case n == 0: - // nothing more - case n == 1: - ah.m = append(ah.m, newInterval16Range(vals[0], vals[0])) - ah.actuallyAdded++ - default: - ah.runstart = vals[0] - ah.actuallyAdded++ - for i := 1; i < n; i++ { - prev = vals[i-1] - cur = vals[i] - ah.add(cur, prev, i) - } - ah.storeIval(ah.runstart, ah.runlen) - } - rc.iv = ah.m - return rc -} - -// newRunContainer16FromBitmapContainer makes a new run container from bc, -// somewhat efficiently. For reference, see the Java -// https://github.com/RoaringBitmap/RoaringBitmap/blob/master/src/main/java/org/roaringbitmap/RunContainer.java#L145-L192 -func newRunContainer16FromBitmapContainer(bc *bitmapContainer) *runContainer16 { - - rc := &runContainer16{} - nbrRuns := bc.numberOfRuns() - if nbrRuns == 0 { - return rc - } - rc.iv = make([]interval16, nbrRuns) - - longCtr := 0 // index of current long in bitmap - curWord := bc.bitmap[0] // its value - runCount := 0 - for { - // potentially multiword advance to first 1 bit - for curWord == 0 && longCtr < len(bc.bitmap)-1 { - longCtr++ - curWord = bc.bitmap[longCtr] - } - - if curWord == 0 { - // wrap up, no more runs - return rc - } - localRunStart := countTrailingZeros(curWord) - runStart := localRunStart + 64*longCtr - // stuff 1s into number's LSBs - curWordWith1s := curWord | (curWord - 1) - - // find the next 0, potentially in a later word - runEnd := 0 - for curWordWith1s == maxWord && longCtr < len(bc.bitmap)-1 { - longCtr++ - curWordWith1s = bc.bitmap[longCtr] - } - - if curWordWith1s == maxWord { - // a final unterminated run of 1s - runEnd = wordSizeInBits + longCtr*64 - rc.iv[runCount].start = uint16(runStart) - rc.iv[runCount].length = uint16(runEnd) - uint16(runStart) - 1 - return rc - } - localRunEnd := countTrailingZeros(^curWordWith1s) - runEnd = localRunEnd + longCtr*64 - rc.iv[runCount].start = uint16(runStart) - rc.iv[runCount].length = uint16(runEnd) - 1 - uint16(runStart) - runCount++ - // now, zero out everything right of runEnd. - curWord = curWordWith1s & (curWordWith1s + 1) - // We've lathered and rinsed, so repeat... - } - -} - -// newRunContainer16FromArray populates a new -// runContainer16 from the contents of arr. -func newRunContainer16FromArray(arr *arrayContainer) *runContainer16 { - // keep this in sync with newRunContainer16FromVals above - - rc := &runContainer16{} - ah := addHelper16{rc: rc} - - n := arr.getCardinality() - var cur, prev uint16 - switch { - case n == 0: - // nothing more - case n == 1: - ah.m = append(ah.m, newInterval16Range(arr.content[0], arr.content[0])) - ah.actuallyAdded++ - default: - ah.runstart = arr.content[0] - ah.actuallyAdded++ - for i := 1; i < n; i++ { - prev = arr.content[i-1] - cur = arr.content[i] - ah.add(cur, prev, i) - } - ah.storeIval(ah.runstart, ah.runlen) - } - rc.iv = ah.m - return rc -} - -// set adds the integers in vals to the set. Vals -// must be sorted in increasing order; if not, you should set -// alreadySorted to false, and we will sort them in place for you. -// (Be aware of this side effect -- it will affect the callers -// view of vals). -// -// If you have a small number of additions to an already -// big runContainer16, calling Add() may be faster. -func (rc *runContainer16) set(alreadySorted bool, vals ...uint16) { - - rc2 := newRunContainer16FromVals(alreadySorted, vals...) - un := rc.union(rc2) - rc.iv = un.iv -} - -// canMerge returns true iff the intervals -// a and b either overlap or they are -// contiguous and so can be merged into -// a single interval. -func canMerge16(a, b interval16) bool { - if int(a.last())+1 < int(b.start) { - return false - } - return int(b.last())+1 >= int(a.start) -} - -// haveOverlap differs from canMerge in that -// it tells you if the intersection of a -// and b would contain an element (otherwise -// it would be the empty set, and we return -// false). -func haveOverlap16(a, b interval16) bool { - if int(a.last())+1 <= int(b.start) { - return false - } - return int(b.last())+1 > int(a.start) -} - -// mergeInterval16s joins a and b into a -// new interval, and panics if it cannot. -func mergeInterval16s(a, b interval16) (res interval16) { - if !canMerge16(a, b) { - panic(fmt.Sprintf("cannot merge %#v and %#v", a, b)) - } - - if b.start < a.start { - res.start = b.start - } else { - res.start = a.start - } - - if b.last() > a.last() { - res.length = b.last() - res.start - } else { - res.length = a.last() - res.start - } - - return -} - -// intersectInterval16s returns the intersection -// of a and b. The isEmpty flag will be true if -// a and b were disjoint. -func intersectInterval16s(a, b interval16) (res interval16, isEmpty bool) { - if !haveOverlap16(a, b) { - isEmpty = true - return - } - if b.start > a.start { - res.start = b.start - } else { - res.start = a.start - } - - bEnd := b.last() - aEnd := a.last() - var resEnd uint16 - - if bEnd < aEnd { - resEnd = bEnd - } else { - resEnd = aEnd - } - res.length = resEnd - res.start - return -} - -// union merges two runContainer16s, producing -// a new runContainer16 with the union of rc and b. -func (rc *runContainer16) union(b *runContainer16) *runContainer16 { - - // rc is also known as 'a' here, but golint insisted we - // call it rc for consistency with the rest of the methods. - - var m []interval16 - - alim := int(len(rc.iv)) - blim := int(len(b.iv)) - - var na int // next from a - var nb int // next from b - - // merged holds the current merge output, which might - // get additional merges before being appended to m. - var merged interval16 - var mergedUsed bool // is merged being used at the moment? - - var cura interval16 // currently considering this interval16 from a - var curb interval16 // currently considering this interval16 from b - - pass := 0 - for na < alim && nb < blim { - pass++ - cura = rc.iv[na] - curb = b.iv[nb] - - if mergedUsed { - mergedUpdated := false - if canMerge16(cura, merged) { - merged = mergeInterval16s(cura, merged) - na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1) - mergedUpdated = true - } - if canMerge16(curb, merged) { - merged = mergeInterval16s(curb, merged) - nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1) - mergedUpdated = true - } - if !mergedUpdated { - // we know that merged is disjoint from cura and curb - m = append(m, merged) - mergedUsed = false - } - continue - - } else { - // !mergedUsed - if !canMerge16(cura, curb) { - if cura.start < curb.start { - m = append(m, cura) - na++ - } else { - m = append(m, curb) - nb++ - } - } else { - merged = mergeInterval16s(cura, curb) - mergedUsed = true - na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1) - nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1) - } - } - } - var aDone, bDone bool - if na >= alim { - aDone = true - } - if nb >= blim { - bDone = true - } - // finish by merging anything remaining into merged we can: - if mergedUsed { - if !aDone { - aAdds: - for na < alim { - cura = rc.iv[na] - if canMerge16(cura, merged) { - merged = mergeInterval16s(cura, merged) - na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1) - } else { - break aAdds - } - } - - } - - if !bDone { - bAdds: - for nb < blim { - curb = b.iv[nb] - if canMerge16(curb, merged) { - merged = mergeInterval16s(curb, merged) - nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1) - } else { - break bAdds - } - } - - } - - m = append(m, merged) - } - if na < alim { - m = append(m, rc.iv[na:]...) - } - if nb < blim { - m = append(m, b.iv[nb:]...) - } - - res := &runContainer16{iv: m} - return res -} - -// unionCardinality returns the cardinality of the merger of two runContainer16s, the union of rc and b. -func (rc *runContainer16) unionCardinality(b *runContainer16) uint { - - // rc is also known as 'a' here, but golint insisted we - // call it rc for consistency with the rest of the methods. - answer := uint(0) - - alim := int(len(rc.iv)) - blim := int(len(b.iv)) - - var na int // next from a - var nb int // next from b - - // merged holds the current merge output, which might - // get additional merges before being appended to m. - var merged interval16 - var mergedUsed bool // is merged being used at the moment? - - var cura interval16 // currently considering this interval16 from a - var curb interval16 // currently considering this interval16 from b - - pass := 0 - for na < alim && nb < blim { - pass++ - cura = rc.iv[na] - curb = b.iv[nb] - - if mergedUsed { - mergedUpdated := false - if canMerge16(cura, merged) { - merged = mergeInterval16s(cura, merged) - na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1) - mergedUpdated = true - } - if canMerge16(curb, merged) { - merged = mergeInterval16s(curb, merged) - nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1) - mergedUpdated = true - } - if !mergedUpdated { - // we know that merged is disjoint from cura and curb - //m = append(m, merged) - answer += uint(merged.last()) - uint(merged.start) + 1 - mergedUsed = false - } - continue - - } else { - // !mergedUsed - if !canMerge16(cura, curb) { - if cura.start < curb.start { - answer += uint(cura.last()) - uint(cura.start) + 1 - //m = append(m, cura) - na++ - } else { - answer += uint(curb.last()) - uint(curb.start) + 1 - //m = append(m, curb) - nb++ - } - } else { - merged = mergeInterval16s(cura, curb) - mergedUsed = true - na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1) - nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1) - } - } - } - var aDone, bDone bool - if na >= alim { - aDone = true - } - if nb >= blim { - bDone = true - } - // finish by merging anything remaining into merged we can: - if mergedUsed { - if !aDone { - aAdds: - for na < alim { - cura = rc.iv[na] - if canMerge16(cura, merged) { - merged = mergeInterval16s(cura, merged) - na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1) - } else { - break aAdds - } - } - - } - - if !bDone { - bAdds: - for nb < blim { - curb = b.iv[nb] - if canMerge16(curb, merged) { - merged = mergeInterval16s(curb, merged) - nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1) - } else { - break bAdds - } - } - - } - - //m = append(m, merged) - answer += uint(merged.last()) - uint(merged.start) + 1 - } - for _, r := range rc.iv[na:] { - answer += uint(r.last()) - uint(r.start) + 1 - } - for _, r := range b.iv[nb:] { - answer += uint(r.last()) - uint(r.start) + 1 - } - return answer -} - -// indexOfIntervalAtOrAfter is a helper for union. -func (rc *runContainer16) indexOfIntervalAtOrAfter(key int, startIndex int) int { - w, already, _ := rc.searchRange(key, startIndex, 0) - if already { - return w - } - return w + 1 -} - -// intersect returns a new runContainer16 holding the -// intersection of rc (also known as 'a') and b. -func (rc *runContainer16) intersect(b *runContainer16) *runContainer16 { - - a := rc - numa := int(len(a.iv)) - numb := int(len(b.iv)) - res := &runContainer16{} - if numa == 0 || numb == 0 { - return res - } - - if numa == 1 && numb == 1 { - if !haveOverlap16(a.iv[0], b.iv[0]) { - return res - } - } - - var output []interval16 - - var acuri int - var bcuri int - - astart := int(a.iv[acuri].start) - bstart := int(b.iv[bcuri].start) - - var intersection interval16 - var leftoverstart int - var isOverlap, isLeftoverA, isLeftoverB bool - var done bool -toploop: - for acuri < numa && bcuri < numb { - - isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection = - intersectWithLeftover16(astart, int(a.iv[acuri].last()), bstart, int(b.iv[bcuri].last())) - - if !isOverlap { - switch { - case astart < bstart: - acuri, done = a.findNextIntervalThatIntersectsStartingFrom(acuri+1, bstart) - if done { - break toploop - } - astart = int(a.iv[acuri].start) - - case astart > bstart: - bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart) - if done { - break toploop - } - bstart = int(b.iv[bcuri].start) - } - - } else { - // isOverlap - output = append(output, intersection) - switch { - case isLeftoverA: - // note that we change astart without advancing acuri, - // since we need to capture any 2ndary intersections with a.iv[acuri] - astart = leftoverstart - bcuri++ - if bcuri >= numb { - break toploop - } - bstart = int(b.iv[bcuri].start) - case isLeftoverB: - // note that we change bstart without advancing bcuri, - // since we need to capture any 2ndary intersections with b.iv[bcuri] - bstart = leftoverstart - acuri++ - if acuri >= numa { - break toploop - } - astart = int(a.iv[acuri].start) - default: - // neither had leftover, both completely consumed - - // advance to next a interval - acuri++ - if acuri >= numa { - break toploop - } - astart = int(a.iv[acuri].start) - - // advance to next b interval - bcuri++ - if bcuri >= numb { - break toploop - } - bstart = int(b.iv[bcuri].start) - } - } - } // end for toploop - - if len(output) == 0 { - return res - } - - res.iv = output - return res -} - -// intersectCardinality returns the cardinality of the -// intersection of rc (also known as 'a') and b. -func (rc *runContainer16) intersectCardinality(b *runContainer16) int { - answer := int(0) - - a := rc - numa := int(len(a.iv)) - numb := int(len(b.iv)) - if numa == 0 || numb == 0 { - return 0 - } - - if numa == 1 && numb == 1 { - if !haveOverlap16(a.iv[0], b.iv[0]) { - return 0 - } - } - - var acuri int - var bcuri int - - astart := int(a.iv[acuri].start) - bstart := int(b.iv[bcuri].start) - - var intersection interval16 - var leftoverstart int - var isOverlap, isLeftoverA, isLeftoverB bool - var done bool - pass := 0 -toploop: - for acuri < numa && bcuri < numb { - pass++ - - isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection = - intersectWithLeftover16(astart, int(a.iv[acuri].last()), bstart, int(b.iv[bcuri].last())) - - if !isOverlap { - switch { - case astart < bstart: - acuri, done = a.findNextIntervalThatIntersectsStartingFrom(acuri+1, bstart) - if done { - break toploop - } - astart = int(a.iv[acuri].start) - - case astart > bstart: - bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart) - if done { - break toploop - } - bstart = int(b.iv[bcuri].start) - } - - } else { - // isOverlap - answer += int(intersection.last()) - int(intersection.start) + 1 - switch { - case isLeftoverA: - // note that we change astart without advancing acuri, - // since we need to capture any 2ndary intersections with a.iv[acuri] - astart = leftoverstart - bcuri++ - if bcuri >= numb { - break toploop - } - bstart = int(b.iv[bcuri].start) - case isLeftoverB: - // note that we change bstart without advancing bcuri, - // since we need to capture any 2ndary intersections with b.iv[bcuri] - bstart = leftoverstart - acuri++ - if acuri >= numa { - break toploop - } - astart = int(a.iv[acuri].start) - default: - // neither had leftover, both completely consumed - - // advance to next a interval - acuri++ - if acuri >= numa { - break toploop - } - astart = int(a.iv[acuri].start) - - // advance to next b interval - bcuri++ - if bcuri >= numb { - break toploop - } - bstart = int(b.iv[bcuri].start) - } - } - } // end for toploop - - return answer -} - -// get returns true iff key is in the container. -func (rc *runContainer16) contains(key uint16) bool { - _, in, _ := rc.search(int(key)) - return in -} - -// numIntervals returns the count of intervals in the container. -func (rc *runContainer16) numIntervals() int { - return len(rc.iv) -} - -// searchRange returns alreadyPresent to indicate if the -// key is already in one of our interval16s. -// -// If key is alreadyPresent, then whichInterval16 tells -// you where. -// -// If key is not already present, then whichInterval16 is -// set as follows: -// -// a) whichInterval16 == len(rc.iv)-1 if key is beyond our -// last interval16 in rc.iv; -// -// b) whichInterval16 == -1 if key is before our first -// interval16 in rc.iv; -// -// c) whichInterval16 is set to the minimum index of rc.iv -// which comes strictly before the key; -// so rc.iv[whichInterval16].last < key, -// and if whichInterval16+1 exists, then key < rc.iv[whichInterval16+1].start -// (Note that whichInterval16+1 won't exist when -// whichInterval16 is the last interval.) -// -// runContainer16.search always returns whichInterval16 < len(rc.iv). -// -// The search space is from startIndex to endxIndex. If endxIndex is set to zero, then there -// no upper bound. -func (rc *runContainer16) searchRange(key int, startIndex int, endxIndex int) (whichInterval16 int, alreadyPresent bool, numCompares int) { - n := int(len(rc.iv)) - if n == 0 { - return -1, false, 0 - } - if endxIndex == 0 { - endxIndex = n - } - - // sort.Search returns the smallest index i - // in [0, n) at which f(i) is true, assuming that on the range [0, n), - // f(i) == true implies f(i+1) == true. - // If there is no such index, Search returns n. - - // For correctness, this began as verbatim snippet from - // sort.Search in the Go standard lib. - // We inline our comparison function for speed, and - // annotate with numCompares - // to observe and test that extra bounds are utilized. - i, j := startIndex, endxIndex - for i < j { - h := i + (j-i)/2 // avoid overflow when computing h as the bisector - // i <= h < j - numCompares++ - if !(key < int(rc.iv[h].start)) { - i = h + 1 - } else { - j = h - } - } - below := i - // end std lib snippet. - - // The above is a simple in-lining and annotation of: - /* below := sort.Search(n, - func(i int) bool { - return key < rc.iv[i].start - }) - */ - whichInterval16 = below - 1 - - if below == n { - // all falses => key is >= start of all interval16s - // ... so does it belong to the last interval16? - if key < int(rc.iv[n-1].last())+1 { - // yes, it belongs to the last interval16 - alreadyPresent = true - return - } - // no, it is beyond the last interval16. - // leave alreadyPreset = false - return - } - - // INVAR: key is below rc.iv[below] - if below == 0 { - // key is before the first first interval16. - // leave alreadyPresent = false - return - } - - // INVAR: key is >= rc.iv[below-1].start and - // key is < rc.iv[below].start - - // is key in below-1 interval16? - if key >= int(rc.iv[below-1].start) && key < int(rc.iv[below-1].last())+1 { - // yes, it is. key is in below-1 interval16. - alreadyPresent = true - return - } - - // INVAR: key >= rc.iv[below-1].endx && key < rc.iv[below].start - // leave alreadyPresent = false - return -} - -// search returns alreadyPresent to indicate if the -// key is already in one of our interval16s. -// -// If key is alreadyPresent, then whichInterval16 tells -// you where. -// -// If key is not already present, then whichInterval16 is -// set as follows: -// -// a) whichInterval16 == len(rc.iv)-1 if key is beyond our -// last interval16 in rc.iv; -// -// b) whichInterval16 == -1 if key is before our first -// interval16 in rc.iv; -// -// c) whichInterval16 is set to the minimum index of rc.iv -// which comes strictly before the key; -// so rc.iv[whichInterval16].last < key, -// and if whichInterval16+1 exists, then key < rc.iv[whichInterval16+1].start -// (Note that whichInterval16+1 won't exist when -// whichInterval16 is the last interval.) -// -// runContainer16.search always returns whichInterval16 < len(rc.iv). -func (rc *runContainer16) search(key int) (whichInterval16 int, alreadyPresent bool, numCompares int) { - return rc.searchRange(key, 0, 0) -} - -// getCardinality returns the count of the integers stored in the -// runContainer16. The running complexity depends on the size -// of the container. -func (rc *runContainer16) getCardinality() int { - // have to compute it - n := 0 - for _, p := range rc.iv { - n += p.runlen() - } - return n -} - -// isEmpty returns true if the container is empty. -// It runs in constant time. -func (rc *runContainer16) isEmpty() bool { - return len(rc.iv) == 0 -} - -// AsSlice decompresses the contents into a []uint16 slice. -func (rc *runContainer16) AsSlice() []uint16 { - s := make([]uint16, rc.getCardinality()) - j := 0 - for _, p := range rc.iv { - for i := p.start; i <= p.last(); i++ { - s[j] = i - j++ - } - } - return s -} - -// newRunContainer16 creates an empty run container. -func newRunContainer16() *runContainer16 { - return &runContainer16{} -} - -// newRunContainer16CopyIv creates a run container, initializing -// with a copy of the supplied iv slice. -func newRunContainer16CopyIv(iv []interval16) *runContainer16 { - rc := &runContainer16{ - iv: make([]interval16, len(iv)), - } - copy(rc.iv, iv) - return rc -} - -func (rc *runContainer16) Clone() *runContainer16 { - rc2 := newRunContainer16CopyIv(rc.iv) - return rc2 -} - -// newRunContainer16TakeOwnership returns a new runContainer16 -// backed by the provided iv slice, which we will -// assume exclusive control over from now on. -func newRunContainer16TakeOwnership(iv []interval16) *runContainer16 { - rc := &runContainer16{ - iv: iv, - } - return rc -} - -const baseRc16Size = int(unsafe.Sizeof(runContainer16{})) -const perIntervalRc16Size = int(unsafe.Sizeof(interval16{})) - -const baseDiskRc16Size = int(unsafe.Sizeof(uint16(0))) - -// see also runContainer16SerializedSizeInBytes(numRuns int) int - -// getSizeInBytes returns the number of bytes of memory -// required by this runContainer16. -func (rc *runContainer16) getSizeInBytes() int { - return perIntervalRc16Size*len(rc.iv) + baseRc16Size -} - -// runContainer16SerializedSizeInBytes returns the number of bytes of disk -// required to hold numRuns in a runContainer16. -func runContainer16SerializedSizeInBytes(numRuns int) int { - return perIntervalRc16Size*numRuns + baseDiskRc16Size -} - -// Add adds a single value k to the set. -func (rc *runContainer16) Add(k uint16) (wasNew bool) { - // TODO comment from runContainer16.java: - // it might be better and simpler to do return - // toBitmapOrArrayContainer(getCardinality()).add(k) - // but note that some unit tests use this method to build up test - // runcontainers without calling runOptimize - - k64 := int(k) - - index, present, _ := rc.search(k64) - if present { - return // already there - } - wasNew = true - - n := int(len(rc.iv)) - if index == -1 { - // we may need to extend the first run - if n > 0 { - if rc.iv[0].start == k+1 { - rc.iv[0].start = k - rc.iv[0].length++ - return - } - } - // nope, k stands alone, starting the new first interval16. - rc.iv = append([]interval16{newInterval16Range(k, k)}, rc.iv...) - return - } - - // are we off the end? handle both index == n and index == n-1: - if index >= n-1 { - if int(rc.iv[n-1].last())+1 == k64 { - rc.iv[n-1].length++ - return - } - rc.iv = append(rc.iv, newInterval16Range(k, k)) - return - } - - // INVAR: index and index+1 both exist, and k goes between them. - // - // Now: add k into the middle, - // possibly fusing with index or index+1 interval16 - // and possibly resulting in fusing of two interval16s - // that had a one integer gap. - - left := index - right := index + 1 - - // are we fusing left and right by adding k? - if int(rc.iv[left].last())+1 == k64 && int(rc.iv[right].start) == k64+1 { - // fuse into left - rc.iv[left].length = rc.iv[right].last() - rc.iv[left].start - // remove redundant right - rc.iv = append(rc.iv[:left+1], rc.iv[right+1:]...) - return - } - - // are we an addition to left? - if int(rc.iv[left].last())+1 == k64 { - // yes - rc.iv[left].length++ - return - } - - // are we an addition to right? - if int(rc.iv[right].start) == k64+1 { - // yes - rc.iv[right].start = k - rc.iv[right].length++ - return - } - - // k makes a standalone new interval16, inserted in the middle - tail := append([]interval16{newInterval16Range(k, k)}, rc.iv[right:]...) - rc.iv = append(rc.iv[:left+1], tail...) - return -} - -// runIterator16 advice: you must call hasNext() -// before calling next()/peekNext() to insure there are contents. -type runIterator16 struct { - rc *runContainer16 - curIndex int - curPosInIndex uint16 -} - -// newRunIterator16 returns a new empty run container. -func (rc *runContainer16) newRunIterator16() *runIterator16 { - return &runIterator16{rc: rc, curIndex: 0, curPosInIndex: 0} -} - -func (rc *runContainer16) iterate(cb func(x uint16) bool) bool { - iterator := runIterator16{rc, 0, 0} - - for iterator.hasNext() { - if !cb(iterator.next()) { - return false - } - } - - return true -} - -// hasNext returns false if calling next will panic. It -// returns true when there is at least one more value -// available in the iteration sequence. -func (ri *runIterator16) hasNext() bool { - return int(len(ri.rc.iv)) > ri.curIndex+1 || - (int(len(ri.rc.iv)) == ri.curIndex+1 && ri.rc.iv[ri.curIndex].length >= ri.curPosInIndex) -} - -// next returns the next value in the iteration sequence. -func (ri *runIterator16) next() uint16 { - next := ri.rc.iv[ri.curIndex].start + ri.curPosInIndex - - if ri.curPosInIndex == ri.rc.iv[ri.curIndex].length { - ri.curPosInIndex = 0 - ri.curIndex++ - } else { - ri.curPosInIndex++ - } - - return next -} - -// peekNext returns the next value in the iteration sequence without advancing the iterator -func (ri *runIterator16) peekNext() uint16 { - return ri.rc.iv[ri.curIndex].start + ri.curPosInIndex -} - -// advanceIfNeeded advances as long as the next value is smaller than minval -func (ri *runIterator16) advanceIfNeeded(minval uint16) { - if !ri.hasNext() || ri.peekNext() >= minval { - return - } - - // interval cannot be -1 because of minval > peekNext - interval, isPresent, _ := ri.rc.searchRange(int(minval), ri.curIndex, int(len(ri.rc.iv))) - - // if the minval is present, set the curPosIndex at the right position - if isPresent { - ri.curIndex = interval - ri.curPosInIndex = minval - ri.rc.iv[ri.curIndex].start - } else { - // otherwise interval is set to to the minimum index of rc.iv - // which comes strictly before the key, that's why we set the next interval - ri.curIndex = interval + 1 - ri.curPosInIndex = 0 - } -} - -// runReverseIterator16 advice: you must call hasNext() -// before calling next() to insure there are contents. -type runReverseIterator16 struct { - rc *runContainer16 - curIndex int // index into rc.iv - curPosInIndex uint16 // offset in rc.iv[curIndex] -} - -// newRunReverseIterator16 returns a new empty run iterator. -func (rc *runContainer16) newRunReverseIterator16() *runReverseIterator16 { - index := int(len(rc.iv)) - 1 - pos := uint16(0) - - if index >= 0 { - pos = rc.iv[index].length - } - - return &runReverseIterator16{ - rc: rc, - curIndex: index, - curPosInIndex: pos, - } -} - -// hasNext returns false if calling next will panic. It -// returns true when there is at least one more value -// available in the iteration sequence. -func (ri *runReverseIterator16) hasNext() bool { - return ri.curIndex > 0 || ri.curIndex == 0 && ri.curPosInIndex >= 0 -} - -// next returns the next value in the iteration sequence. -func (ri *runReverseIterator16) next() uint16 { - next := ri.rc.iv[ri.curIndex].start + ri.curPosInIndex - - if ri.curPosInIndex > 0 { - ri.curPosInIndex-- - } else { - ri.curIndex-- - - if ri.curIndex >= 0 { - ri.curPosInIndex = ri.rc.iv[ri.curIndex].length - } - } - - return next -} - -func (rc *runContainer16) newManyRunIterator16() *runIterator16 { - return rc.newRunIterator16() -} - -// hs are the high bits to include to avoid needing to reiterate over the buffer in NextMany -func (ri *runIterator16) nextMany(hs uint32, buf []uint32) int { - n := 0 - - if !ri.hasNext() { - return n - } - - // start and end are inclusive - for n < len(buf) { - moreVals := 0 - - if ri.rc.iv[ri.curIndex].length >= ri.curPosInIndex { - // add as many as you can from this seq - moreVals = minOfInt(int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex)+1, len(buf)-n) - base := uint32(ri.rc.iv[ri.curIndex].start+ri.curPosInIndex) | hs - - // allows BCE - buf2 := buf[n : n+moreVals] - for i := range buf2 { - buf2[i] = base + uint32(i) - } - - // update values - n += moreVals - } - - if moreVals+int(ri.curPosInIndex) > int(ri.rc.iv[ri.curIndex].length) { - ri.curPosInIndex = 0 - ri.curIndex++ - - if ri.curIndex == int(len(ri.rc.iv)) { - break - } - } else { - ri.curPosInIndex += uint16(moreVals) //moreVals always fits in uint16 - } - } - - return n -} - -func (ri *runIterator16) nextMany64(hs uint64, buf []uint64) int { - n := 0 - - if !ri.hasNext() { - return n - } - - // start and end are inclusive - for n < len(buf) { - moreVals := 0 - - if ri.rc.iv[ri.curIndex].length >= ri.curPosInIndex { - // add as many as you can from this seq - moreVals = minOfInt(int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex)+1, len(buf)-n) - base := uint64(ri.rc.iv[ri.curIndex].start+ri.curPosInIndex) | hs - - // allows BCE - buf2 := buf[n : n+moreVals] - for i := range buf2 { - buf2[i] = base + uint64(i) - } - - // update values - n += moreVals - } - - if moreVals+int(ri.curPosInIndex) > int(ri.rc.iv[ri.curIndex].length) { - ri.curPosInIndex = 0 - ri.curIndex++ - - if ri.curIndex == int(len(ri.rc.iv)) { - break - } - } else { - ri.curPosInIndex += uint16(moreVals) //moreVals always fits in uint16 - } - } - - return n -} - -// remove removes key from the container. -func (rc *runContainer16) removeKey(key uint16) (wasPresent bool) { - - var index int - index, wasPresent, _ = rc.search(int(key)) - if !wasPresent { - return // already removed, nothing to do. - } - pos := key - rc.iv[index].start - rc.deleteAt(&index, &pos) - return -} - -// internal helper functions - -func (rc *runContainer16) deleteAt(curIndex *int, curPosInIndex *uint16) { - ci := *curIndex - pos := *curPosInIndex - - // are we first, last, or in the middle of our interval16? - switch { - case pos == 0: - if int(rc.iv[ci].length) == 0 { - // our interval disappears - rc.iv = append(rc.iv[:ci], rc.iv[ci+1:]...) - // curIndex stays the same, since the delete did - // the advance for us. - *curPosInIndex = 0 - } else { - rc.iv[ci].start++ // no longer overflowable - rc.iv[ci].length-- - } - case pos == rc.iv[ci].length: - // length - rc.iv[ci].length-- - // our interval16 cannot disappear, else we would have been pos == 0, case first above. - *curPosInIndex-- - // if we leave *curIndex alone, then Next() will work properly even after the delete. - default: - //middle - // split into two, adding an interval16 - new0 := newInterval16Range(rc.iv[ci].start, rc.iv[ci].start+*curPosInIndex-1) - - new1start := int(rc.iv[ci].start+*curPosInIndex) + 1 - if new1start > int(MaxUint16) { - panic("overflow?!?!") - } - new1 := newInterval16Range(uint16(new1start), rc.iv[ci].last()) - tail := append([]interval16{new0, new1}, rc.iv[ci+1:]...) - rc.iv = append(rc.iv[:ci], tail...) - // update curIndex and curPosInIndex - *curIndex++ - *curPosInIndex = 0 - } - -} - -func have4Overlap16(astart, alast, bstart, blast int) bool { - if alast+1 <= bstart { - return false - } - return blast+1 > astart -} - -func intersectWithLeftover16(astart, alast, bstart, blast int) (isOverlap, isLeftoverA, isLeftoverB bool, leftoverstart int, intersection interval16) { - if !have4Overlap16(astart, alast, bstart, blast) { - return - } - isOverlap = true - - // do the intersection: - if bstart > astart { - intersection.start = uint16(bstart) - } else { - intersection.start = uint16(astart) - } - - switch { - case blast < alast: - isLeftoverA = true - leftoverstart = blast + 1 - intersection.length = uint16(blast) - intersection.start - case alast < blast: - isLeftoverB = true - leftoverstart = alast + 1 - intersection.length = uint16(alast) - intersection.start - default: - // alast == blast - intersection.length = uint16(alast) - intersection.start - } - - return -} - -func (rc *runContainer16) findNextIntervalThatIntersectsStartingFrom(startIndex int, key int) (index int, done bool) { - w, _, _ := rc.searchRange(key, startIndex, 0) - // rc.search always returns w < len(rc.iv) - if w < startIndex { - // not found and comes before lower bound startIndex, - // so just use the lower bound. - if startIndex == int(len(rc.iv)) { - // also this bump up means that we are done - return startIndex, true - } - return startIndex, false - } - - return w, false -} - -func sliceToString16(m []interval16) string { - s := "" - for i := range m { - s += fmt.Sprintf("%v: %s, ", i, m[i]) - } - return s -} - -// helper for invert -func (rc *runContainer16) invertlastInterval(origin uint16, lastIdx int) []interval16 { - cur := rc.iv[lastIdx] - if cur.last() == MaxUint16 { - if cur.start == origin { - return nil // empty container - } - return []interval16{newInterval16Range(origin, cur.start-1)} - } - if cur.start == origin { - return []interval16{newInterval16Range(cur.last()+1, MaxUint16)} - } - // invert splits - return []interval16{ - newInterval16Range(origin, cur.start-1), - newInterval16Range(cur.last()+1, MaxUint16), - } -} - -// invert returns a new container (not inplace), that is -// the inversion of rc. For each bit b in rc, the -// returned value has !b -func (rc *runContainer16) invert() *runContainer16 { - ni := len(rc.iv) - var m []interval16 - switch ni { - case 0: - return &runContainer16{iv: []interval16{newInterval16Range(0, MaxUint16)}} - case 1: - return &runContainer16{iv: rc.invertlastInterval(0, 0)} - } - var invstart int - ult := ni - 1 - for i, cur := range rc.iv { - if i == ult { - // invertlastInteval will add both intervals (b) and (c) in - // diagram below. - m = append(m, rc.invertlastInterval(uint16(invstart), i)...) - break - } - // INVAR: i and cur are not the last interval, there is a next at i+1 - // - // ........[cur.start, cur.last] ...... [next.start, next.last].... - // ^ ^ ^ - // (a) (b) (c) - // - // Now: we add interval (a); but if (a) is empty, for cur.start==0, we skip it. - if cur.start > 0 { - m = append(m, newInterval16Range(uint16(invstart), cur.start-1)) - } - invstart = int(cur.last() + 1) - } - return &runContainer16{iv: m} -} - -func (iv interval16) equal(b interval16) bool { - return iv.start == b.start && iv.length == b.length -} - -func (iv interval16) isSuperSetOf(b interval16) bool { - return iv.start <= b.start && b.last() <= iv.last() -} - -func (iv interval16) subtractInterval(del interval16) (left []interval16, delcount int) { - isect, isEmpty := intersectInterval16s(iv, del) - - if isEmpty { - return nil, 0 - } - if del.isSuperSetOf(iv) { - return nil, iv.runlen() - } - - switch { - case isect.start > iv.start && isect.last() < iv.last(): - new0 := newInterval16Range(iv.start, isect.start-1) - new1 := newInterval16Range(isect.last()+1, iv.last()) - return []interval16{new0, new1}, isect.runlen() - case isect.start == iv.start: - return []interval16{newInterval16Range(isect.last()+1, iv.last())}, isect.runlen() - default: - return []interval16{newInterval16Range(iv.start, isect.start-1)}, isect.runlen() - } -} - -func (rc *runContainer16) isubtract(del interval16) { - origiv := make([]interval16, len(rc.iv)) - copy(origiv, rc.iv) - n := int(len(rc.iv)) - if n == 0 { - return // already done. - } - - _, isEmpty := intersectInterval16s(newInterval16Range(rc.iv[0].start, rc.iv[n-1].last()), del) - if isEmpty { - return // done - } - - // INVAR there is some intersection between rc and del - istart, startAlready, _ := rc.search(int(del.start)) - ilast, lastAlready, _ := rc.search(int(del.last())) - if istart == -1 { - if ilast == n-1 && !lastAlready { - rc.iv = nil - return - } - } - // some intervals will remain - switch { - case startAlready && lastAlready: - res0, _ := rc.iv[istart].subtractInterval(del) - - // would overwrite values in iv b/c res0 can have len 2. so - // write to origiv instead. - lost := 1 + ilast - istart - changeSize := int(len(res0)) - lost - newSize := int(len(rc.iv)) + changeSize - - // rc.iv = append(pre, caboose...) - // return - - if ilast != istart { - res1, _ := rc.iv[ilast].subtractInterval(del) - res0 = append(res0, res1...) - changeSize = int(len(res0)) - lost - newSize = int(len(rc.iv)) + changeSize - } - switch { - case changeSize < 0: - // shrink - copy(rc.iv[istart+int(len(res0)):], rc.iv[ilast+1:]) - copy(rc.iv[istart:istart+int(len(res0))], res0) - rc.iv = rc.iv[:newSize] - return - case changeSize == 0: - // stay the same - copy(rc.iv[istart:istart+int(len(res0))], res0) - return - default: - // changeSize > 0 is only possible when ilast == istart. - // Hence we now know: changeSize == 1 and len(res0) == 2 - rc.iv = append(rc.iv, interval16{}) - // len(rc.iv) is correct now, no need to rc.iv = rc.iv[:newSize] - - // copy the tail into place - copy(rc.iv[ilast+2:], rc.iv[ilast+1:]) - // copy the new item(s) into place - copy(rc.iv[istart:istart+2], res0) - return - } - - case !startAlready && !lastAlready: - // we get to discard whole intervals - - // from the search() definition: - - // if del.start is not present, then istart is - // set as follows: - // - // a) istart == n-1 if del.start is beyond our - // last interval16 in rc.iv; - // - // b) istart == -1 if del.start is before our first - // interval16 in rc.iv; - // - // c) istart is set to the minimum index of rc.iv - // which comes strictly before the del.start; - // so del.start > rc.iv[istart].last, - // and if istart+1 exists, then del.start < rc.iv[istart+1].startx - - // if del.last is not present, then ilast is - // set as follows: - // - // a) ilast == n-1 if del.last is beyond our - // last interval16 in rc.iv; - // - // b) ilast == -1 if del.last is before our first - // interval16 in rc.iv; - // - // c) ilast is set to the minimum index of rc.iv - // which comes strictly before the del.last; - // so del.last > rc.iv[ilast].last, - // and if ilast+1 exists, then del.last < rc.iv[ilast+1].start - - // INVAR: istart >= 0 - pre := rc.iv[:istart+1] - if ilast == n-1 { - rc.iv = pre - return - } - // INVAR: ilast < n-1 - lost := ilast - istart - changeSize := -lost - newSize := int(len(rc.iv)) + changeSize - if changeSize != 0 { - copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:]) - } - rc.iv = rc.iv[:newSize] - return - - case startAlready && !lastAlready: - // we can only shrink or stay the same size - // i.e. we either eliminate the whole interval, - // or just cut off the right side. - res0, _ := rc.iv[istart].subtractInterval(del) - if len(res0) > 0 { - // len(res) must be 1 - rc.iv[istart] = res0[0] - } - lost := 1 + (ilast - istart) - changeSize := int(len(res0)) - lost - newSize := int(len(rc.iv)) + changeSize - if changeSize != 0 { - copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:]) - } - rc.iv = rc.iv[:newSize] - return - - case !startAlready && lastAlready: - // we can only shrink or stay the same size - res1, _ := rc.iv[ilast].subtractInterval(del) - lost := ilast - istart - changeSize := int(len(res1)) - lost - newSize := int(len(rc.iv)) + changeSize - if changeSize != 0 { - // move the tail first to make room for res1 - copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:]) - } - copy(rc.iv[istart+1:], res1) - rc.iv = rc.iv[:newSize] - return - } -} - -// compute rc minus b, and return the result as a new value (not inplace). -// port of run_container_andnot from CRoaring... -// https://github.com/RoaringBitmap/CRoaring/blob/master/src/containers/run.c#L435-L496 -func (rc *runContainer16) AndNotRunContainer16(b *runContainer16) *runContainer16 { - - if len(b.iv) == 0 || len(rc.iv) == 0 { - return rc - } - - dst := newRunContainer16() - apos := 0 - bpos := 0 - - a := rc - - astart := a.iv[apos].start - alast := a.iv[apos].last() - bstart := b.iv[bpos].start - blast := b.iv[bpos].last() - - alen := len(a.iv) - blen := len(b.iv) - - for apos < alen && bpos < blen { - switch { - case alast < bstart: - // output the first run - dst.iv = append(dst.iv, newInterval16Range(astart, alast)) - apos++ - if apos < alen { - astart = a.iv[apos].start - alast = a.iv[apos].last() - } - case blast < astart: - // exit the second run - bpos++ - if bpos < blen { - bstart = b.iv[bpos].start - blast = b.iv[bpos].last() - } - default: - // a: [ ] - // b: [ ] - // alast >= bstart - // blast >= astart - if astart < bstart { - dst.iv = append(dst.iv, newInterval16Range(astart, bstart-1)) - } - if alast > blast { - astart = blast + 1 - } else { - apos++ - if apos < alen { - astart = a.iv[apos].start - alast = a.iv[apos].last() - } - } - } - } - if apos < alen { - dst.iv = append(dst.iv, newInterval16Range(astart, alast)) - apos++ - if apos < alen { - dst.iv = append(dst.iv, a.iv[apos:]...) - } - } - - return dst -} - -func (rc *runContainer16) numberOfRuns() (nr int) { - return len(rc.iv) -} - -func (rc *runContainer16) containerType() contype { - return run16Contype -} - -func (rc *runContainer16) equals16(srb *runContainer16) bool { - // Check if the containers are the same object. - if rc == srb { - return true - } - - if len(srb.iv) != len(rc.iv) { - return false - } - - for i, v := range rc.iv { - if v != srb.iv[i] { - return false - } - } - return true -} - -// compile time verify we meet interface requirements -var _ container = &runContainer16{} - -func (rc *runContainer16) clone() container { - return newRunContainer16CopyIv(rc.iv) -} - -func (rc *runContainer16) minimum() uint16 { - return rc.iv[0].start // assume not empty -} - -func (rc *runContainer16) maximum() uint16 { - return rc.iv[len(rc.iv)-1].last() // assume not empty -} - -func (rc *runContainer16) isFull() bool { - return (len(rc.iv) == 1) && ((rc.iv[0].start == 0) && (rc.iv[0].last() == MaxUint16)) -} - -func (rc *runContainer16) and(a container) container { - if rc.isFull() { - return a.clone() - } - switch c := a.(type) { - case *runContainer16: - return rc.intersect(c) - case *arrayContainer: - return rc.andArray(c) - case *bitmapContainer: - return rc.andBitmapContainer(c) - } - panic("unsupported container type") -} - -func (rc *runContainer16) andCardinality(a container) int { - switch c := a.(type) { - case *runContainer16: - return int(rc.intersectCardinality(c)) - case *arrayContainer: - return rc.andArrayCardinality(c) - case *bitmapContainer: - return rc.andBitmapContainerCardinality(c) - } - panic("unsupported container type") -} - -// andBitmapContainer finds the intersection of rc and b. -func (rc *runContainer16) andBitmapContainer(bc *bitmapContainer) container { - bc2 := newBitmapContainerFromRun(rc) - return bc2.andBitmap(bc) -} - -func (rc *runContainer16) andArrayCardinality(ac *arrayContainer) int { - pos := 0 - answer := 0 - maxpos := ac.getCardinality() - if maxpos == 0 { - return 0 // won't happen in actual code - } - v := ac.content[pos] -mainloop: - for _, p := range rc.iv { - for v < p.start { - pos++ - if pos == maxpos { - break mainloop - } - v = ac.content[pos] - } - for v <= p.last() { - answer++ - pos++ - if pos == maxpos { - break mainloop - } - v = ac.content[pos] - } - } - return answer -} - -func (rc *runContainer16) iand(a container) container { - if rc.isFull() { - return a.clone() - } - switch c := a.(type) { - case *runContainer16: - return rc.inplaceIntersect(c) - case *arrayContainer: - return rc.andArray(c) - case *bitmapContainer: - return rc.iandBitmapContainer(c) - } - panic("unsupported container type") -} - -func (rc *runContainer16) inplaceIntersect(rc2 *runContainer16) container { - sect := rc.intersect(rc2) - *rc = *sect - return rc -} - -func (rc *runContainer16) iandBitmapContainer(bc *bitmapContainer) container { - isect := rc.andBitmapContainer(bc) - *rc = *newRunContainer16FromContainer(isect) - return rc -} - -func (rc *runContainer16) andArray(ac *arrayContainer) container { - if len(rc.iv) == 0 { - return newArrayContainer() - } - - acCardinality := ac.getCardinality() - c := newArrayContainerCapacity(acCardinality) - - for rlePos, arrayPos := 0, 0; arrayPos < acCardinality; { - iv := rc.iv[rlePos] - arrayVal := ac.content[arrayPos] - - for iv.last() < arrayVal { - rlePos++ - if rlePos == len(rc.iv) { - return c - } - iv = rc.iv[rlePos] - } - - if iv.start > arrayVal { - arrayPos = advanceUntil(ac.content, arrayPos, len(ac.content), iv.start) - } else { - c.content = append(c.content, arrayVal) - arrayPos++ - } - } - return c -} - -func (rc *runContainer16) andNot(a container) container { - switch c := a.(type) { - case *arrayContainer: - return rc.andNotArray(c) - case *bitmapContainer: - return rc.andNotBitmap(c) - case *runContainer16: - return rc.andNotRunContainer16(c) - } - panic("unsupported container type") -} - -func (rc *runContainer16) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int { - k := i - var val int - for _, p := range rc.iv { - n := p.runlen() - for j := int(0); j < n; j++ { - val = int(p.start) + j - x[k] = uint32(val) | mask - k++ - } - } - return k -} - -func (rc *runContainer16) getShortIterator() shortPeekable { - return rc.newRunIterator16() -} - -func (rc *runContainer16) getReverseIterator() shortIterable { - return rc.newRunReverseIterator16() -} - -func (rc *runContainer16) getManyIterator() manyIterable { - return rc.newManyRunIterator16() -} - -// add the values in the range [firstOfRange, endx). endx -// is still abe to express 2^16 because it is an int not an uint16. -func (rc *runContainer16) iaddRange(firstOfRange, endx int) container { - - if firstOfRange > endx { - panic(fmt.Sprintf("invalid %v = endx > firstOfRange", endx)) - } - if firstOfRange == endx { - return rc - } - addme := newRunContainer16TakeOwnership([]interval16{ - { - start: uint16(firstOfRange), - length: uint16(endx - 1 - firstOfRange), - }, - }) - *rc = *rc.union(addme) - return rc -} - -// remove the values in the range [firstOfRange,endx) -func (rc *runContainer16) iremoveRange(firstOfRange, endx int) container { - if firstOfRange > endx { - panic(fmt.Sprintf("request to iremove empty set [%v, %v),"+ - " nothing to do.", firstOfRange, endx)) - } - // empty removal - if firstOfRange == endx { - return rc - } - x := newInterval16Range(uint16(firstOfRange), uint16(endx-1)) - rc.isubtract(x) - return rc -} - -// not flip the values in the range [firstOfRange,endx) -func (rc *runContainer16) not(firstOfRange, endx int) container { - if firstOfRange > endx { - panic(fmt.Sprintf("invalid %v = endx > firstOfRange = %v", endx, firstOfRange)) - } - - return rc.Not(firstOfRange, endx) -} - -// Not flips the values in the range [firstOfRange,endx). -// This is not inplace. Only the returned value has the flipped bits. -// -// Currently implemented as (!A intersect B) union (A minus B), -// where A is rc, and B is the supplied [firstOfRange, endx) interval. -// -// TODO(time optimization): convert this to a single pass -// algorithm by copying AndNotRunContainer16() and modifying it. -// Current routine is correct but -// makes 2 more passes through the arrays than should be -// strictly necessary. Measure both ways though--this may not matter. -func (rc *runContainer16) Not(firstOfRange, endx int) *runContainer16 { - - if firstOfRange > endx { - panic(fmt.Sprintf("invalid %v = endx > firstOfRange == %v", endx, firstOfRange)) - } - - if firstOfRange >= endx { - return rc.Clone() - } - - a := rc - // algo: - // (!A intersect B) union (A minus B) - - nota := a.invert() - - bs := []interval16{newInterval16Range(uint16(firstOfRange), uint16(endx-1))} - b := newRunContainer16TakeOwnership(bs) - - notAintersectB := nota.intersect(b) - - aMinusB := a.AndNotRunContainer16(b) - - rc2 := notAintersectB.union(aMinusB) - return rc2 -} - -// equals is now logical equals; it does not require the -// same underlying container type. -func (rc *runContainer16) equals(o container) bool { - srb, ok := o.(*runContainer16) - - if !ok { - // maybe value instead of pointer - val, valok := o.(*runContainer16) - if valok { - srb = val - ok = true - } - } - if ok { - // Check if the containers are the same object. - if rc == srb { - return true - } - - if len(srb.iv) != len(rc.iv) { - return false - } - - for i, v := range rc.iv { - if v != srb.iv[i] { - return false - } - } - return true - } - - // use generic comparison - if o.getCardinality() != rc.getCardinality() { - return false - } - rit := rc.getShortIterator() - bit := o.getShortIterator() - - //k := 0 - for rit.hasNext() { - if bit.next() != rit.next() { - return false - } - //k++ - } - return true -} - -func (rc *runContainer16) iaddReturnMinimized(x uint16) container { - rc.Add(x) - return rc -} - -func (rc *runContainer16) iadd(x uint16) (wasNew bool) { - return rc.Add(x) -} - -func (rc *runContainer16) iremoveReturnMinimized(x uint16) container { - rc.removeKey(x) - return rc -} - -func (rc *runContainer16) iremove(x uint16) bool { - return rc.removeKey(x) -} - -func (rc *runContainer16) or(a container) container { - if rc.isFull() { - return rc.clone() - } - switch c := a.(type) { - case *runContainer16: - return rc.union(c) - case *arrayContainer: - return rc.orArray(c) - case *bitmapContainer: - return rc.orBitmapContainer(c) - } - panic("unsupported container type") -} - -func (rc *runContainer16) orCardinality(a container) int { - switch c := a.(type) { - case *runContainer16: - return int(rc.unionCardinality(c)) - case *arrayContainer: - return rc.orArrayCardinality(c) - case *bitmapContainer: - return rc.orBitmapContainerCardinality(c) - } - panic("unsupported container type") -} - -// orBitmapContainer finds the union of rc and bc. -func (rc *runContainer16) orBitmapContainer(bc *bitmapContainer) container { - bc2 := newBitmapContainerFromRun(rc) - return bc2.iorBitmap(bc) -} - -func (rc *runContainer16) andBitmapContainerCardinality(bc *bitmapContainer) int { - answer := 0 - for i := range rc.iv { - answer += bc.getCardinalityInRange(uint(rc.iv[i].start), uint(rc.iv[i].last())+1) - } - //bc.computeCardinality() - return answer -} - -func (rc *runContainer16) orBitmapContainerCardinality(bc *bitmapContainer) int { - return rc.getCardinality() + bc.getCardinality() - rc.andBitmapContainerCardinality(bc) -} - -// orArray finds the union of rc and ac. -func (rc *runContainer16) orArray(ac *arrayContainer) container { - if ac.isEmpty() { - return rc.clone() - } - if rc.isEmpty() { - return ac.clone() - } - intervals, cardMinusOne := runArrayUnionToRuns(rc, ac) - result := newRunContainer16TakeOwnership(intervals) - if len(intervals) >= 2048 && cardMinusOne >= arrayDefaultMaxSize { - return newBitmapContainerFromRun(result) - } - if len(intervals)*2 > 1+int(cardMinusOne) { - return result.toArrayContainer() - } - return result -} - -// orArray finds the union of rc and ac. -func (rc *runContainer16) orArrayCardinality(ac *arrayContainer) int { - return ac.getCardinality() + rc.getCardinality() - rc.andArrayCardinality(ac) -} - -func (rc *runContainer16) ior(a container) container { - if rc.isFull() { - return rc - } - switch c := a.(type) { - case *runContainer16: - return rc.inplaceUnion(c) - case *arrayContainer: - return rc.iorArray(c) - case *bitmapContainer: - return rc.iorBitmapContainer(c) - } - panic("unsupported container type") -} - -func (rc *runContainer16) inplaceUnion(rc2 *runContainer16) container { - for _, p := range rc2.iv { - last := int(p.last()) - for i := int(p.start); i <= last; i++ { - rc.Add(uint16(i)) - } - } - return rc -} - -func (rc *runContainer16) iorBitmapContainer(bc *bitmapContainer) container { - - it := bc.getShortIterator() - for it.hasNext() { - rc.Add(it.next()) - } - return rc -} - -func (rc *runContainer16) iorArray(ac *arrayContainer) container { - if rc.isEmpty() { - return ac.clone() - } - if ac.isEmpty() { - return rc - } - var cardMinusOne uint16 - //TODO: perform the union algorithm in-place using rc.iv - // this can be done with methods like the in-place array container union - // but maybe lazily moving the remaining elements back. - rc.iv, cardMinusOne = runArrayUnionToRuns(rc, ac) - if len(rc.iv) >= 2048 && cardMinusOne >= arrayDefaultMaxSize { - return newBitmapContainerFromRun(rc) - } - if len(rc.iv)*2 > 1+int(cardMinusOne) { - return rc.toArrayContainer() - } - return rc -} - -func runArrayUnionToRuns(rc *runContainer16, ac *arrayContainer) ([]interval16, uint16) { - pos1 := 0 - pos2 := 0 - length1 := len(ac.content) - length2 := len(rc.iv) - target := make([]interval16, 0, len(rc.iv)) - // have to find the first range - // options are - // 1. from array container - // 2. from run container - var previousInterval interval16 - var cardMinusOne uint16 - if ac.content[0] < rc.iv[0].start { - previousInterval.start = ac.content[0] - previousInterval.length = 0 - pos1++ - } else { - previousInterval.start = rc.iv[0].start - previousInterval.length = rc.iv[0].length - pos2++ - } - - for pos1 < length1 || pos2 < length2 { - if pos1 < length1 { - s1 := ac.content[pos1] - if s1 <= previousInterval.start+previousInterval.length { - pos1++ - continue - } - if previousInterval.last() < MaxUint16 && previousInterval.last()+1 == s1 { - previousInterval.length++ - pos1++ - continue - } - } - if pos2 < length2 { - range2 := rc.iv[pos2] - if range2.start <= previousInterval.last() || range2.start > 0 && range2.start-1 == previousInterval.last() { - pos2++ - if previousInterval.last() < range2.last() { - previousInterval.length = range2.last() - previousInterval.start - } - continue - } - } - cardMinusOne += previousInterval.length + 1 - target = append(target, previousInterval) - if pos2 == length2 || pos1 < length1 && ac.content[pos1] < rc.iv[pos2].start { - previousInterval.start = ac.content[pos1] - previousInterval.length = 0 - pos1++ - } else { - previousInterval = rc.iv[pos2] - pos2++ - } - } - cardMinusOne += previousInterval.length - target = append(target, previousInterval) - - return target, cardMinusOne -} - -// lazyIOR is described (not yet implemented) in -// this nice note from @lemire on -// https://github.com/RoaringBitmap/roaring/pull/70#issuecomment-263613737 -// -// Description of lazyOR and lazyIOR from @lemire: -// -// Lazy functions are optional and can be simply -// wrapper around non-lazy functions. -// -// The idea of "laziness" is as follows. It is -// inspired by the concept of lazy evaluation -// you might be familiar with (functional programming -// and all that). So a roaring bitmap is -// such that all its containers are, in some -// sense, chosen to use as little memory as -// possible. This is nice. Also, all bitsets -// are "cardinality aware" so that you can do -// fast rank/select queries, or query the -// cardinality of the whole bitmap... very fast, -// without latency. -// -// However, imagine that you are aggregating 100 -// bitmaps together. So you OR the first two, then OR -// that with the third one and so forth. Clearly, -// intermediate bitmaps don't need to be as -// compressed as possible, right? They can be -// in a "dirty state". You only need the end -// result to be in a nice state... which you -// can achieve by calling repairAfterLazy at the end. -// -// The Java/C code does something special for -// the in-place lazy OR runs. The idea is that -// instead of taking two run containers and -// generating a new one, we actually try to -// do the computation in-place through a -// technique invented by @gssiyankai (pinging him!). -// What you do is you check whether the host -// run container has lots of extra capacity. -// If it does, you move its data at the end of -// the backing array, and then you write -// the answer at the beginning. What this -// trick does is minimize memory allocations. -func (rc *runContainer16) lazyIOR(a container) container { - // not lazy at the moment - return rc.ior(a) -} - -// lazyOR is described above in lazyIOR. -func (rc *runContainer16) lazyOR(a container) container { - // not lazy at the moment - return rc.or(a) -} - -func (rc *runContainer16) intersects(a container) bool { - // TODO: optimize by doing inplace/less allocation - isect := rc.and(a) - return !isect.isEmpty() -} - -func (rc *runContainer16) xor(a container) container { - switch c := a.(type) { - case *arrayContainer: - return rc.xorArray(c) - case *bitmapContainer: - return rc.xorBitmap(c) - case *runContainer16: - return rc.xorRunContainer16(c) - } - panic("unsupported container type") -} - -func (rc *runContainer16) iandNot(a container) container { - switch c := a.(type) { - case *arrayContainer: - return rc.iandNotArray(c) - case *bitmapContainer: - return rc.iandNotBitmap(c) - case *runContainer16: - return rc.iandNotRunContainer16(c) - } - panic("unsupported container type") -} - -// flip the values in the range [firstOfRange,endx) -func (rc *runContainer16) inot(firstOfRange, endx int) container { - if firstOfRange > endx { - panic(fmt.Sprintf("invalid %v = endx > firstOfRange = %v", endx, firstOfRange)) - } - if firstOfRange > endx { - return rc - } - // TODO: minimize copies, do it all inplace; not() makes a copy. - rc = rc.Not(firstOfRange, endx) - return rc -} - -func (rc *runContainer16) rank(x uint16) int { - n := int(len(rc.iv)) - xx := int(x) - w, already, _ := rc.search(xx) - if w < 0 { - return 0 - } - if !already && w == n-1 { - return rc.getCardinality() - } - var rnk int - if !already { - for i := int(0); i <= w; i++ { - rnk += rc.iv[i].runlen() - } - return int(rnk) - } - for i := int(0); i < w; i++ { - rnk += rc.iv[i].runlen() - } - rnk += int(x-rc.iv[w].start) + 1 - return int(rnk) -} - -func (rc *runContainer16) selectInt(x uint16) int { - var offset int - for k := range rc.iv { - nextOffset := offset + rc.iv[k].runlen() - if nextOffset > int(x) { - return int(int(rc.iv[k].start) + (int(x) - offset)) - } - offset = nextOffset - } - panic("cannot select x") -} - -func (rc *runContainer16) andNotRunContainer16(b *runContainer16) container { - return rc.AndNotRunContainer16(b) -} - -func (rc *runContainer16) andNotArray(ac *arrayContainer) container { - rcb := rc.toBitmapContainer() - acb := ac.toBitmapContainer() - return rcb.andNotBitmap(acb) -} - -func (rc *runContainer16) andNotBitmap(bc *bitmapContainer) container { - rcb := rc.toBitmapContainer() - return rcb.andNotBitmap(bc) -} - -func (rc *runContainer16) toBitmapContainer() *bitmapContainer { - bc := newBitmapContainer() - for i := range rc.iv { - bc.iaddRange(int(rc.iv[i].start), int(rc.iv[i].last())+1) - } - bc.computeCardinality() - return bc -} - -func (rc *runContainer16) iandNotRunContainer16(x2 *runContainer16) container { - rcb := rc.toBitmapContainer() - x2b := x2.toBitmapContainer() - rcb.iandNotBitmapSurely(x2b) - // TODO: check size and optimize the return value - // TODO: is inplace modification really required? If not, elide the copy. - rc2 := newRunContainer16FromBitmapContainer(rcb) - *rc = *rc2 - return rc -} - -func (rc *runContainer16) iandNotArray(ac *arrayContainer) container { - rcb := rc.toBitmapContainer() - acb := ac.toBitmapContainer() - rcb.iandNotBitmapSurely(acb) - // TODO: check size and optimize the return value - // TODO: is inplace modification really required? If not, elide the copy. - rc2 := newRunContainer16FromBitmapContainer(rcb) - *rc = *rc2 - return rc -} - -func (rc *runContainer16) iandNotBitmap(bc *bitmapContainer) container { - rcb := rc.toBitmapContainer() - rcb.iandNotBitmapSurely(bc) - // TODO: check size and optimize the return value - // TODO: is inplace modification really required? If not, elide the copy. - rc2 := newRunContainer16FromBitmapContainer(rcb) - *rc = *rc2 - return rc -} - -func (rc *runContainer16) xorRunContainer16(x2 *runContainer16) container { - rcb := rc.toBitmapContainer() - x2b := x2.toBitmapContainer() - return rcb.xorBitmap(x2b) -} - -func (rc *runContainer16) xorArray(ac *arrayContainer) container { - rcb := rc.toBitmapContainer() - acb := ac.toBitmapContainer() - return rcb.xorBitmap(acb) -} - -func (rc *runContainer16) xorBitmap(bc *bitmapContainer) container { - rcb := rc.toBitmapContainer() - return rcb.xorBitmap(bc) -} - -// convert to bitmap or array *if needed* -func (rc *runContainer16) toEfficientContainer() container { - sizeAsRunContainer := rc.getSizeInBytes() - sizeAsBitmapContainer := bitmapContainerSizeInBytes() - card := rc.getCardinality() - sizeAsArrayContainer := arrayContainerSizeInBytes(card) - if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) { - return rc - } - if card <= arrayDefaultMaxSize { - return rc.toArrayContainer() - } - bc := newBitmapContainerFromRun(rc) - return bc -} - -func (rc *runContainer16) toArrayContainer() *arrayContainer { - ac := newArrayContainer() - for i := range rc.iv { - ac.iaddRange(int(rc.iv[i].start), int(rc.iv[i].last())+1) - } - return ac -} - -func newRunContainer16FromContainer(c container) *runContainer16 { - - switch x := c.(type) { - case *runContainer16: - return x.Clone() - case *arrayContainer: - return newRunContainer16FromArray(x) - case *bitmapContainer: - return newRunContainer16FromBitmapContainer(x) - } - panic("unsupported container type") -} - -// And finds the intersection of rc and b. -func (rc *runContainer16) And(b *Bitmap) *Bitmap { - out := NewBitmap() - for _, p := range rc.iv { - plast := p.last() - for i := p.start; i <= plast; i++ { - if b.Contains(uint32(i)) { - out.Add(uint32(i)) - } - } - } - return out -} - -// Xor returns the exclusive-or of rc and b. -func (rc *runContainer16) Xor(b *Bitmap) *Bitmap { - out := b.Clone() - for _, p := range rc.iv { - plast := p.last() - for v := p.start; v <= plast; v++ { - w := uint32(v) - if out.Contains(w) { - out.RemoveRange(uint64(w), uint64(w+1)) - } else { - out.Add(w) - } - } - } - return out -} - -// Or returns the union of rc and b. -func (rc *runContainer16) Or(b *Bitmap) *Bitmap { - out := b.Clone() - for _, p := range rc.iv { - plast := p.last() - for v := p.start; v <= plast; v++ { - out.Add(uint32(v)) - } - } - return out -} - -// serializedSizeInBytes returns the number of bytes of memory -// required by this runContainer16. This is for the -// Roaring format, as specified https://github.com/RoaringBitmap/RoaringFormatSpec/ -func (rc *runContainer16) serializedSizeInBytes() int { - // number of runs in one uint16, then each run - // needs two more uint16 - return 2 + len(rc.iv)*4 -} - -func (rc *runContainer16) addOffset(x uint16) (container, container) { - var low, high *runContainer16 - - if len(rc.iv) == 0 { - return nil, nil - } - - first := uint32(rc.iv[0].start) + uint32(x) - if highbits(first) == 0 { - // Some elements will fall into low part, allocate a container. - // Checking the first one is enough because they are ordered. - low = newRunContainer16() - } - last := uint32(rc.iv[len(rc.iv)-1].start) - last += uint32(rc.iv[len(rc.iv)-1].length) - last += uint32(x) - if highbits(last) > 0 { - // Some elements will fall into high part, allocate a container. - // Checking the last one is enough because they are ordered. - high = newRunContainer16() - } - - for _, iv := range rc.iv { - val := int(iv.start) + int(x) - finalVal := int(val) + int(iv.length) - if val <= 0xffff { - if finalVal <= 0xffff { - low.iv = append(low.iv, interval16{uint16(val), iv.length}) - } else { - low.iv = append(low.iv, interval16{uint16(val), uint16(0xffff - val)}) - high.iv = append(high.iv, interval16{uint16(0), uint16(finalVal & 0xffff)}) - } - } else { - high.iv = append(high.iv, interval16{uint16(val & 0xffff), iv.length}) - } - } - - // Ensure proper nil interface. - if low == nil { - return nil, high - } - if high == nil { - return low, nil - } - - return low, high -} diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization.go b/vendor/github.com/RoaringBitmap/roaring/serialization.go deleted file mode 100644 index dbfecc846d..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/serialization.go +++ /dev/null @@ -1,18 +0,0 @@ -package roaring - -import ( - "encoding/binary" - "io" -) - -// writeTo for runContainer16 follows this -// spec: https://github.com/RoaringBitmap/RoaringFormatSpec -func (b *runContainer16) writeTo(stream io.Writer) (int, error) { - buf := make([]byte, 2+4*len(b.iv)) - binary.LittleEndian.PutUint16(buf[0:], uint16(len(b.iv))) - for i, v := range b.iv { - binary.LittleEndian.PutUint16(buf[2+i*4:], v.start) - binary.LittleEndian.PutUint16(buf[2+2+i*4:], v.length) - } - return stream.Write(buf) -} diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go b/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go deleted file mode 100644 index 7e1f1802cb..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go +++ /dev/null @@ -1,145 +0,0 @@ -//go:build (!amd64 && !386 && !arm && !arm64 && !ppc64le && !mipsle && !mips64le && !mips64p32le && !wasm) || appengine -// +build !amd64,!386,!arm,!arm64,!ppc64le,!mipsle,!mips64le,!mips64p32le,!wasm appengine - -package roaring - -import ( - "encoding/binary" - "errors" - "io" -) - -func (b *arrayContainer) writeTo(stream io.Writer) (int, error) { - buf := make([]byte, 2*len(b.content)) - for i, v := range b.content { - base := i * 2 - buf[base] = byte(v) - buf[base+1] = byte(v >> 8) - } - return stream.Write(buf) -} - -func (b *arrayContainer) readFrom(stream io.Reader) (int, error) { - err := binary.Read(stream, binary.LittleEndian, b.content) - if err != nil { - return 0, err - } - return 2 * len(b.content), nil -} - -func (b *bitmapContainer) writeTo(stream io.Writer) (int, error) { - if b.cardinality <= arrayDefaultMaxSize { - return 0, errors.New("refusing to write bitmap container with cardinality of array container") - } - - // Write set - buf := make([]byte, 8*len(b.bitmap)) - for i, v := range b.bitmap { - base := i * 8 - buf[base] = byte(v) - buf[base+1] = byte(v >> 8) - buf[base+2] = byte(v >> 16) - buf[base+3] = byte(v >> 24) - buf[base+4] = byte(v >> 32) - buf[base+5] = byte(v >> 40) - buf[base+6] = byte(v >> 48) - buf[base+7] = byte(v >> 56) - } - return stream.Write(buf) -} - -func (b *bitmapContainer) readFrom(stream io.Reader) (int, error) { - err := binary.Read(stream, binary.LittleEndian, b.bitmap) - if err != nil { - return 0, err - } - b.computeCardinality() - return 8 * len(b.bitmap), nil -} - -func (bc *bitmapContainer) asLittleEndianByteSlice() []byte { - by := make([]byte, len(bc.bitmap)*8) - for i := range bc.bitmap { - binary.LittleEndian.PutUint64(by[i*8:], bc.bitmap[i]) - } - return by -} - -func uint64SliceAsByteSlice(slice []uint64) []byte { - by := make([]byte, len(slice)*8) - - for i, v := range slice { - binary.LittleEndian.PutUint64(by[i*8:], v) - } - - return by -} - -func uint16SliceAsByteSlice(slice []uint16) []byte { - by := make([]byte, len(slice)*2) - - for i, v := range slice { - binary.LittleEndian.PutUint16(by[i*2:], v) - } - - return by -} - -func interval16SliceAsByteSlice(slice []interval16) []byte { - by := make([]byte, len(slice)*4) - - for i, v := range slice { - binary.LittleEndian.PutUint16(by[i*2:], v.start) - binary.LittleEndian.PutUint16(by[i*2+2:], v.length) - } - - return by -} - -func byteSliceAsUint16Slice(slice []byte) []uint16 { - if len(slice)%2 != 0 { - panic("Slice size should be divisible by 2") - } - - b := make([]uint16, len(slice)/2) - - for i := range b { - b[i] = binary.LittleEndian.Uint16(slice[2*i:]) - } - - return b -} - -func byteSliceAsUint64Slice(slice []byte) []uint64 { - if len(slice)%8 != 0 { - panic("Slice size should be divisible by 8") - } - - b := make([]uint64, len(slice)/8) - - for i := range b { - b[i] = binary.LittleEndian.Uint64(slice[8*i:]) - } - - return b -} - -// Converts a byte slice to a interval16 slice. -// The function assumes that the slice byte buffer is run container data -// encoded according to Roaring Format Spec -func byteSliceAsInterval16Slice(byteSlice []byte) []interval16 { - if len(byteSlice)%4 != 0 { - panic("Slice size should be divisible by 4") - } - - intervalSlice := make([]interval16, len(byteSlice)/4) - - for i := range intervalSlice { - intervalSlice[i] = interval16{ - start: binary.LittleEndian.Uint16(byteSlice[i*4:]), - length: binary.LittleEndian.Uint16(byteSlice[i*4+2:]), - } - } - - return intervalSlice -} diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go deleted file mode 100644 index 6e3a5d554c..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go +++ /dev/null @@ -1,662 +0,0 @@ -//go:build (386 && !appengine) || (amd64 && !appengine) || (arm && !appengine) || (arm64 && !appengine) || (ppc64le && !appengine) || (mipsle && !appengine) || (mips64le && !appengine) || (mips64p32le && !appengine) || (wasm && !appengine) -// +build 386,!appengine amd64,!appengine arm,!appengine arm64,!appengine ppc64le,!appengine mipsle,!appengine mips64le,!appengine mips64p32le,!appengine wasm,!appengine - -package roaring - -import ( - "encoding/binary" - "errors" - "io" - "reflect" - "runtime" - "unsafe" -) - -func (ac *arrayContainer) writeTo(stream io.Writer) (int, error) { - buf := uint16SliceAsByteSlice(ac.content) - return stream.Write(buf) -} - -func (bc *bitmapContainer) writeTo(stream io.Writer) (int, error) { - if bc.cardinality <= arrayDefaultMaxSize { - return 0, errors.New("refusing to write bitmap container with cardinality of array container") - } - buf := uint64SliceAsByteSlice(bc.bitmap) - return stream.Write(buf) -} - -func uint64SliceAsByteSlice(slice []uint64) []byte { - // make a new slice header - header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) - - // update its capacity and length - header.Len *= 8 - header.Cap *= 8 - - // instantiate result and use KeepAlive so data isn't unmapped. - result := *(*[]byte)(unsafe.Pointer(&header)) - runtime.KeepAlive(&slice) - - // return it - return result -} - -func uint16SliceAsByteSlice(slice []uint16) []byte { - // make a new slice header - header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) - - // update its capacity and length - header.Len *= 2 - header.Cap *= 2 - - // instantiate result and use KeepAlive so data isn't unmapped. - result := *(*[]byte)(unsafe.Pointer(&header)) - runtime.KeepAlive(&slice) - - // return it - return result -} - -func interval16SliceAsByteSlice(slice []interval16) []byte { - // make a new slice header - header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) - - // update its capacity and length - header.Len *= 4 - header.Cap *= 4 - - // instantiate result and use KeepAlive so data isn't unmapped. - result := *(*[]byte)(unsafe.Pointer(&header)) - runtime.KeepAlive(&slice) - - // return it - return result -} - -func (bc *bitmapContainer) asLittleEndianByteSlice() []byte { - return uint64SliceAsByteSlice(bc.bitmap) -} - -// Deserialization code follows - -// // -// These methods (byteSliceAsUint16Slice,...) do not make copies, -// they are pointer-based (unsafe). The caller is responsible to -// ensure that the input slice does not get garbage collected, deleted -// or modified while you hold the returned slince. -// // -func byteSliceAsUint16Slice(slice []byte) (result []uint16) { // here we create a new slice holder - if len(slice)%2 != 0 { - panic("Slice size should be divisible by 2") - } - // reference: https://go101.org/article/unsafe.html - - // make a new slice header - bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) - rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) - - // transfer the data from the given slice to a new variable (our result) - rHeader.Data = bHeader.Data - rHeader.Len = bHeader.Len / 2 - rHeader.Cap = bHeader.Cap / 2 - - // instantiate result and use KeepAlive so data isn't unmapped. - runtime.KeepAlive(&slice) // it is still crucial, GC can free it) - - // return result - return -} - -func byteSliceAsUint64Slice(slice []byte) (result []uint64) { - if len(slice)%8 != 0 { - panic("Slice size should be divisible by 8") - } - // reference: https://go101.org/article/unsafe.html - - // make a new slice header - bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) - rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) - - // transfer the data from the given slice to a new variable (our result) - rHeader.Data = bHeader.Data - rHeader.Len = bHeader.Len / 8 - rHeader.Cap = bHeader.Cap / 8 - - // instantiate result and use KeepAlive so data isn't unmapped. - runtime.KeepAlive(&slice) // it is still crucial, GC can free it) - - // return result - return -} - -func byteSliceAsInterval16Slice(slice []byte) (result []interval16) { - if len(slice)%4 != 0 { - panic("Slice size should be divisible by 4") - } - // reference: https://go101.org/article/unsafe.html - - // make a new slice header - bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) - rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) - - // transfer the data from the given slice to a new variable (our result) - rHeader.Data = bHeader.Data - rHeader.Len = bHeader.Len / 4 - rHeader.Cap = bHeader.Cap / 4 - - // instantiate result and use KeepAlive so data isn't unmapped. - runtime.KeepAlive(&slice) // it is still crucial, GC can free it) - - // return result - return -} - -func byteSliceAsContainerSlice(slice []byte) (result []container) { - var c container - containerSize := int(unsafe.Sizeof(c)) - - if len(slice)%containerSize != 0 { - panic("Slice size should be divisible by unsafe.Sizeof(container)") - } - // reference: https://go101.org/article/unsafe.html - - // make a new slice header - bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) - rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) - - // transfer the data from the given slice to a new variable (our result) - rHeader.Data = bHeader.Data - rHeader.Len = bHeader.Len / containerSize - rHeader.Cap = bHeader.Cap / containerSize - - // instantiate result and use KeepAlive so data isn't unmapped. - runtime.KeepAlive(&slice) // it is still crucial, GC can free it) - - // return result - return -} - -func byteSliceAsBitsetSlice(slice []byte) (result []bitmapContainer) { - bitsetSize := int(unsafe.Sizeof(bitmapContainer{})) - if len(slice)%bitsetSize != 0 { - panic("Slice size should be divisible by unsafe.Sizeof(bitmapContainer)") - } - // reference: https://go101.org/article/unsafe.html - - // make a new slice header - bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) - rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) - - // transfer the data from the given slice to a new variable (our result) - rHeader.Data = bHeader.Data - rHeader.Len = bHeader.Len / bitsetSize - rHeader.Cap = bHeader.Cap / bitsetSize - - // instantiate result and use KeepAlive so data isn't unmapped. - runtime.KeepAlive(&slice) // it is still crucial, GC can free it) - - // return result - return -} - -func byteSliceAsArraySlice(slice []byte) (result []arrayContainer) { - arraySize := int(unsafe.Sizeof(arrayContainer{})) - if len(slice)%arraySize != 0 { - panic("Slice size should be divisible by unsafe.Sizeof(arrayContainer)") - } - // reference: https://go101.org/article/unsafe.html - - // make a new slice header - bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) - rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) - - // transfer the data from the given slice to a new variable (our result) - rHeader.Data = bHeader.Data - rHeader.Len = bHeader.Len / arraySize - rHeader.Cap = bHeader.Cap / arraySize - - // instantiate result and use KeepAlive so data isn't unmapped. - runtime.KeepAlive(&slice) // it is still crucial, GC can free it) - - // return result - return -} - -func byteSliceAsRun16Slice(slice []byte) (result []runContainer16) { - run16Size := int(unsafe.Sizeof(runContainer16{})) - if len(slice)%run16Size != 0 { - panic("Slice size should be divisible by unsafe.Sizeof(runContainer16)") - } - // reference: https://go101.org/article/unsafe.html - - // make a new slice header - bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) - rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) - - // transfer the data from the given slice to a new variable (our result) - rHeader.Data = bHeader.Data - rHeader.Len = bHeader.Len / run16Size - rHeader.Cap = bHeader.Cap / run16Size - - // instantiate result and use KeepAlive so data isn't unmapped. - runtime.KeepAlive(&slice) // it is still crucial, GC can free it) - - // return result - return -} - -func byteSliceAsBoolSlice(slice []byte) (result []bool) { - boolSize := int(unsafe.Sizeof(true)) - if len(slice)%boolSize != 0 { - panic("Slice size should be divisible by unsafe.Sizeof(bool)") - } - // reference: https://go101.org/article/unsafe.html - - // make a new slice header - bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice)) - rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result)) - - // transfer the data from the given slice to a new variable (our result) - rHeader.Data = bHeader.Data - rHeader.Len = bHeader.Len / boolSize - rHeader.Cap = bHeader.Cap / boolSize - - // instantiate result and use KeepAlive so data isn't unmapped. - runtime.KeepAlive(&slice) // it is still crucial, GC can free it) - - // return result - return -} - -// FrozenView creates a static view of a serialized bitmap stored in buf. -// It uses CRoaring's frozen bitmap format. -// -// The format specification is available here: -// https://github.com/RoaringBitmap/CRoaring/blob/2c867e9f9c9e2a3a7032791f94c4c7ae3013f6e0/src/roaring.c#L2756-L2783 -// -// The provided byte array (buf) is expected to be a constant. -// The function makes the best effort attempt not to copy data. -// Only little endian is supported. The function will err if it detects a big -// endian serialized file. -// You should take care not to modify buff as it will likely result in -// unexpected program behavior. -// If said buffer comes from a memory map, it's advisable to give it read -// only permissions, either at creation or by calling Mprotect from the -// golang.org/x/sys/unix package. -// -// Resulting bitmaps are effectively immutable in the following sense: -// a copy-on-write marker is used so that when you modify the resulting -// bitmap, copies of selected data (containers) are made. -// You should *not* change the copy-on-write status of the resulting -// bitmaps (SetCopyOnWrite). -// -// If buf becomes unavailable, then a bitmap created with -// FromBuffer would be effectively broken. Furthermore, any -// bitmap derived from this bitmap (e.g., via Or, And) might -// also be broken. Thus, before making buf unavailable, you should -// call CloneCopyOnWriteContainers on all such bitmaps. -func (rb *Bitmap) FrozenView(buf []byte) error { - return rb.highlowcontainer.frozenView(buf) -} - -/* Verbatim specification from CRoaring. - * - * FROZEN SERIALIZATION FORMAT DESCRIPTION - * - * -- (beginning must be aligned by 32 bytes) -- - * uint64_t[BITSET_CONTAINER_SIZE_IN_WORDS * num_bitset_containers] - * rle16_t[total number of rle elements in all run containers] - * uint16_t[total number of array elements in all array containers] - * uint16_t[num_containers] - * uint16_t[num_containers] - * uint8_t[num_containers] - *
uint32_t - * - *
is a 4-byte value which is a bit union of frozenCookie (15 bits) - * and the number of containers (17 bits). - * - * stores number of elements for every container. - * Its meaning depends on container type. - * For array and bitset containers, this value is the container cardinality minus one. - * For run container, it is the number of rle_t elements (n_runs). - * - * ,, are flat arrays of elements of - * all containers of respective type. - * - * <*_data> and are kept close together because they are not accessed - * during deserilization. This may reduce IO in case of large mmaped bitmaps. - * All members have their native alignments during deserilization except
, - * which is not guaranteed to be aligned by 4 bytes. - */ -const frozenCookie = 13766 - -var ( - // ErrFrozenBitmapInvalidCookie is returned when the header does not contain the frozenCookie. - ErrFrozenBitmapInvalidCookie = errors.New("header does not contain the frozenCookie") - // ErrFrozenBitmapBigEndian is returned when the header is big endian. - ErrFrozenBitmapBigEndian = errors.New("loading big endian frozen bitmaps is not supported") - // ErrFrozenBitmapIncomplete is returned when the buffer is too small to contain a frozen bitmap. - ErrFrozenBitmapIncomplete = errors.New("input buffer too small to contain a frozen bitmap") - // ErrFrozenBitmapOverpopulated is returned when the number of containers is too large. - ErrFrozenBitmapOverpopulated = errors.New("too many containers") - // ErrFrozenBitmapUnexpectedData is returned when the buffer contains unexpected data. - ErrFrozenBitmapUnexpectedData = errors.New("spurious data in input") - // ErrFrozenBitmapInvalidTypecode is returned when the typecode is invalid. - ErrFrozenBitmapInvalidTypecode = errors.New("unrecognized typecode") - // ErrFrozenBitmapBufferTooSmall is returned when the buffer is too small. - ErrFrozenBitmapBufferTooSmall = errors.New("buffer too small") -) - -func (ra *roaringArray) frozenView(buf []byte) error { - if len(buf) < 4 { - return ErrFrozenBitmapIncomplete - } - - headerBE := binary.BigEndian.Uint32(buf[len(buf)-4:]) - if headerBE&0x7fff == frozenCookie { - return ErrFrozenBitmapBigEndian - } - - header := binary.LittleEndian.Uint32(buf[len(buf)-4:]) - buf = buf[:len(buf)-4] - - if header&0x7fff != frozenCookie { - return ErrFrozenBitmapInvalidCookie - } - - nCont := int(header >> 15) - if nCont > (1 << 16) { - return ErrFrozenBitmapOverpopulated - } - - // 1 byte per type, 2 bytes per key, 2 bytes per count. - if len(buf) < 5*nCont { - return ErrFrozenBitmapIncomplete - } - - types := buf[len(buf)-nCont:] - buf = buf[:len(buf)-nCont] - - counts := byteSliceAsUint16Slice(buf[len(buf)-2*nCont:]) - buf = buf[:len(buf)-2*nCont] - - keys := byteSliceAsUint16Slice(buf[len(buf)-2*nCont:]) - buf = buf[:len(buf)-2*nCont] - - nBitmap, nArray, nRun := 0, 0, 0 - nArrayEl, nRunEl := 0, 0 - for i, t := range types { - switch t { - case 1: - nBitmap++ - case 2: - nArray++ - nArrayEl += int(counts[i]) + 1 - case 3: - nRun++ - nRunEl += int(counts[i]) - default: - return ErrFrozenBitmapInvalidTypecode - } - } - - if len(buf) < (1<<13)*nBitmap+4*nRunEl+2*nArrayEl { - return ErrFrozenBitmapIncomplete - } - - bitsetsArena := byteSliceAsUint64Slice(buf[:(1<<13)*nBitmap]) - buf = buf[(1<<13)*nBitmap:] - - runsArena := byteSliceAsInterval16Slice(buf[:4*nRunEl]) - buf = buf[4*nRunEl:] - - arraysArena := byteSliceAsUint16Slice(buf[:2*nArrayEl]) - buf = buf[2*nArrayEl:] - - if len(buf) != 0 { - return ErrFrozenBitmapUnexpectedData - } - - var c container - containersSz := int(unsafe.Sizeof(c)) * nCont - bitsetsSz := int(unsafe.Sizeof(bitmapContainer{})) * nBitmap - arraysSz := int(unsafe.Sizeof(arrayContainer{})) * nArray - runsSz := int(unsafe.Sizeof(runContainer16{})) * nRun - needCOWSz := int(unsafe.Sizeof(true)) * nCont - - bitmapArenaSz := containersSz + bitsetsSz + arraysSz + runsSz + needCOWSz - bitmapArena := make([]byte, bitmapArenaSz) - - containers := byteSliceAsContainerSlice(bitmapArena[:containersSz]) - bitmapArena = bitmapArena[containersSz:] - - bitsets := byteSliceAsBitsetSlice(bitmapArena[:bitsetsSz]) - bitmapArena = bitmapArena[bitsetsSz:] - - arrays := byteSliceAsArraySlice(bitmapArena[:arraysSz]) - bitmapArena = bitmapArena[arraysSz:] - - runs := byteSliceAsRun16Slice(bitmapArena[:runsSz]) - bitmapArena = bitmapArena[runsSz:] - - needCOW := byteSliceAsBoolSlice(bitmapArena) - - iBitset, iArray, iRun := 0, 0, 0 - for i, t := range types { - needCOW[i] = true - - switch t { - case 1: - containers[i] = &bitsets[iBitset] - bitsets[iBitset].cardinality = int(counts[i]) + 1 - bitsets[iBitset].bitmap = bitsetsArena[:1024] - bitsetsArena = bitsetsArena[1024:] - iBitset++ - case 2: - containers[i] = &arrays[iArray] - sz := int(counts[i]) + 1 - arrays[iArray].content = arraysArena[:sz] - arraysArena = arraysArena[sz:] - iArray++ - case 3: - containers[i] = &runs[iRun] - runs[iRun].iv = runsArena[:counts[i]] - runsArena = runsArena[counts[i]:] - iRun++ - } - } - - // Not consuming the full input is a bug. - if iBitset != nBitmap || len(bitsetsArena) != 0 || - iArray != nArray || len(arraysArena) != 0 || - iRun != nRun || len(runsArena) != 0 { - panic("we missed something") - } - - ra.keys = keys - ra.containers = containers - ra.needCopyOnWrite = needCOW - ra.copyOnWrite = true - - return nil -} - -// GetFrozenSizeInBytes returns the size in bytes of the frozen bitmap. -func (rb *Bitmap) GetFrozenSizeInBytes() uint64 { - nBits, nArrayEl, nRunEl := uint64(0), uint64(0), uint64(0) - for _, c := range rb.highlowcontainer.containers { - switch v := c.(type) { - case *bitmapContainer: - nBits++ - case *arrayContainer: - nArrayEl += uint64(len(v.content)) - case *runContainer16: - nRunEl += uint64(len(v.iv)) - } - } - return 4 + 5*uint64(len(rb.highlowcontainer.containers)) + - (nBits << 13) + 2*nArrayEl + 4*nRunEl -} - -// Freeze serializes the bitmap in the CRoaring's frozen format. -func (rb *Bitmap) Freeze() ([]byte, error) { - sz := rb.GetFrozenSizeInBytes() - buf := make([]byte, sz) - _, err := rb.FreezeTo(buf) - return buf, err -} - -// FreezeTo serializes the bitmap in the CRoaring's frozen format. -func (rb *Bitmap) FreezeTo(buf []byte) (int, error) { - containers := rb.highlowcontainer.containers - nCont := len(containers) - - nBits, nArrayEl, nRunEl := 0, 0, 0 - for _, c := range containers { - switch v := c.(type) { - case *bitmapContainer: - nBits++ - case *arrayContainer: - nArrayEl += len(v.content) - case *runContainer16: - nRunEl += len(v.iv) - } - } - - serialSize := 4 + 5*nCont + (1<<13)*nBits + 4*nRunEl + 2*nArrayEl - if len(buf) < serialSize { - return 0, ErrFrozenBitmapBufferTooSmall - } - - bitsArena := byteSliceAsUint64Slice(buf[:(1<<13)*nBits]) - buf = buf[(1<<13)*nBits:] - - runsArena := byteSliceAsInterval16Slice(buf[:4*nRunEl]) - buf = buf[4*nRunEl:] - - arraysArena := byteSliceAsUint16Slice(buf[:2*nArrayEl]) - buf = buf[2*nArrayEl:] - - keys := byteSliceAsUint16Slice(buf[:2*nCont]) - buf = buf[2*nCont:] - - counts := byteSliceAsUint16Slice(buf[:2*nCont]) - buf = buf[2*nCont:] - - types := buf[:nCont] - buf = buf[nCont:] - - header := uint32(frozenCookie | (nCont << 15)) - binary.LittleEndian.PutUint32(buf[:4], header) - - copy(keys, rb.highlowcontainer.keys[:]) - - for i, c := range containers { - switch v := c.(type) { - case *bitmapContainer: - copy(bitsArena, v.bitmap) - bitsArena = bitsArena[1024:] - counts[i] = uint16(v.cardinality - 1) - types[i] = 1 - case *arrayContainer: - copy(arraysArena, v.content) - arraysArena = arraysArena[len(v.content):] - elems := len(v.content) - counts[i] = uint16(elems - 1) - types[i] = 2 - case *runContainer16: - copy(runsArena, v.iv) - runs := len(v.iv) - runsArena = runsArena[runs:] - counts[i] = uint16(runs) - types[i] = 3 - } - } - - return serialSize, nil -} - -// WriteFrozenTo serializes the bitmap in the CRoaring's frozen format. -func (rb *Bitmap) WriteFrozenTo(wr io.Writer) (int, error) { - // FIXME: this is a naive version that iterates 4 times through the - // containers and allocates 3*len(containers) bytes; it's quite likely - // it can be done more efficiently. - containers := rb.highlowcontainer.containers - written := 0 - - for _, c := range containers { - c, ok := c.(*bitmapContainer) - if !ok { - continue - } - n, err := wr.Write(uint64SliceAsByteSlice(c.bitmap)) - written += n - if err != nil { - return written, err - } - } - - for _, c := range containers { - c, ok := c.(*runContainer16) - if !ok { - continue - } - n, err := wr.Write(interval16SliceAsByteSlice(c.iv)) - written += n - if err != nil { - return written, err - } - } - - for _, c := range containers { - c, ok := c.(*arrayContainer) - if !ok { - continue - } - n, err := wr.Write(uint16SliceAsByteSlice(c.content)) - written += n - if err != nil { - return written, err - } - } - - n, err := wr.Write(uint16SliceAsByteSlice(rb.highlowcontainer.keys)) - written += n - if err != nil { - return written, err - } - - countTypeBuf := make([]byte, 3*len(containers)) - counts := byteSliceAsUint16Slice(countTypeBuf[:2*len(containers)]) - types := countTypeBuf[2*len(containers):] - - for i, c := range containers { - switch c := c.(type) { - case *bitmapContainer: - counts[i] = uint16(c.cardinality - 1) - types[i] = 1 - case *arrayContainer: - elems := len(c.content) - counts[i] = uint16(elems - 1) - types[i] = 2 - case *runContainer16: - runs := len(c.iv) - counts[i] = uint16(runs) - types[i] = 3 - } - } - - n, err = wr.Write(countTypeBuf) - written += n - if err != nil { - return written, err - } - - header := uint32(frozenCookie | (len(containers) << 15)) - if err := binary.Write(wr, binary.LittleEndian, header); err != nil { - return written, err - } - written += 4 - - return written, nil -} diff --git a/vendor/github.com/RoaringBitmap/roaring/serializationfuzz.go b/vendor/github.com/RoaringBitmap/roaring/serializationfuzz.go deleted file mode 100644 index c7fed02b13..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/serializationfuzz.go +++ /dev/null @@ -1,22 +0,0 @@ -//go:build gofuzz -// +build gofuzz - -package roaring - -import "bytes" - -func FuzzSerializationStream(data []byte) int { - newrb := NewBitmap() - if _, err := newrb.ReadFrom(bytes.NewReader(data)); err != nil { - return 0 - } - return 1 -} - -func FuzzSerializationBuffer(data []byte) int { - newrb := NewBitmap() - if _, err := newrb.FromBuffer(data); err != nil { - return 0 - } - return 1 -} diff --git a/vendor/github.com/RoaringBitmap/roaring/setutil.go b/vendor/github.com/RoaringBitmap/roaring/setutil.go deleted file mode 100644 index 663c4fa37e..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/setutil.go +++ /dev/null @@ -1,550 +0,0 @@ -package roaring - -func equal(a, b []uint16) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} - -func difference(set1 []uint16, set2 []uint16, buffer []uint16) int { - if 0 == len(set2) { - buffer = buffer[:len(set1)] - for k := 0; k < len(set1); k++ { - buffer[k] = set1[k] - } - return len(set1) - } - if 0 == len(set1) { - return 0 - } - pos := 0 - k1 := 0 - k2 := 0 - buffer = buffer[:cap(buffer)] - s1 := set1[k1] - s2 := set2[k2] - for { - if s1 < s2 { - buffer[pos] = s1 - pos++ - k1++ - if k1 >= len(set1) { - break - } - s1 = set1[k1] - } else if s1 == s2 { - k1++ - k2++ - if k1 >= len(set1) { - break - } - s1 = set1[k1] - if k2 >= len(set2) { - for ; k1 < len(set1); k1++ { - buffer[pos] = set1[k1] - pos++ - } - break - } - s2 = set2[k2] - } else { // if (val1>val2) - k2++ - if k2 >= len(set2) { - for ; k1 < len(set1); k1++ { - buffer[pos] = set1[k1] - pos++ - } - break - } - s2 = set2[k2] - } - } - return pos - -} - -func exclusiveUnion2by2(set1 []uint16, set2 []uint16, buffer []uint16) int { - if 0 == len(set2) { - buffer = buffer[:len(set1)] - copy(buffer, set1[:]) - return len(set1) - } - if 0 == len(set1) { - buffer = buffer[:len(set2)] - copy(buffer, set2[:]) - return len(set2) - } - pos := 0 - k1 := 0 - k2 := 0 - s1 := set1[k1] - s2 := set2[k2] - buffer = buffer[:cap(buffer)] - for { - if s1 < s2 { - buffer[pos] = s1 - pos++ - k1++ - if k1 >= len(set1) { - for ; k2 < len(set2); k2++ { - buffer[pos] = set2[k2] - pos++ - } - break - } - s1 = set1[k1] - } else if s1 == s2 { - k1++ - k2++ - if k1 >= len(set1) { - for ; k2 < len(set2); k2++ { - buffer[pos] = set2[k2] - pos++ - } - break - } - if k2 >= len(set2) { - for ; k1 < len(set1); k1++ { - buffer[pos] = set1[k1] - pos++ - } - break - } - s1 = set1[k1] - s2 = set2[k2] - } else { // if (val1>val2) - buffer[pos] = s2 - pos++ - k2++ - if k2 >= len(set2) { - for ; k1 < len(set1); k1++ { - buffer[pos] = set1[k1] - pos++ - } - break - } - s2 = set2[k2] - } - } - return pos -} - -func union2by2Cardinality(set1 []uint16, set2 []uint16) int { - pos := 0 - k1 := 0 - k2 := 0 - if 0 == len(set2) { - return len(set1) - } - if 0 == len(set1) { - return len(set2) - } - s1 := set1[k1] - s2 := set2[k2] - for { - if s1 < s2 { - pos++ - k1++ - if k1 >= len(set1) { - pos += len(set2) - k2 - break - } - s1 = set1[k1] - } else if s1 == s2 { - pos++ - k1++ - k2++ - if k1 >= len(set1) { - pos += len(set2) - k2 - break - } - if k2 >= len(set2) { - pos += len(set1) - k1 - break - } - s1 = set1[k1] - s2 = set2[k2] - } else { // if (set1[k1]>set2[k2]) - pos++ - k2++ - if k2 >= len(set2) { - pos += len(set1) - k1 - break - } - s2 = set2[k2] - } - } - return pos -} - -func intersection2by2( - set1 []uint16, - set2 []uint16, - buffer []uint16) int { - - if len(set1)*64 < len(set2) { - return onesidedgallopingintersect2by2(set1, set2, buffer) - } else if len(set2)*64 < len(set1) { - return onesidedgallopingintersect2by2(set2, set1, buffer) - } else { - return localintersect2by2(set1, set2, buffer) - } -} - -func intersection2by2Cardinality( - set1 []uint16, - set2 []uint16) int { - - if len(set1)*64 < len(set2) { - return onesidedgallopingintersect2by2Cardinality(set1, set2) - } else if len(set2)*64 < len(set1) { - return onesidedgallopingintersect2by2Cardinality(set2, set1) - } else { - return localintersect2by2Cardinality(set1, set2) - } -} - -func intersects2by2( - set1 []uint16, - set2 []uint16) bool { - // could be optimized if one set is much larger than the other one - if (0 == len(set1)) || (0 == len(set2)) { - return false - } - k1 := 0 - k2 := 0 - s1 := set1[k1] - s2 := set2[k2] -mainwhile: - for { - - if s2 < s1 { - for { - k2++ - if k2 == len(set2) { - break mainwhile - } - s2 = set2[k2] - if s2 >= s1 { - break - } - } - } - if s1 < s2 { - for { - k1++ - if k1 == len(set1) { - break mainwhile - } - s1 = set1[k1] - if s1 >= s2 { - break - } - } - - } else { - // (set2[k2] == set1[k1]) - return true - } - } - return false -} - -func localintersect2by2( - set1 []uint16, - set2 []uint16, - buffer []uint16) int { - - if (0 == len(set1)) || (0 == len(set2)) { - return 0 - } - k1 := 0 - k2 := 0 - pos := 0 - buffer = buffer[:cap(buffer)] - s1 := set1[k1] - s2 := set2[k2] -mainwhile: - for { - if s2 < s1 { - for { - k2++ - if k2 == len(set2) { - break mainwhile - } - s2 = set2[k2] - if s2 >= s1 { - break - } - } - } - if s1 < s2 { - for { - k1++ - if k1 == len(set1) { - break mainwhile - } - s1 = set1[k1] - if s1 >= s2 { - break - } - } - - } else { - // (set2[k2] == set1[k1]) - buffer[pos] = s1 - pos++ - k1++ - if k1 == len(set1) { - break - } - s1 = set1[k1] - k2++ - if k2 == len(set2) { - break - } - s2 = set2[k2] - } - } - return pos -} - -func localintersect2by2Cardinality( - set1 []uint16, - set2 []uint16) int { - - if (0 == len(set1)) || (0 == len(set2)) { - return 0 - } - k1 := 0 - k2 := 0 - pos := 0 - s1 := set1[k1] - s2 := set2[k2] -mainwhile: - for { - if s2 < s1 { - for { - k2++ - if k2 == len(set2) { - break mainwhile - } - s2 = set2[k2] - if s2 >= s1 { - break - } - } - } - if s1 < s2 { - for { - k1++ - if k1 == len(set1) { - break mainwhile - } - s1 = set1[k1] - if s1 >= s2 { - break - } - } - - } else { - // (set2[k2] == set1[k1]) - pos++ - k1++ - if k1 == len(set1) { - break - } - s1 = set1[k1] - k2++ - if k2 == len(set2) { - break - } - s2 = set2[k2] - } - } - return pos -} - -func advanceUntil( - array []uint16, - pos int, - length int, - min uint16) int { - lower := pos + 1 - - if lower >= length || array[lower] >= min { - return lower - } - - spansize := 1 - - for lower+spansize < length && array[lower+spansize] < min { - spansize *= 2 - } - var upper int - if lower+spansize < length { - upper = lower + spansize - } else { - upper = length - 1 - } - - if array[upper] == min { - return upper - } - - if array[upper] < min { - // means - // array - // has no - // item - // >= min - // pos = array.length; - return length - } - - // we know that the next-smallest span was too small - lower += (spansize >> 1) - - mid := 0 - for lower+1 != upper { - mid = (lower + upper) >> 1 - if array[mid] == min { - return mid - } else if array[mid] < min { - lower = mid - } else { - upper = mid - } - } - return upper - -} - -func onesidedgallopingintersect2by2( - smallset []uint16, - largeset []uint16, - buffer []uint16) int { - - if 0 == len(smallset) { - return 0 - } - buffer = buffer[:cap(buffer)] - k1 := 0 - k2 := 0 - pos := 0 - s1 := largeset[k1] - s2 := smallset[k2] -mainwhile: - - for { - if s1 < s2 { - k1 = advanceUntil(largeset, k1, len(largeset), s2) - if k1 == len(largeset) { - break mainwhile - } - s1 = largeset[k1] - } - if s2 < s1 { - k2++ - if k2 == len(smallset) { - break mainwhile - } - s2 = smallset[k2] - } else { - - buffer[pos] = s2 - pos++ - k2++ - if k2 == len(smallset) { - break - } - s2 = smallset[k2] - k1 = advanceUntil(largeset, k1, len(largeset), s2) - if k1 == len(largeset) { - break mainwhile - } - s1 = largeset[k1] - } - - } - return pos -} - -func onesidedgallopingintersect2by2Cardinality( - smallset []uint16, - largeset []uint16) int { - - if 0 == len(smallset) { - return 0 - } - k1 := 0 - k2 := 0 - pos := 0 - s1 := largeset[k1] - s2 := smallset[k2] -mainwhile: - - for { - if s1 < s2 { - k1 = advanceUntil(largeset, k1, len(largeset), s2) - if k1 == len(largeset) { - break mainwhile - } - s1 = largeset[k1] - } - if s2 < s1 { - k2++ - if k2 == len(smallset) { - break mainwhile - } - s2 = smallset[k2] - } else { - - pos++ - k2++ - if k2 == len(smallset) { - break - } - s2 = smallset[k2] - k1 = advanceUntil(largeset, k1, len(largeset), s2) - if k1 == len(largeset) { - break mainwhile - } - s1 = largeset[k1] - } - - } - return pos -} - -func binarySearch(array []uint16, ikey uint16) int { - low := 0 - high := len(array) - 1 - for low+16 <= high { - middleIndex := int(uint32(low+high) >> 1) - middleValue := array[middleIndex] - if middleValue < ikey { - low = middleIndex + 1 - } else if middleValue > ikey { - high = middleIndex - 1 - } else { - return middleIndex - } - } - for ; low <= high; low++ { - val := array[low] - if val >= ikey { - if val == ikey { - return low - } - break - } - } - return -(low + 1) -} diff --git a/vendor/github.com/RoaringBitmap/roaring/setutil_arm64.go b/vendor/github.com/RoaringBitmap/roaring/setutil_arm64.go deleted file mode 100644 index 3e089650ed..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/setutil_arm64.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build arm64 && !gccgo && !appengine -// +build arm64,!gccgo,!appengine - -package roaring - -//go:noescape -func union2by2(set1 []uint16, set2 []uint16, buffer []uint16) (size int) diff --git a/vendor/github.com/RoaringBitmap/roaring/setutil_arm64.s b/vendor/github.com/RoaringBitmap/roaring/setutil_arm64.s deleted file mode 100644 index e4f0f20473..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/setutil_arm64.s +++ /dev/null @@ -1,132 +0,0 @@ -// +build arm64,!gccgo,!appengine - -#include "textflag.h" - - -// This implements union2by2 using golang's version of arm64 assembly -// The algorithm is very similar to the generic one, -// but makes better use of arm64 features so is notably faster. -// The basic algorithm structure is as follows: -// 1. If either set is empty, copy the other set into the buffer and return the length -// 2. Otherwise, load the first element of each set into a variable (s1 and s2). -// 3. a. Compare the values of s1 and s2. - // b. add the smaller one to the buffer. - // c. perform a bounds check before incrementing. - // If one set is finished, copy the rest of the other set over. - // d. update s1 and or s2 to the next value, continue loop. - // - // Past the fact of the algorithm, this code makes use of several arm64 features - // Condition Codes: - // arm64's CMP operation sets 4 bits that can be used for branching, - // rather than just true or false. - // As a consequence, a single comparison gives enough information to distinguish the three cases - // - // Post-increment pointers after load/store: - // Instructions like `MOVHU.P 2(R0), R6` - // increment the register by a specified amount, in this example 2. - // Because uint16's are exactly 2 bytes and the length of the slices - // is part of the slice header, - // there is no need to separately track the index into the slice. - // Instead, the code can calculate the final read value and compare against that, - // using the post-increment reads to move the pointers along. - // - // TODO: CALL out to memmove once the list is exhausted. - // Right now it moves the necessary shorts so that the remaining count - // is a multiple of 4 and then copies 64 bits at a time. - -TEXT ·union2by2(SB), NOSPLIT, $0-80 - // R0, R1, and R2 for the pointers to the three slices - MOVD set1+0(FP), R0 - MOVD set2+24(FP), R1 - MOVD buffer+48(FP), R2 - - //R3 and R4 will be the values at which we will have finished reading set1 and set2. - // R3 should be R0 + 2 * set1_len+8(FP) - MOVD set1_len+8(FP), R3 - MOVD set2_len+32(FP), R4 - - ADD R3<<1, R0, R3 - ADD R4<<1, R1, R4 - - - //Rather than counting the number of elements added separately - //Save the starting register of buffer. - MOVD buffer+48(FP), R5 - - // set1 is empty, just flush set2 - CMP R0, R3 - BEQ flush_right - - // set2 is empty, just flush set1 - CMP R1, R4 - BEQ flush_left - - // R6, R7 are the working space for s1 and s2 - MOVD ZR, R6 - MOVD ZR, R7 - - MOVHU.P 2(R0), R6 - MOVHU.P 2(R1), R7 -loop: - - CMP R6, R7 - BEQ pop_both // R6 == R7 - BLS pop_right // R6 > R7 -//pop_left: // R6 < R7 - MOVHU.P R6, 2(R2) - CMP R0, R3 - BEQ pop_then_flush_right - MOVHU.P 2(R0), R6 - JMP loop -pop_both: - MOVHU.P R6, 2(R2) //could also use R7, since they are equal - CMP R0, R3 - BEQ flush_right - CMP R1, R4 - BEQ flush_left - MOVHU.P 2(R0), R6 - MOVHU.P 2(R1), R7 - JMP loop -pop_right: - MOVHU.P R7, 2(R2) - CMP R1, R4 - BEQ pop_then_flush_left - MOVHU.P 2(R1), R7 - JMP loop - -pop_then_flush_right: - MOVHU.P R7, 2(R2) -flush_right: - MOVD R1, R0 - MOVD R4, R3 - JMP flush_left -pop_then_flush_left: - MOVHU.P R6, 2(R2) -flush_left: - CMP R0, R3 - BEQ return - //figure out how many bytes to slough off. Must be a multiple of two - SUB R0, R3, R4 - ANDS $6, R4 - BEQ long_flush //handles the 0 mod 8 case - SUBS $4, R4, R4 // since possible values are 2, 4, 6, this splits evenly - BLT pop_single // exactly the 2 case - MOVW.P 4(R0), R6 - MOVW.P R6, 4(R2) - BEQ long_flush // we're now aligned by 64 bits, as R4==4, otherwise 2 more -pop_single: - MOVHU.P 2(R0), R6 - MOVHU.P R6, 2(R2) -long_flush: - // at this point we know R3 - R0 is a multiple of 8. - CMP R0, R3 - BEQ return - MOVD.P 8(R0), R6 - MOVD.P R6, 8(R2) - JMP long_flush -return: - // number of shorts written is (R5 - R2) >> 1 - SUB R5, R2 - LSR $1, R2, R2 - MOVD R2, size+72(FP) - RET diff --git a/vendor/github.com/RoaringBitmap/roaring/setutil_generic.go b/vendor/github.com/RoaringBitmap/roaring/setutil_generic.go deleted file mode 100644 index 4755fd5482..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/setutil_generic.go +++ /dev/null @@ -1,64 +0,0 @@ -//go:build !arm64 || gccgo || appengine -// +build !arm64 gccgo appengine - -package roaring - -func union2by2(set1 []uint16, set2 []uint16, buffer []uint16) int { - pos := 0 - k1 := 0 - k2 := 0 - if 0 == len(set2) { - buffer = buffer[:len(set1)] - copy(buffer, set1[:]) - return len(set1) - } - if 0 == len(set1) { - buffer = buffer[:len(set2)] - copy(buffer, set2[:]) - return len(set2) - } - s1 := set1[k1] - s2 := set2[k2] - buffer = buffer[:cap(buffer)] - for { - if s1 < s2 { - buffer[pos] = s1 - pos++ - k1++ - if k1 >= len(set1) { - copy(buffer[pos:], set2[k2:]) - pos += len(set2) - k2 - break - } - s1 = set1[k1] - } else if s1 == s2 { - buffer[pos] = s1 - pos++ - k1++ - k2++ - if k1 >= len(set1) { - copy(buffer[pos:], set2[k2:]) - pos += len(set2) - k2 - break - } - if k2 >= len(set2) { - copy(buffer[pos:], set1[k1:]) - pos += len(set1) - k1 - break - } - s1 = set1[k1] - s2 = set2[k2] - } else { // if (set1[k1]>set2[k2]) - buffer[pos] = s2 - pos++ - k2++ - if k2 >= len(set2) { - copy(buffer[pos:], set1[k1:]) - pos += len(set1) - k1 - break - } - s2 = set2[k2] - } - } - return pos -} diff --git a/vendor/github.com/RoaringBitmap/roaring/shortiterator.go b/vendor/github.com/RoaringBitmap/roaring/shortiterator.go deleted file mode 100644 index 15b78bd0c1..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/shortiterator.go +++ /dev/null @@ -1,52 +0,0 @@ -package roaring - -type shortIterable interface { - hasNext() bool - next() uint16 -} - -type shortPeekable interface { - shortIterable - peekNext() uint16 - advanceIfNeeded(minval uint16) -} - -type shortIterator struct { - slice []uint16 - loc int -} - -func (si *shortIterator) hasNext() bool { - return si.loc < len(si.slice) -} - -func (si *shortIterator) next() uint16 { - a := si.slice[si.loc] - si.loc++ - return a -} - -func (si *shortIterator) peekNext() uint16 { - return si.slice[si.loc] -} - -func (si *shortIterator) advanceIfNeeded(minval uint16) { - if si.hasNext() && si.peekNext() < minval { - si.loc = advanceUntil(si.slice, si.loc, len(si.slice), minval) - } -} - -type reverseIterator struct { - slice []uint16 - loc int -} - -func (si *reverseIterator) hasNext() bool { - return si.loc >= 0 -} - -func (si *reverseIterator) next() uint16 { - a := si.slice[si.loc] - si.loc-- - return a -} diff --git a/vendor/github.com/RoaringBitmap/roaring/smat.go b/vendor/github.com/RoaringBitmap/roaring/smat.go deleted file mode 100644 index c52c5f07cf..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/smat.go +++ /dev/null @@ -1,384 +0,0 @@ -//go:build gofuzz -// +build gofuzz - -/* -# Instructions for smat testing for roaring - -[smat](https://github.com/mschoch/smat) is a framework that provides -state machine assisted fuzz testing. - -To run the smat tests for roaring... - -## Prerequisites - - $ go get github.com/dvyukov/go-fuzz/go-fuzz - $ go get github.com/dvyukov/go-fuzz/go-fuzz-build - -## Steps - -1. Generate initial smat corpus: -``` - go test -tags=gofuzz -run=TestGenerateSmatCorpus -``` - -2. Build go-fuzz test program with instrumentation: -``` - go-fuzz-build -func FuzzSmat github.com/RoaringBitmap/roaring -``` - -3. Run go-fuzz: -``` - go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 -``` - -You should see output like... -``` -2016/09/16 13:58:35 slaves: 8, corpus: 1 (3s ago), crashers: 0, restarts: 1/0, execs: 0 (0/sec), cover: 0, uptime: 3s -2016/09/16 13:58:38 slaves: 8, corpus: 1 (6s ago), crashers: 0, restarts: 1/0, execs: 0 (0/sec), cover: 0, uptime: 6s -2016/09/16 13:58:41 slaves: 8, corpus: 1 (9s ago), crashers: 0, restarts: 1/44, execs: 44 (5/sec), cover: 0, uptime: 9s -2016/09/16 13:58:44 slaves: 8, corpus: 1 (12s ago), crashers: 0, restarts: 1/45, execs: 45 (4/sec), cover: 0, uptime: 12s -2016/09/16 13:58:47 slaves: 8, corpus: 1 (15s ago), crashers: 0, restarts: 1/46, execs: 46 (3/sec), cover: 0, uptime: 15s -2016/09/16 13:58:50 slaves: 8, corpus: 1 (18s ago), crashers: 0, restarts: 1/47, execs: 47 (3/sec), cover: 0, uptime: 18s -2016/09/16 13:58:53 slaves: 8, corpus: 1 (21s ago), crashers: 0, restarts: 1/63, execs: 63 (3/sec), cover: 0, uptime: 21s -2016/09/16 13:58:56 slaves: 8, corpus: 1 (24s ago), crashers: 0, restarts: 1/65, execs: 65 (3/sec), cover: 0, uptime: 24s -2016/09/16 13:58:59 slaves: 8, corpus: 1 (27s ago), crashers: 0, restarts: 1/66, execs: 66 (2/sec), cover: 0, uptime: 27s -2016/09/16 13:59:02 slaves: 8, corpus: 1 (30s ago), crashers: 0, restarts: 1/67, execs: 67 (2/sec), cover: 0, uptime: 30s -2016/09/16 13:59:05 slaves: 8, corpus: 1 (33s ago), crashers: 0, restarts: 1/83, execs: 83 (3/sec), cover: 0, uptime: 33s -2016/09/16 13:59:08 slaves: 8, corpus: 1 (36s ago), crashers: 0, restarts: 1/84, execs: 84 (2/sec), cover: 0, uptime: 36s -2016/09/16 13:59:11 slaves: 8, corpus: 2 (0s ago), crashers: 0, restarts: 1/85, execs: 85 (2/sec), cover: 0, uptime: 39s -2016/09/16 13:59:14 slaves: 8, corpus: 17 (2s ago), crashers: 0, restarts: 1/86, execs: 86 (2/sec), cover: 480, uptime: 42s -2016/09/16 13:59:17 slaves: 8, corpus: 17 (5s ago), crashers: 0, restarts: 1/66, execs: 132 (3/sec), cover: 487, uptime: 45s -2016/09/16 13:59:20 slaves: 8, corpus: 17 (8s ago), crashers: 0, restarts: 1/440, execs: 2645 (55/sec), cover: 487, uptime: 48s - -``` - -Let it run, and if the # of crashers is > 0, check out the reports in -the workdir where you should be able to find the panic goroutine stack -traces. -*/ - -package roaring - -import ( - "fmt" - "sort" - - "github.com/bits-and-blooms/bitset" - "github.com/mschoch/smat" -) - -// fuzz test using state machine driven by byte stream. -func FuzzSmat(data []byte) int { - return smat.Fuzz(&smatContext{}, smat.ActionID('S'), smat.ActionID('T'), - smatActionMap, data) -} - -var smatDebug = false - -func smatLog(prefix, format string, args ...interface{}) { - if smatDebug { - fmt.Print(prefix) - fmt.Printf(format, args...) - } -} - -type smatContext struct { - pairs []*smatPair - - // Two registers, x & y. - x int - y int - - actions int -} - -type smatPair struct { - bm *Bitmap - bs *bitset.BitSet -} - -// ------------------------------------------------------------------ - -var smatActionMap = smat.ActionMap{ - smat.ActionID('X'): smatAction("x++", smatWrap(func(c *smatContext) { c.x++ })), - smat.ActionID('x'): smatAction("x--", smatWrap(func(c *smatContext) { c.x-- })), - smat.ActionID('Y'): smatAction("y++", smatWrap(func(c *smatContext) { c.y++ })), - smat.ActionID('y'): smatAction("y--", smatWrap(func(c *smatContext) { c.y-- })), - smat.ActionID('*'): smatAction("x*y", smatWrap(func(c *smatContext) { c.x = c.x * c.y })), - smat.ActionID('<'): smatAction("x<<", smatWrap(func(c *smatContext) { c.x = c.x << 1 })), - - smat.ActionID('^'): smatAction("swap", smatWrap(func(c *smatContext) { c.x, c.y = c.y, c.x })), - - smat.ActionID('['): smatAction(" pushPair", smatWrap(smatPushPair)), - smat.ActionID(']'): smatAction(" popPair", smatWrap(smatPopPair)), - - smat.ActionID('B'): smatAction(" setBit", smatWrap(smatSetBit)), - smat.ActionID('b'): smatAction(" removeBit", smatWrap(smatRemoveBit)), - - smat.ActionID('o'): smatAction(" or", smatWrap(smatOr)), - smat.ActionID('a'): smatAction(" and", smatWrap(smatAnd)), - - smat.ActionID('#'): smatAction(" cardinality", smatWrap(smatCardinality)), - - smat.ActionID('O'): smatAction(" orCardinality", smatWrap(smatOrCardinality)), - smat.ActionID('A'): smatAction(" andCardinality", smatWrap(smatAndCardinality)), - - smat.ActionID('c'): smatAction(" clear", smatWrap(smatClear)), - smat.ActionID('r'): smatAction(" runOptimize", smatWrap(smatRunOptimize)), - - smat.ActionID('e'): smatAction(" isEmpty", smatWrap(smatIsEmpty)), - - smat.ActionID('i'): smatAction(" intersects", smatWrap(smatIntersects)), - - smat.ActionID('f'): smatAction(" flip", smatWrap(smatFlip)), - - smat.ActionID('-'): smatAction(" difference", smatWrap(smatDifference)), -} - -var smatRunningPercentActions []smat.PercentAction - -func init() { - var ids []int - for actionId := range smatActionMap { - ids = append(ids, int(actionId)) - } - sort.Ints(ids) - - pct := 100 / len(smatActionMap) - for _, actionId := range ids { - smatRunningPercentActions = append(smatRunningPercentActions, - smat.PercentAction{pct, smat.ActionID(actionId)}) - } - - smatActionMap[smat.ActionID('S')] = smatAction("SETUP", smatSetupFunc) - smatActionMap[smat.ActionID('T')] = smatAction("TEARDOWN", smatTeardownFunc) -} - -// We only have one smat state: running. -func smatRunning(next byte) smat.ActionID { - return smat.PercentExecute(next, smatRunningPercentActions...) -} - -func smatAction(name string, f func(ctx smat.Context) (smat.State, error)) func(smat.Context) (smat.State, error) { - return func(ctx smat.Context) (smat.State, error) { - c := ctx.(*smatContext) - c.actions++ - - smatLog(" ", "%s\n", name) - - return f(ctx) - } -} - -// Creates an smat action func based on a simple callback. -func smatWrap(cb func(c *smatContext)) func(smat.Context) (next smat.State, err error) { - return func(ctx smat.Context) (next smat.State, err error) { - c := ctx.(*smatContext) - cb(c) - return smatRunning, nil - } -} - -// Invokes a callback function with the input v bounded to len(c.pairs). -func (c *smatContext) withPair(v int, cb func(*smatPair)) { - if len(c.pairs) > 0 { - if v < 0 { - v = -v - } - v = v % len(c.pairs) - cb(c.pairs[v]) - } -} - -// ------------------------------------------------------------------ - -func smatSetupFunc(ctx smat.Context) (next smat.State, err error) { - return smatRunning, nil -} - -func smatTeardownFunc(ctx smat.Context) (next smat.State, err error) { - return nil, err -} - -// ------------------------------------------------------------------ - -func smatPushPair(c *smatContext) { - c.pairs = append(c.pairs, &smatPair{ - bm: NewBitmap(), - bs: bitset.New(100), - }) -} - -func smatPopPair(c *smatContext) { - if len(c.pairs) > 0 { - c.pairs = c.pairs[0 : len(c.pairs)-1] - } -} - -func smatSetBit(c *smatContext) { - c.withPair(c.x, func(p *smatPair) { - y := uint32(c.y) - p.bm.AddInt(int(y)) - p.bs.Set(uint(y)) - p.checkEquals() - }) -} - -func smatRemoveBit(c *smatContext) { - c.withPair(c.x, func(p *smatPair) { - y := uint32(c.y) - p.bm.Remove(y) - p.bs.Clear(uint(y)) - p.checkEquals() - }) -} - -func smatAnd(c *smatContext) { - c.withPair(c.x, func(px *smatPair) { - c.withPair(c.y, func(py *smatPair) { - px.bm.And(py.bm) - px.bs = px.bs.Intersection(py.bs) - px.checkEquals() - py.checkEquals() - }) - }) -} - -func smatOr(c *smatContext) { - c.withPair(c.x, func(px *smatPair) { - c.withPair(c.y, func(py *smatPair) { - px.bm.Or(py.bm) - px.bs = px.bs.Union(py.bs) - px.checkEquals() - py.checkEquals() - }) - }) -} - -func smatAndCardinality(c *smatContext) { - c.withPair(c.x, func(px *smatPair) { - c.withPair(c.y, func(py *smatPair) { - c0 := px.bm.AndCardinality(py.bm) - c1 := px.bs.IntersectionCardinality(py.bs) - if c0 != uint64(c1) { - panic("expected same add cardinality") - } - px.checkEquals() - py.checkEquals() - }) - }) -} - -func smatOrCardinality(c *smatContext) { - c.withPair(c.x, func(px *smatPair) { - c.withPair(c.y, func(py *smatPair) { - c0 := px.bm.OrCardinality(py.bm) - c1 := px.bs.UnionCardinality(py.bs) - if c0 != uint64(c1) { - panic("expected same or cardinality") - } - px.checkEquals() - py.checkEquals() - }) - }) -} - -func smatRunOptimize(c *smatContext) { - c.withPair(c.x, func(px *smatPair) { - px.bm.RunOptimize() - px.checkEquals() - }) -} - -func smatClear(c *smatContext) { - c.withPair(c.x, func(px *smatPair) { - px.bm.Clear() - px.bs = px.bs.ClearAll() - px.checkEquals() - }) -} - -func smatCardinality(c *smatContext) { - c.withPair(c.x, func(px *smatPair) { - c0 := px.bm.GetCardinality() - c1 := px.bs.Count() - if c0 != uint64(c1) { - panic("expected same cardinality") - } - }) -} - -func smatIsEmpty(c *smatContext) { - c.withPair(c.x, func(px *smatPair) { - c0 := px.bm.IsEmpty() - c1 := px.bs.None() - if c0 != c1 { - panic("expected same is empty") - } - }) -} - -func smatIntersects(c *smatContext) { - c.withPair(c.x, func(px *smatPair) { - c.withPair(c.y, func(py *smatPair) { - v0 := px.bm.Intersects(py.bm) - v1 := px.bs.IntersectionCardinality(py.bs) > 0 - if v0 != v1 { - panic("intersects not equal") - } - - px.checkEquals() - py.checkEquals() - }) - }) -} - -func smatFlip(c *smatContext) { - c.withPair(c.x, func(p *smatPair) { - y := uint32(c.y) - p.bm.Flip(uint64(y), uint64(y)+1) - p.bs = p.bs.Flip(uint(y)) - p.checkEquals() - }) -} - -func smatDifference(c *smatContext) { - c.withPair(c.x, func(px *smatPair) { - c.withPair(c.y, func(py *smatPair) { - px.bm.AndNot(py.bm) - px.bs = px.bs.Difference(py.bs) - px.checkEquals() - py.checkEquals() - }) - }) -} - -func (p *smatPair) checkEquals() { - if !p.equalsBitSet(p.bs, p.bm) { - panic("bitset mismatch") - } -} - -func (p *smatPair) equalsBitSet(a *bitset.BitSet, b *Bitmap) bool { - for i, e := a.NextSet(0); e; i, e = a.NextSet(i + 1) { - if !b.ContainsInt(int(i)) { - fmt.Printf("in a bitset, not b bitmap, i: %d\n", i) - fmt.Printf(" a bitset: %s\n b bitmap: %s\n", - a.String(), b.String()) - return false - } - } - - i := b.Iterator() - for i.HasNext() { - v := i.Next() - if !a.Test(uint(v)) { - fmt.Printf("in b bitmap, not a bitset, v: %d\n", v) - fmt.Printf(" a bitset: %s\n b bitmap: %s\n", - a.String(), b.String()) - return false - } - } - - return true -} diff --git a/vendor/github.com/RoaringBitmap/roaring/util.go b/vendor/github.com/RoaringBitmap/roaring/util.go deleted file mode 100644 index 48b9d5a10e..0000000000 --- a/vendor/github.com/RoaringBitmap/roaring/util.go +++ /dev/null @@ -1,305 +0,0 @@ -package roaring - -import ( - "math" - "math/rand" - "sort" -) - -const ( - arrayDefaultMaxSize = 4096 // containers with 4096 or fewer integers should be array containers. - arrayLazyLowerBound = 1024 - maxCapacity = 1 << 16 - serialCookieNoRunContainer = 12346 // only arrays and bitmaps - invalidCardinality = -1 - serialCookie = 12347 // runs, arrays, and bitmaps - noOffsetThreshold = 4 - - // MaxUint32 is the largest uint32 value. - MaxUint32 = math.MaxUint32 - - // MaxRange is One more than the maximum allowed bitmap bit index. For use as an upper - // bound for ranges. - MaxRange uint64 = MaxUint32 + 1 - - // MaxUint16 is the largest 16 bit unsigned int. - // This is the largest value an interval16 can store. - MaxUint16 = math.MaxUint16 - - // Compute wordSizeInBytes, the size of a word in bytes. - _m = ^uint64(0) - _logS = _m>>8&1 + _m>>16&1 + _m>>32&1 - wordSizeInBytes = 1 << _logS - - // other constants used in ctz_generic.go - wordSizeInBits = wordSizeInBytes << 3 // word size in bits -) - -const maxWord = 1< arrayDefaultMaxSize { - // bitmapContainer - return maxCapacity / 8 - } - // arrayContainer - return 2 * card -} - -func fill(arr []uint64, val uint64) { - for i := range arr { - arr[i] = val - } -} -func fillRange(arr []uint64, start, end int, val uint64) { - for i := start; i < end; i++ { - arr[i] = val - } -} - -func fillArrayAND(container []uint16, bitmap1, bitmap2 []uint64) { - if len(bitmap1) != len(bitmap2) { - panic("array lengths don't match") - } - // TODO: rewrite in assembly - pos := 0 - for k := range bitmap1 { - bitset := bitmap1[k] & bitmap2[k] - for bitset != 0 { - t := bitset & -bitset - container[pos] = uint16((k*64 + int(popcount(t-1)))) - pos = pos + 1 - bitset ^= t - } - } -} - -func fillArrayANDNOT(container []uint16, bitmap1, bitmap2 []uint64) { - if len(bitmap1) != len(bitmap2) { - panic("array lengths don't match") - } - // TODO: rewrite in assembly - pos := 0 - for k := range bitmap1 { - bitset := bitmap1[k] &^ bitmap2[k] - for bitset != 0 { - t := bitset & -bitset - container[pos] = uint16((k*64 + int(popcount(t-1)))) - pos = pos + 1 - bitset ^= t - } - } -} - -func fillArrayXOR(container []uint16, bitmap1, bitmap2 []uint64) { - if len(bitmap1) != len(bitmap2) { - panic("array lengths don't match") - } - // TODO: rewrite in assembly - pos := 0 - for k := 0; k < len(bitmap1); k++ { - bitset := bitmap1[k] ^ bitmap2[k] - for bitset != 0 { - t := bitset & -bitset - container[pos] = uint16((k*64 + int(popcount(t-1)))) - pos = pos + 1 - bitset ^= t - } - } -} - -func highbits(x uint32) uint16 { - return uint16(x >> 16) -} -func lowbits(x uint32) uint16 { - return uint16(x & maxLowBit) -} - -const maxLowBit = 0xFFFF - -func flipBitmapRange(bitmap []uint64, start int, end int) { - if start >= end { - return - } - firstword := start / 64 - endword := (end - 1) / 64 - bitmap[firstword] ^= ^(^uint64(0) << uint(start%64)) - for i := firstword; i < endword; i++ { - bitmap[i] = ^bitmap[i] - } - bitmap[endword] ^= ^uint64(0) >> (uint(-end) % 64) -} - -func resetBitmapRange(bitmap []uint64, start int, end int) { - if start >= end { - return - } - firstword := start / 64 - endword := (end - 1) / 64 - if firstword == endword { - bitmap[firstword] &= ^((^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64))) - return - } - bitmap[firstword] &= ^(^uint64(0) << uint(start%64)) - for i := firstword + 1; i < endword; i++ { - bitmap[i] = 0 - } - bitmap[endword] &= ^(^uint64(0) >> (uint(-end) % 64)) - -} - -func setBitmapRange(bitmap []uint64, start int, end int) { - if start >= end { - return - } - firstword := start / 64 - endword := (end - 1) / 64 - if firstword == endword { - bitmap[firstword] |= (^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64)) - return - } - bitmap[firstword] |= ^uint64(0) << uint(start%64) - for i := firstword + 1; i < endword; i++ { - bitmap[i] = ^uint64(0) - } - bitmap[endword] |= ^uint64(0) >> (uint(-end) % 64) -} - -func flipBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int { - before := wordCardinalityForBitmapRange(bitmap, start, end) - flipBitmapRange(bitmap, start, end) - after := wordCardinalityForBitmapRange(bitmap, start, end) - return int(after - before) -} - -func resetBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int { - before := wordCardinalityForBitmapRange(bitmap, start, end) - resetBitmapRange(bitmap, start, end) - after := wordCardinalityForBitmapRange(bitmap, start, end) - return int(after - before) -} - -func setBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int { - before := wordCardinalityForBitmapRange(bitmap, start, end) - setBitmapRange(bitmap, start, end) - after := wordCardinalityForBitmapRange(bitmap, start, end) - return int(after - before) -} - -func wordCardinalityForBitmapRange(bitmap []uint64, start int, end int) uint64 { - answer := uint64(0) - if start >= end { - return answer - } - firstword := start / 64 - endword := (end - 1) / 64 - for i := firstword; i <= endword; i++ { - answer += popcount(bitmap[i]) - } - return answer -} - -func selectBitPosition(w uint64, j int) int { - seen := 0 - - // Divide 64bit - part := w & 0xFFFFFFFF - n := popcount(part) - if n <= uint64(j) { - part = w >> 32 - seen += 32 - j -= int(n) - } - w = part - - // Divide 32bit - part = w & 0xFFFF - n = popcount(part) - if n <= uint64(j) { - part = w >> 16 - seen += 16 - j -= int(n) - } - w = part - - // Divide 16bit - part = w & 0xFF - n = popcount(part) - if n <= uint64(j) { - part = w >> 8 - seen += 8 - j -= int(n) - } - w = part - - // Lookup in final byte - var counter uint - for counter = 0; counter < 8; counter++ { - j -= int((w >> counter) & 1) - if j < 0 { - break - } - } - return seen + int(counter) - -} - -func panicOn(err error) { - if err != nil { - panic(err) - } -} - -type ph struct { - orig int - rand int -} - -type pha []ph - -func (p pha) Len() int { return len(p) } -func (p pha) Less(i, j int) bool { return p[i].rand < p[j].rand } -func (p pha) Swap(i, j int) { p[i], p[j] = p[j], p[i] } - -func getRandomPermutation(n int) []int { - r := make([]ph, n) - for i := 0; i < n; i++ { - r[i].orig = i - r[i].rand = rand.Intn(1 << 29) - } - sort.Sort(pha(r)) - m := make([]int, n) - for i := range m { - m[i] = r[i].orig - } - return m -} - -func minOfInt(a, b int) int { - if a < b { - return a - } - return b -} - -func maxOfInt(a, b int) int { - if a > b { - return a - } - return b -} - -func maxOfUint16(a, b uint16) uint16 { - if a > b { - return a - } - return b -} - -func minOfUint16(a, b uint16) uint16 { - if a < b { - return a - } - return b -} diff --git a/vendor/github.com/bits-and-blooms/bitset/.gitignore b/vendor/github.com/bits-and-blooms/bitset/.gitignore deleted file mode 100644 index 5c204d28b0..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/.gitignore +++ /dev/null @@ -1,26 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -*.test -*.prof - -target diff --git a/vendor/github.com/bits-and-blooms/bitset/.travis.yml b/vendor/github.com/bits-and-blooms/bitset/.travis.yml deleted file mode 100644 index 094aa5ce07..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/.travis.yml +++ /dev/null @@ -1,37 +0,0 @@ -language: go - -sudo: false - -branches: - except: - - release - -branches: - only: - - master - - travis - -go: - - "1.11.x" - - tip - -matrix: - allow_failures: - - go: tip - -before_install: - - if [ -n "$GH_USER" ]; then git config --global github.user ${GH_USER}; fi; - - if [ -n "$GH_TOKEN" ]; then git config --global github.token ${GH_TOKEN}; fi; - - go get github.com/mattn/goveralls - -before_script: - - make deps - -script: - - make qa - -after_failure: - - cat ./target/test/report.xml - -after_success: - - if [ "$TRAVIS_GO_VERSION" = "1.11.1" ]; then $HOME/gopath/bin/goveralls -covermode=count -coverprofile=target/report/coverage.out -service=travis-ci; fi; diff --git a/vendor/github.com/bits-and-blooms/bitset/LICENSE b/vendor/github.com/bits-and-blooms/bitset/LICENSE deleted file mode 100644 index 59cab8a939..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2014 Will Fitzgerald. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/bits-and-blooms/bitset/README.md b/vendor/github.com/bits-and-blooms/bitset/README.md deleted file mode 100644 index 848234e2fc..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/README.md +++ /dev/null @@ -1,152 +0,0 @@ -# bitset - -*Go language library to map between non-negative integers and boolean values* - -[![Test](https://github.com/bits-and-blooms/bitset/workflows/Test/badge.svg)](https://github.com/willf/bitset/actions?query=workflow%3ATest) -[![Go Report Card](https://goreportcard.com/badge/github.com/willf/bitset)](https://goreportcard.com/report/github.com/willf/bitset) -[![PkgGoDev](https://pkg.go.dev/badge/github.com/bits-and-blooms/bitset?tab=doc)](https://pkg.go.dev/github.com/bits-and-blooms/bitset?tab=doc) - - -This library is part of the [awesome go collection](https://github.com/avelino/awesome-go). It is used in production by several important systems: - -* [beego](https://github.com/beego/beego) -* [CubeFS](https://github.com/cubefs/cubefs) -* [Amazon EKS Distro](https://github.com/aws/eks-distro) -* [sourcegraph](https://github.com/sourcegraph/sourcegraph) -* [torrent](https://github.com/anacrolix/torrent) - - -## Description - -Package bitset implements bitsets, a mapping between non-negative integers and boolean values. -It should be more efficient than map[uint] bool. - -It provides methods for setting, clearing, flipping, and testing individual integers. - -But it also provides set intersection, union, difference, complement, and symmetric operations, as well as tests to check whether any, all, or no bits are set, and querying a bitset's current length and number of positive bits. - -BitSets are expanded to the size of the largest set bit; the memory allocation is approximately Max bits, where Max is the largest set bit. BitSets are never shrunk. On creation, a hint can be given for the number of bits that will be used. - -Many of the methods, including Set, Clear, and Flip, return a BitSet pointer, which allows for chaining. - -### Example use: - -```go -package main - -import ( - "fmt" - "math/rand" - - "github.com/bits-and-blooms/bitset" -) - -func main() { - fmt.Printf("Hello from BitSet!\n") - var b bitset.BitSet - // play some Go Fish - for i := 0; i < 100; i++ { - card1 := uint(rand.Intn(52)) - card2 := uint(rand.Intn(52)) - b.Set(card1) - if b.Test(card2) { - fmt.Println("Go Fish!") - } - b.Clear(card1) - } - - // Chaining - b.Set(10).Set(11) - - for i, e := b.NextSet(0); e; i, e = b.NextSet(i + 1) { - fmt.Println("The following bit is set:", i) - } - if b.Intersection(bitset.New(100).Set(10)).Count() == 1 { - fmt.Println("Intersection works.") - } else { - fmt.Println("Intersection doesn't work???") - } -} -``` - - -Package documentation is at: https://pkg.go.dev/github.com/bits-and-blooms/bitset?tab=doc - -## Serialization - - -You may serialize a bitset safely and portably to a stream -of bytes as follows: -```Go - const length = 9585 - const oneEvery = 97 - bs := bitset.New(length) - // Add some bits - for i := uint(0); i < length; i += oneEvery { - bs = bs.Set(i) - } - - var buf bytes.Buffer - n, err := bs.WriteTo(&buf) - if err != nil { - // failure - } - // Here n == buf.Len() -``` -You can later deserialize the result as follows: - -```Go - // Read back from buf - bs = bitset.New() - n, err = bs.ReadFrom(&buf) - if err != nil { - // error - } - // n is the number of bytes read -``` - -The `ReadFrom` function attempts to read the data into the existing -BitSet instance, to minimize memory allocations. - - -*Performance tip*: -When reading and writing to a file or a network connection, you may get better performance by -wrapping your streams with `bufio` instances. - -E.g., -```Go - f, err := os.Create("myfile") - w := bufio.NewWriter(f) -``` -```Go - f, err := os.Open("myfile") - r := bufio.NewReader(f) -``` - -## Memory Usage - -The memory usage of a bitset using `N` bits is at least `N/8` bytes. The number of bits in a bitset is at least as large as one plus the greatest bit index you have accessed. Thus it is possible to run out of memory while using a bitset. If you have lots of bits, you might prefer compressed bitsets, like the [Roaring bitmaps](http://roaringbitmap.org) and its [Go implementation](https://github.com/RoaringBitmap/roaring). - -## Implementation Note - -Go 1.9 introduced a native `math/bits` library. We provide backward compatibility to Go 1.7, which might be removed. - -It is possible that a later version will match the `math/bits` return signature for counts (which is `int`, rather than our library's `uint64`). If so, the version will be bumped. - -## Installation - -```bash -go get github.com/bits-and-blooms/bitset -``` - -## Contributing - -If you wish to contribute to this project, please branch and issue a pull request against master ("[GitHub Flow](https://guides.github.com/introduction/flow/)") - -## Running all tests - -Before committing the code, please check if it passes tests, has adequate coverage, etc. -```bash -go test -go test -cover -``` diff --git a/vendor/github.com/bits-and-blooms/bitset/SECURITY.md b/vendor/github.com/bits-and-blooms/bitset/SECURITY.md deleted file mode 100644 index f888420c3b..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/SECURITY.md +++ /dev/null @@ -1,5 +0,0 @@ -# Security Policy - -## Reporting a Vulnerability - -You can report privately a vulnerability by email at daniel@lemire.me (current maintainer). diff --git a/vendor/github.com/bits-and-blooms/bitset/azure-pipelines.yml b/vendor/github.com/bits-and-blooms/bitset/azure-pipelines.yml deleted file mode 100644 index f9b2959184..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/azure-pipelines.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Go -# Build your Go project. -# Add steps that test, save build artifacts, deploy, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/go - -trigger: -- master - -pool: - vmImage: 'Ubuntu-16.04' - -variables: - GOBIN: '$(GOPATH)/bin' # Go binaries path - GOROOT: '/usr/local/go1.11' # Go installation path - GOPATH: '$(system.defaultWorkingDirectory)/gopath' # Go workspace path - modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)' # Path to the module's code - -steps: -- script: | - mkdir -p '$(GOBIN)' - mkdir -p '$(GOPATH)/pkg' - mkdir -p '$(modulePath)' - shopt -s extglob - shopt -s dotglob - mv !(gopath) '$(modulePath)' - echo '##vso[task.prependpath]$(GOBIN)' - echo '##vso[task.prependpath]$(GOROOT)/bin' - displayName: 'Set up the Go workspace' - -- script: | - go version - go get -v -t -d ./... - if [ -f Gopkg.toml ]; then - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - dep ensure - fi - go build -v . - workingDirectory: '$(modulePath)' - displayName: 'Get dependencies, then build' diff --git a/vendor/github.com/bits-and-blooms/bitset/bitset.go b/vendor/github.com/bits-and-blooms/bitset/bitset.go deleted file mode 100644 index 8fb9e9fa20..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/bitset.go +++ /dev/null @@ -1,1137 +0,0 @@ -/* -Package bitset implements bitsets, a mapping -between non-negative integers and boolean values. It should be more -efficient than map[uint] bool. - -It provides methods for setting, clearing, flipping, and testing -individual integers. - -But it also provides set intersection, union, difference, -complement, and symmetric operations, as well as tests to -check whether any, all, or no bits are set, and querying a -bitset's current length and number of positive bits. - -BitSets are expanded to the size of the largest set bit; the -memory allocation is approximately Max bits, where Max is -the largest set bit. BitSets are never shrunk. On creation, -a hint can be given for the number of bits that will be used. - -Many of the methods, including Set,Clear, and Flip, return -a BitSet pointer, which allows for chaining. - -Example use: - - import "bitset" - var b BitSet - b.Set(10).Set(11) - if b.Test(1000) { - b.Clear(1000) - } - if B.Intersection(bitset.New(100).Set(10)).Count() > 1 { - fmt.Println("Intersection works.") - } - -As an alternative to BitSets, one should check out the 'big' package, -which provides a (less set-theoretical) view of bitsets. -*/ -package bitset - -import ( - "bytes" - "encoding/base64" - "encoding/binary" - "encoding/json" - "errors" - "fmt" - "io" - "strconv" -) - -// the wordSize of a bit set -const wordSize = uint(64) - -// the wordSize of a bit set in bytes -const wordBytes = wordSize / 8 - -// log2WordSize is lg(wordSize) -const log2WordSize = uint(6) - -// allBits has every bit set -const allBits uint64 = 0xffffffffffffffff - -// default binary BigEndian -var binaryOrder binary.ByteOrder = binary.BigEndian - -// default json encoding base64.URLEncoding -var base64Encoding = base64.URLEncoding - -// Base64StdEncoding Marshal/Unmarshal BitSet with base64.StdEncoding(Default: base64.URLEncoding) -func Base64StdEncoding() { base64Encoding = base64.StdEncoding } - -// LittleEndian Marshal/Unmarshal Binary as Little Endian(Default: binary.BigEndian) -func LittleEndian() { binaryOrder = binary.LittleEndian } - -// A BitSet is a set of bits. The zero value of a BitSet is an empty set of length 0. -type BitSet struct { - length uint - set []uint64 -} - -// Error is used to distinguish errors (panics) generated in this package. -type Error string - -// safeSet will fixup b.set to be non-nil and return the field value -func (b *BitSet) safeSet() []uint64 { - if b.set == nil { - b.set = make([]uint64, wordsNeeded(0)) - } - return b.set -} - -// SetBitsetFrom fills the bitset with an array of integers without creating a new BitSet instance -func (b *BitSet) SetBitsetFrom(buf []uint64) { - b.length = uint(len(buf)) * 64 - b.set = buf -} - -// From is a constructor used to create a BitSet from an array of integers -func From(buf []uint64) *BitSet { - return FromWithLength(uint(len(buf))*64, buf) -} - -// FromWithLength constructs from an array of integers and length. -func FromWithLength(len uint, set []uint64) *BitSet { - return &BitSet{len, set} -} - -// Bytes returns the bitset as array of integers -func (b *BitSet) Bytes() []uint64 { - return b.set -} - -// wordsNeeded calculates the number of words needed for i bits -func wordsNeeded(i uint) int { - if i > (Cap() - wordSize + 1) { - return int(Cap() >> log2WordSize) - } - return int((i + (wordSize - 1)) >> log2WordSize) -} - -// wordsNeededUnbound calculates the number of words needed for i bits, possibly exceeding the capacity. -// This function is useful if you know that the capacity cannot be exceeded (e.g., you have an existing bitmap). -func wordsNeededUnbound(i uint) int { - return int((i + (wordSize - 1)) >> log2WordSize) -} - -// wordsIndex calculates the index of words in a `uint64` -func wordsIndex(i uint) uint { - return i & (wordSize - 1) -} - -// New creates a new BitSet with a hint that length bits will be required -func New(length uint) (bset *BitSet) { - defer func() { - if r := recover(); r != nil { - bset = &BitSet{ - 0, - make([]uint64, 0), - } - } - }() - - bset = &BitSet{ - length, - make([]uint64, wordsNeeded(length)), - } - - return bset -} - -// Cap returns the total possible capacity, or number of bits -func Cap() uint { - return ^uint(0) -} - -// Len returns the number of bits in the BitSet. -// Note the difference to method Count, see example. -func (b *BitSet) Len() uint { - return b.length -} - -// extendSet adds additional words to incorporate new bits if needed -func (b *BitSet) extendSet(i uint) { - if i >= Cap() { - panic("You are exceeding the capacity") - } - nsize := wordsNeeded(i + 1) - if b.set == nil { - b.set = make([]uint64, nsize) - } else if cap(b.set) >= nsize { - b.set = b.set[:nsize] // fast resize - } else if len(b.set) < nsize { - newset := make([]uint64, nsize, 2*nsize) // increase capacity 2x - copy(newset, b.set) - b.set = newset - } - b.length = i + 1 -} - -// Test whether bit i is set. -func (b *BitSet) Test(i uint) bool { - if i >= b.length { - return false - } - return b.set[i>>log2WordSize]&(1<= Cap(), this function will panic. -// Warning: using a very large value for 'i' -// may lead to a memory shortage and a panic: the caller is responsible -// for providing sensible parameters in line with their memory capacity. -func (b *BitSet) Set(i uint) *BitSet { - if i >= b.length { // if we need more bits, make 'em - b.extendSet(i) - } - b.set[i>>log2WordSize] |= 1 << wordsIndex(i) - return b -} - -// Clear bit i to 0 -func (b *BitSet) Clear(i uint) *BitSet { - if i >= b.length { - return b - } - b.set[i>>log2WordSize] &^= 1 << wordsIndex(i) - return b -} - -// SetTo sets bit i to value. -// If i>= Cap(), this function will panic. -// Warning: using a very large value for 'i' -// may lead to a memory shortage and a panic: the caller is responsible -// for providing sensible parameters in line with their memory capacity. -func (b *BitSet) SetTo(i uint, value bool) *BitSet { - if value { - return b.Set(i) - } - return b.Clear(i) -} - -// Flip bit at i. -// If i>= Cap(), this function will panic. -// Warning: using a very large value for 'i' -// may lead to a memory shortage and a panic: the caller is responsible -// for providing sensible parameters in line with their memory capacity. -func (b *BitSet) Flip(i uint) *BitSet { - if i >= b.length { - return b.Set(i) - } - b.set[i>>log2WordSize] ^= 1 << wordsIndex(i) - return b -} - -// FlipRange bit in [start, end). -// If end>= Cap(), this function will panic. -// Warning: using a very large value for 'end' -// may lead to a memory shortage and a panic: the caller is responsible -// for providing sensible parameters in line with their memory capacity. -func (b *BitSet) FlipRange(start, end uint) *BitSet { - if start >= end { - return b - } - if end-1 >= b.length { // if we need more bits, make 'em - b.extendSet(end - 1) - } - var startWord uint = start >> log2WordSize - var endWord uint = end >> log2WordSize - b.set[startWord] ^= ^(^uint64(0) << wordsIndex(start)) - if endWord > 0 { - // bounds check elimination - data := b.set - _ = data[endWord-1] - for i := startWord; i < endWord; i++ { - data[i] = ^data[i] - } - } - if end&(wordSize-1) != 0 { - b.set[endWord] ^= ^uint64(0) >> wordsIndex(-end) - } - return b -} - -// Shrink shrinks BitSet so that the provided value is the last possible -// set value. It clears all bits > the provided index and reduces the size -// and length of the set. -// -// Note that the parameter value is not the new length in bits: it is the -// maximal value that can be stored in the bitset after the function call. -// The new length in bits is the parameter value + 1. Thus it is not possible -// to use this function to set the length to 0, the minimal value of the length -// after this function call is 1. -// -// A new slice is allocated to store the new bits, so you may see an increase in -// memory usage until the GC runs. Normally this should not be a problem, but if you -// have an extremely large BitSet its important to understand that the old BitSet will -// remain in memory until the GC frees it. -func (b *BitSet) Shrink(lastbitindex uint) *BitSet { - length := lastbitindex + 1 - idx := wordsNeeded(length) - if idx > len(b.set) { - return b - } - shrunk := make([]uint64, idx) - copy(shrunk, b.set[:idx]) - b.set = shrunk - b.length = length - lastWordUsedBits := length % 64 - if lastWordUsedBits != 0 { - b.set[idx-1] &= allBits >> uint64(64-wordsIndex(lastWordUsedBits)) - } - return b -} - -// Compact shrinks BitSet to so that we preserve all set bits, while minimizing -// memory usage. Compact calls Shrink. -func (b *BitSet) Compact() *BitSet { - idx := len(b.set) - 1 - for ; idx >= 0 && b.set[idx] == 0; idx-- { - } - newlength := uint((idx + 1) << log2WordSize) - if newlength >= b.length { - return b // nothing to do - } - if newlength > 0 { - return b.Shrink(newlength - 1) - } - // We preserve one word - return b.Shrink(63) -} - -// InsertAt takes an index which indicates where a bit should be -// inserted. Then it shifts all the bits in the set to the left by 1, starting -// from the given index position, and sets the index position to 0. -// -// Depending on the size of your BitSet, and where you are inserting the new entry, -// this method could be extremely slow and in some cases might cause the entire BitSet -// to be recopied. -func (b *BitSet) InsertAt(idx uint) *BitSet { - insertAtElement := idx >> log2WordSize - - // if length of set is a multiple of wordSize we need to allocate more space first - if b.isLenExactMultiple() { - b.set = append(b.set, uint64(0)) - } - - var i uint - for i = uint(len(b.set) - 1); i > insertAtElement; i-- { - // all elements above the position where we want to insert can simply by shifted - b.set[i] <<= 1 - - // we take the most significant bit of the previous element and set it as - // the least significant bit of the current element - b.set[i] |= (b.set[i-1] & 0x8000000000000000) >> 63 - } - - // generate a mask to extract the data that we need to shift left - // within the element where we insert a bit - dataMask := uint64(1)< 0x40000 { - buffer.WriteString("...") - break - } - buffer.WriteString(strconv.FormatInt(int64(i), 10)) - i, e = b.NextSet(i + 1) - if e { - buffer.WriteString(",") - } - } - buffer.WriteString("}") - return buffer.String() -} - -// DeleteAt deletes the bit at the given index position from -// within the bitset -// All the bits residing on the left of the deleted bit get -// shifted right by 1 -// The running time of this operation may potentially be -// relatively slow, O(length) -func (b *BitSet) DeleteAt(i uint) *BitSet { - // the index of the slice element where we'll delete a bit - deleteAtElement := i >> log2WordSize - - // generate a mask for the data that needs to be shifted right - // within that slice element that gets modified - dataMask := ^((uint64(1) << wordsIndex(i)) - 1) - - // extract the data that we'll shift right from the slice element - data := b.set[deleteAtElement] & dataMask - - // set the masked area to 0 while leaving the rest as it is - b.set[deleteAtElement] &= ^dataMask - - // shift the previously extracted data to the right and then - // set it in the previously masked area - b.set[deleteAtElement] |= (data >> 1) & dataMask - - // loop over all the consecutive slice elements to copy each - // lowest bit into the highest position of the previous element, - // then shift the entire content to the right by 1 - for i := int(deleteAtElement) + 1; i < len(b.set); i++ { - b.set[i-1] |= (b.set[i] & 1) << 63 - b.set[i] >>= 1 - } - - b.length = b.length - 1 - - return b -} - -// NextSet returns the next bit set from the specified index, -// including possibly the current index -// along with an error code (true = valid, false = no set bit found) -// for i,e := v.NextSet(0); e; i,e = v.NextSet(i + 1) {...} -// -// Users concerned with performance may want to use NextSetMany to -// retrieve several values at once. -func (b *BitSet) NextSet(i uint) (uint, bool) { - x := int(i >> log2WordSize) - if x >= len(b.set) { - return 0, false - } - w := b.set[x] - w = w >> wordsIndex(i) - if w != 0 { - return i + trailingZeroes64(w), true - } - x++ - // bounds check elimination in the loop - if x < 0 { - return 0, false - } - for x < len(b.set) { - if b.set[x] != 0 { - return uint(x)*wordSize + trailingZeroes64(b.set[x]), true - } - x++ - - } - return 0, false -} - -// NextSetMany returns many next bit sets from the specified index, -// including possibly the current index and up to cap(buffer). -// If the returned slice has len zero, then no more set bits were found -// -// buffer := make([]uint, 256) // this should be reused -// j := uint(0) -// j, buffer = bitmap.NextSetMany(j, buffer) -// for ; len(buffer) > 0; j, buffer = bitmap.NextSetMany(j,buffer) { -// for k := range buffer { -// do something with buffer[k] -// } -// j += 1 -// } -// -// It is possible to retrieve all set bits as follow: -// -// indices := make([]uint, bitmap.Count()) -// bitmap.NextSetMany(0, indices) -// -// However if bitmap.Count() is large, it might be preferable to -// use several calls to NextSetMany, for performance reasons. -func (b *BitSet) NextSetMany(i uint, buffer []uint) (uint, []uint) { - myanswer := buffer - capacity := cap(buffer) - x := int(i >> log2WordSize) - if x >= len(b.set) || capacity == 0 { - return 0, myanswer[:0] - } - skip := wordsIndex(i) - word := b.set[x] >> skip - myanswer = myanswer[:capacity] - size := int(0) - for word != 0 { - r := trailingZeroes64(word) - t := word & ((^word) + 1) - myanswer[size] = r + i - size++ - if size == capacity { - goto End - } - word = word ^ t - } - x++ - for idx, word := range b.set[x:] { - for word != 0 { - r := trailingZeroes64(word) - t := word & ((^word) + 1) - myanswer[size] = r + (uint(x+idx) << 6) - size++ - if size == capacity { - goto End - } - word = word ^ t - } - } -End: - if size > 0 { - return myanswer[size-1], myanswer[:size] - } - return 0, myanswer[:0] -} - -// NextClear returns the next clear bit from the specified index, -// including possibly the current index -// along with an error code (true = valid, false = no bit found i.e. all bits are set) -func (b *BitSet) NextClear(i uint) (uint, bool) { - x := int(i >> log2WordSize) - if x >= len(b.set) { - return 0, false - } - w := b.set[x] - w = w >> wordsIndex(i) - wA := allBits >> wordsIndex(i) - index := i + trailingZeroes64(^w) - if w != wA && index < b.length { - return index, true - } - x++ - // bounds check elimination in the loop - if x < 0 { - return 0, false - } - for x < len(b.set) { - if b.set[x] != allBits { - index = uint(x)*wordSize + trailingZeroes64(^b.set[x]) - if index < b.length { - return index, true - } - } - x++ - } - return 0, false -} - -// ClearAll clears the entire BitSet -func (b *BitSet) ClearAll() *BitSet { - if b != nil && b.set != nil { - for i := range b.set { - b.set[i] = 0 - } - } - return b -} - -// wordCount returns the number of words used in a bit set -func (b *BitSet) wordCount() int { - return wordsNeededUnbound(b.length) -} - -// Clone this BitSet -func (b *BitSet) Clone() *BitSet { - c := New(b.length) - if b.set != nil { // Clone should not modify current object - copy(c.set, b.set) - } - return c -} - -// Copy into a destination BitSet using the Go array copy semantics: -// the number of bits copied is the minimum of the number of bits in the current -// BitSet (Len()) and the destination Bitset. -// We return the number of bits copied in the destination BitSet. -func (b *BitSet) Copy(c *BitSet) (count uint) { - if c == nil { - return - } - if b.set != nil { // Copy should not modify current object - copy(c.set, b.set) - } - count = c.length - if b.length < c.length { - count = b.length - } - // Cleaning the last word is needed to keep the invariant that other functions, such as Count, require - // that any bits in the last word that would exceed the length of the bitmask are set to 0. - c.cleanLastWord() - return -} - -// CopyFull copies into a destination BitSet such that the destination is -// identical to the source after the operation, allocating memory if necessary. -func (b *BitSet) CopyFull(c *BitSet) { - if c == nil { - return - } - c.length = b.length - if len(b.set) == 0 { - if c.set != nil { - c.set = c.set[:0] - } - } else { - if cap(c.set) < len(b.set) { - c.set = make([]uint64, len(b.set)) - } else { - c.set = c.set[:len(b.set)] - } - copy(c.set, b.set) - } -} - -// Count (number of set bits). -// Also known as "popcount" or "population count". -func (b *BitSet) Count() uint { - if b != nil && b.set != nil { - return uint(popcntSlice(b.set)) - } - return 0 -} - -// Equal tests the equivalence of two BitSets. -// False if they are of different sizes, otherwise true -// only if all the same bits are set -func (b *BitSet) Equal(c *BitSet) bool { - if c == nil || b == nil { - return c == b - } - if b.length != c.length { - return false - } - if b.length == 0 { // if they have both length == 0, then could have nil set - return true - } - wn := b.wordCount() - // bounds check elimination - if wn <= 0 { - return true - } - _ = b.set[wn-1] - _ = c.set[wn-1] - for p := 0; p < wn; p++ { - if c.set[p] != b.set[p] { - return false - } - } - return true -} - -func panicIfNull(b *BitSet) { - if b == nil { - panic(Error("BitSet must not be null")) - } -} - -// Difference of base set and other set -// This is the BitSet equivalent of &^ (and not) -func (b *BitSet) Difference(compare *BitSet) (result *BitSet) { - panicIfNull(b) - panicIfNull(compare) - result = b.Clone() // clone b (in case b is bigger than compare) - l := compare.wordCount() - if l > b.wordCount() { - l = b.wordCount() - } - for i := 0; i < l; i++ { - result.set[i] = b.set[i] &^ compare.set[i] - } - return -} - -// DifferenceCardinality computes the cardinality of the differnce -func (b *BitSet) DifferenceCardinality(compare *BitSet) uint { - panicIfNull(b) - panicIfNull(compare) - l := compare.wordCount() - if l > b.wordCount() { - l = b.wordCount() - } - cnt := uint64(0) - cnt += popcntMaskSlice(b.set[:l], compare.set[:l]) - cnt += popcntSlice(b.set[l:]) - return uint(cnt) -} - -// InPlaceDifference computes the difference of base set and other set -// This is the BitSet equivalent of &^ (and not) -func (b *BitSet) InPlaceDifference(compare *BitSet) { - panicIfNull(b) - panicIfNull(compare) - l := compare.wordCount() - if l > b.wordCount() { - l = b.wordCount() - } - if l <= 0 { - return - } - // bounds check elimination - data, cmpData := b.set, compare.set - _ = data[l-1] - _ = cmpData[l-1] - for i := 0; i < l; i++ { - data[i] &^= cmpData[i] - } -} - -// Convenience function: return two bitsets ordered by -// increasing length. Note: neither can be nil -func sortByLength(a *BitSet, b *BitSet) (ap *BitSet, bp *BitSet) { - if a.length <= b.length { - ap, bp = a, b - } else { - ap, bp = b, a - } - return -} - -// Intersection of base set and other set -// This is the BitSet equivalent of & (and) -func (b *BitSet) Intersection(compare *BitSet) (result *BitSet) { - panicIfNull(b) - panicIfNull(compare) - b, compare = sortByLength(b, compare) - result = New(b.length) - for i, word := range b.set { - result.set[i] = word & compare.set[i] - } - return -} - -// IntersectionCardinality computes the cardinality of the union -func (b *BitSet) IntersectionCardinality(compare *BitSet) uint { - panicIfNull(b) - panicIfNull(compare) - b, compare = sortByLength(b, compare) - cnt := popcntAndSlice(b.set, compare.set) - return uint(cnt) -} - -// InPlaceIntersection destructively computes the intersection of -// base set and the compare set. -// This is the BitSet equivalent of & (and) -func (b *BitSet) InPlaceIntersection(compare *BitSet) { - panicIfNull(b) - panicIfNull(compare) - l := compare.wordCount() - if l > b.wordCount() { - l = b.wordCount() - } - if l > 0 { - // bounds check elimination - data, cmpData := b.set, compare.set - _ = data[l-1] - _ = cmpData[l-1] - - for i := 0; i < l; i++ { - data[i] &= cmpData[i] - } - } - if l >= 0 { - for i := l; i < len(b.set); i++ { - b.set[i] = 0 - } - } - if compare.length > 0 { - if compare.length-1 >= b.length { - b.extendSet(compare.length - 1) - } - } -} - -// Union of base set and other set -// This is the BitSet equivalent of | (or) -func (b *BitSet) Union(compare *BitSet) (result *BitSet) { - panicIfNull(b) - panicIfNull(compare) - b, compare = sortByLength(b, compare) - result = compare.Clone() - for i, word := range b.set { - result.set[i] = word | compare.set[i] - } - return -} - -// UnionCardinality computes the cardinality of the uniton of the base set -// and the compare set. -func (b *BitSet) UnionCardinality(compare *BitSet) uint { - panicIfNull(b) - panicIfNull(compare) - b, compare = sortByLength(b, compare) - cnt := popcntOrSlice(b.set, compare.set) - if len(compare.set) > len(b.set) { - cnt += popcntSlice(compare.set[len(b.set):]) - } - return uint(cnt) -} - -// InPlaceUnion creates the destructive union of base set and compare set. -// This is the BitSet equivalent of | (or). -func (b *BitSet) InPlaceUnion(compare *BitSet) { - panicIfNull(b) - panicIfNull(compare) - l := compare.wordCount() - if l > b.wordCount() { - l = b.wordCount() - } - if compare.length > 0 && compare.length-1 >= b.length { - b.extendSet(compare.length - 1) - } - if l > 0 { - // bounds check elimination - data, cmpData := b.set, compare.set - _ = data[l-1] - _ = cmpData[l-1] - - for i := 0; i < l; i++ { - data[i] |= cmpData[i] - } - } - if len(compare.set) > l { - for i := l; i < len(compare.set); i++ { - b.set[i] = compare.set[i] - } - } -} - -// SymmetricDifference of base set and other set -// This is the BitSet equivalent of ^ (xor) -func (b *BitSet) SymmetricDifference(compare *BitSet) (result *BitSet) { - panicIfNull(b) - panicIfNull(compare) - b, compare = sortByLength(b, compare) - // compare is bigger, so clone it - result = compare.Clone() - for i, word := range b.set { - result.set[i] = word ^ compare.set[i] - } - return -} - -// SymmetricDifferenceCardinality computes the cardinality of the symmetric difference -func (b *BitSet) SymmetricDifferenceCardinality(compare *BitSet) uint { - panicIfNull(b) - panicIfNull(compare) - b, compare = sortByLength(b, compare) - cnt := popcntXorSlice(b.set, compare.set) - if len(compare.set) > len(b.set) { - cnt += popcntSlice(compare.set[len(b.set):]) - } - return uint(cnt) -} - -// InPlaceSymmetricDifference creates the destructive SymmetricDifference of base set and other set -// This is the BitSet equivalent of ^ (xor) -func (b *BitSet) InPlaceSymmetricDifference(compare *BitSet) { - panicIfNull(b) - panicIfNull(compare) - l := compare.wordCount() - if l > b.wordCount() { - l = b.wordCount() - } - if compare.length > 0 && compare.length-1 >= b.length { - b.extendSet(compare.length - 1) - } - if l > 0 { - // bounds check elimination - data, cmpData := b.set, compare.set - _ = data[l-1] - _ = cmpData[l-1] - for i := 0; i < l; i++ { - data[i] ^= cmpData[i] - } - } - if len(compare.set) > l { - for i := l; i < len(compare.set); i++ { - b.set[i] = compare.set[i] - } - } -} - -// Is the length an exact multiple of word sizes? -func (b *BitSet) isLenExactMultiple() bool { - return wordsIndex(b.length) == 0 -} - -// Clean last word by setting unused bits to 0 -func (b *BitSet) cleanLastWord() { - if !b.isLenExactMultiple() { - b.set[len(b.set)-1] &= allBits >> (wordSize - wordsIndex(b.length)) - } -} - -// Complement computes the (local) complement of a bitset (up to length bits) -func (b *BitSet) Complement() (result *BitSet) { - panicIfNull(b) - result = New(b.length) - for i, word := range b.set { - result.set[i] = ^word - } - result.cleanLastWord() - return -} - -// All returns true if all bits are set, false otherwise. Returns true for -// empty sets. -func (b *BitSet) All() bool { - panicIfNull(b) - return b.Count() == b.length -} - -// None returns true if no bit is set, false otherwise. Returns true for -// empty sets. -func (b *BitSet) None() bool { - panicIfNull(b) - if b != nil && b.set != nil { - for _, word := range b.set { - if word > 0 { - return false - } - } - } - return true -} - -// Any returns true if any bit is set, false otherwise -func (b *BitSet) Any() bool { - panicIfNull(b) - return !b.None() -} - -// IsSuperSet returns true if this is a superset of the other set -func (b *BitSet) IsSuperSet(other *BitSet) bool { - l := other.wordCount() - if b.wordCount() < l { - l = b.wordCount() - } - for i, word := range other.set[:l] { - if b.set[i]&word != word { - return false - } - } - return popcntSlice(other.set[l:]) == 0 -} - -// IsStrictSuperSet returns true if this is a strict superset of the other set -func (b *BitSet) IsStrictSuperSet(other *BitSet) bool { - return b.Count() > other.Count() && b.IsSuperSet(other) -} - -// DumpAsBits dumps a bit set as a string of bits -func (b *BitSet) DumpAsBits() string { - if b.set == nil { - return "." - } - buffer := bytes.NewBufferString("") - i := len(b.set) - 1 - for ; i >= 0; i-- { - fmt.Fprintf(buffer, "%064b.", b.set[i]) - } - return buffer.String() -} - -// BinaryStorageSize returns the binary storage requirements (see WriteTo) in bytes. -func (b *BitSet) BinaryStorageSize() int { - return int(wordBytes + wordBytes*uint(b.wordCount())) -} - -func readUint64Array(reader io.Reader, data []uint64) error { - length := len(data) - bufferSize := 128 - buffer := make([]byte, bufferSize*int(wordBytes)) - for i := 0; i < length; i += bufferSize { - end := i + bufferSize - if end > length { - end = length - buffer = buffer[:wordBytes*uint(end-i)] - } - chunk := data[i:end] - if _, err := io.ReadFull(reader, buffer); err != nil { - return err - } - for i := range chunk { - chunk[i] = uint64(binaryOrder.Uint64(buffer[8*i:])) - } - } - return nil -} - -func writeUint64Array(writer io.Writer, data []uint64) error { - bufferSize := 128 - buffer := make([]byte, bufferSize*int(wordBytes)) - for i := 0; i < len(data); i += bufferSize { - end := i + bufferSize - if end > len(data) { - end = len(data) - buffer = buffer[:wordBytes*uint(end-i)] - } - chunk := data[i:end] - for i, x := range chunk { - binaryOrder.PutUint64(buffer[8*i:], x) - } - _, err := writer.Write(buffer) - if err != nil { - return err - } - } - return nil -} - -// WriteTo writes a BitSet to a stream. The format is: -// 1. uint64 length -// 2. []uint64 set -// Upon success, the number of bytes written is returned. -// -// Performance: if this function is used to write to a disk or network -// connection, it might be beneficial to wrap the stream in a bufio.Writer. -// E.g., -// -// f, err := os.Create("myfile") -// w := bufio.NewWriter(f) -func (b *BitSet) WriteTo(stream io.Writer) (int64, error) { - length := uint64(b.length) - // Write length - err := binary.Write(stream, binaryOrder, &length) - if err != nil { - // Upon failure, we do not guarantee that we - // return the number of bytes written. - return int64(0), err - } - err = writeUint64Array(stream, b.set[:b.wordCount()]) - if err != nil { - // Upon failure, we do not guarantee that we - // return the number of bytes written. - return int64(wordBytes), err - } - return int64(b.BinaryStorageSize()), nil -} - -// ReadFrom reads a BitSet from a stream written using WriteTo -// The format is: -// 1. uint64 length -// 2. []uint64 set -// Upon success, the number of bytes read is returned. -// If the current BitSet is not large enough to hold the data, -// it is extended. In case of error, the BitSet is either -// left unchanged or made empty if the error occurs too late -// to preserve the content. -// -// Performance: if this function is used to read from a disk or network -// connection, it might be beneficial to wrap the stream in a bufio.Reader. -// E.g., -// -// f, err := os.Open("myfile") -// r := bufio.NewReader(f) -func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { - var length uint64 - err := binary.Read(stream, binaryOrder, &length) - if err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return 0, err - } - newlength := uint(length) - - if uint64(newlength) != length { - return 0, errors.New("unmarshalling error: type mismatch") - } - nWords := wordsNeeded(uint(newlength)) - if cap(b.set) >= nWords { - b.set = b.set[:nWords] - } else { - b.set = make([]uint64, nWords) - } - - b.length = newlength - - err = readUint64Array(stream, b.set) - if err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - // We do not want to leave the BitSet partially filled as - // it is error prone. - b.set = b.set[:0] - b.length = 0 - return 0, err - } - - return int64(b.BinaryStorageSize()), nil -} - -// MarshalBinary encodes a BitSet into a binary form and returns the result. -func (b *BitSet) MarshalBinary() ([]byte, error) { - var buf bytes.Buffer - _, err := b.WriteTo(&buf) - if err != nil { - return []byte{}, err - } - - return buf.Bytes(), err -} - -// UnmarshalBinary decodes the binary form generated by MarshalBinary. -func (b *BitSet) UnmarshalBinary(data []byte) error { - buf := bytes.NewReader(data) - _, err := b.ReadFrom(buf) - return err -} - -// MarshalJSON marshals a BitSet as a JSON structure -func (b BitSet) MarshalJSON() ([]byte, error) { - buffer := bytes.NewBuffer(make([]byte, 0, b.BinaryStorageSize())) - _, err := b.WriteTo(buffer) - if err != nil { - return nil, err - } - - // URLEncode all bytes - return json.Marshal(base64Encoding.EncodeToString(buffer.Bytes())) -} - -// UnmarshalJSON unmarshals a BitSet from JSON created using MarshalJSON -func (b *BitSet) UnmarshalJSON(data []byte) error { - // Unmarshal as string - var s string - err := json.Unmarshal(data, &s) - if err != nil { - return err - } - - // URLDecode string - buf, err := base64Encoding.DecodeString(s) - if err != nil { - return err - } - - _, err = b.ReadFrom(bytes.NewReader(buf)) - return err -} diff --git a/vendor/github.com/bits-and-blooms/bitset/popcnt.go b/vendor/github.com/bits-and-blooms/bitset/popcnt.go deleted file mode 100644 index 76577a8382..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/popcnt.go +++ /dev/null @@ -1,53 +0,0 @@ -package bitset - -// bit population count, take from -// https://code.google.com/p/go/issues/detail?id=4988#c11 -// credit: https://code.google.com/u/arnehormann/ -func popcount(x uint64) (n uint64) { - x -= (x >> 1) & 0x5555555555555555 - x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 - x += x >> 4 - x &= 0x0f0f0f0f0f0f0f0f - x *= 0x0101010101010101 - return x >> 56 -} - -func popcntSliceGo(s []uint64) uint64 { - cnt := uint64(0) - for _, x := range s { - cnt += popcount(x) - } - return cnt -} - -func popcntMaskSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcount(s[i] &^ m[i]) - } - return cnt -} - -func popcntAndSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcount(s[i] & m[i]) - } - return cnt -} - -func popcntOrSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcount(s[i] | m[i]) - } - return cnt -} - -func popcntXorSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) - for i := range s { - cnt += popcount(s[i] ^ m[i]) - } - return cnt -} diff --git a/vendor/github.com/bits-and-blooms/bitset/popcnt_19.go b/vendor/github.com/bits-and-blooms/bitset/popcnt_19.go deleted file mode 100644 index 7855c04b5b..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/popcnt_19.go +++ /dev/null @@ -1,62 +0,0 @@ -//go:build go1.9 -// +build go1.9 - -package bitset - -import "math/bits" - -func popcntSlice(s []uint64) uint64 { - var cnt int - for _, x := range s { - cnt += bits.OnesCount64(x) - } - return uint64(cnt) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - var cnt int - // this explicit check eliminates a bounds check in the loop - if len(m) < len(s) { - panic("mask slice is too short") - } - for i := range s { - cnt += bits.OnesCount64(s[i] &^ m[i]) - } - return uint64(cnt) -} - -func popcntAndSlice(s, m []uint64) uint64 { - var cnt int - // this explicit check eliminates a bounds check in the loop - if len(m) < len(s) { - panic("mask slice is too short") - } - for i := range s { - cnt += bits.OnesCount64(s[i] & m[i]) - } - return uint64(cnt) -} - -func popcntOrSlice(s, m []uint64) uint64 { - var cnt int - // this explicit check eliminates a bounds check in the loop - if len(m) < len(s) { - panic("mask slice is too short") - } - for i := range s { - cnt += bits.OnesCount64(s[i] | m[i]) - } - return uint64(cnt) -} - -func popcntXorSlice(s, m []uint64) uint64 { - var cnt int - // this explicit check eliminates a bounds check in the loop - if len(m) < len(s) { - panic("mask slice is too short") - } - for i := range s { - cnt += bits.OnesCount64(s[i] ^ m[i]) - } - return uint64(cnt) -} diff --git a/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.go b/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.go deleted file mode 100644 index 116e044407..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.go +++ /dev/null @@ -1,68 +0,0 @@ -//go:build !go1.9 && amd64 && !appengine -// +build !go1.9,amd64,!appengine - -package bitset - -// *** the following functions are defined in popcnt_amd64.s - -//go:noescape - -func hasAsm() bool - -// useAsm is a flag used to select the GO or ASM implementation of the popcnt function -var useAsm = hasAsm() - -//go:noescape - -func popcntSliceAsm(s []uint64) uint64 - -//go:noescape - -func popcntMaskSliceAsm(s, m []uint64) uint64 - -//go:noescape - -func popcntAndSliceAsm(s, m []uint64) uint64 - -//go:noescape - -func popcntOrSliceAsm(s, m []uint64) uint64 - -//go:noescape - -func popcntXorSliceAsm(s, m []uint64) uint64 - -func popcntSlice(s []uint64) uint64 { - if useAsm { - return popcntSliceAsm(s) - } - return popcntSliceGo(s) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - if useAsm { - return popcntMaskSliceAsm(s, m) - } - return popcntMaskSliceGo(s, m) -} - -func popcntAndSlice(s, m []uint64) uint64 { - if useAsm { - return popcntAndSliceAsm(s, m) - } - return popcntAndSliceGo(s, m) -} - -func popcntOrSlice(s, m []uint64) uint64 { - if useAsm { - return popcntOrSliceAsm(s, m) - } - return popcntOrSliceGo(s, m) -} - -func popcntXorSlice(s, m []uint64) uint64 { - if useAsm { - return popcntXorSliceAsm(s, m) - } - return popcntXorSliceGo(s, m) -} diff --git a/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.s b/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.s deleted file mode 100644 index 666c0dcc17..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.s +++ /dev/null @@ -1,104 +0,0 @@ -// +build !go1.9 -// +build amd64,!appengine - -TEXT ·hasAsm(SB),4,$0-1 -MOVQ $1, AX -CPUID -SHRQ $23, CX -ANDQ $1, CX -MOVB CX, ret+0(FP) -RET - -#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 - -TEXT ·popcntSliceAsm(SB),4,$0-32 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntSliceEnd -popcntSliceLoop: -BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX -ADDQ DX, AX -ADDQ $8, SI -LOOP popcntSliceLoop -popcntSliceEnd: -MOVQ AX, ret+24(FP) -RET - -TEXT ·popcntMaskSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntMaskSliceEnd -MOVQ m+24(FP), DI -popcntMaskSliceLoop: -MOVQ (DI), DX -NOTQ DX -ANDQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntMaskSliceLoop -popcntMaskSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntAndSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntAndSliceEnd -MOVQ m+24(FP), DI -popcntAndSliceLoop: -MOVQ (DI), DX -ANDQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntAndSliceLoop -popcntAndSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntOrSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntOrSliceEnd -MOVQ m+24(FP), DI -popcntOrSliceLoop: -MOVQ (DI), DX -ORQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntOrSliceLoop -popcntOrSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntXorSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntXorSliceEnd -MOVQ m+24(FP), DI -popcntXorSliceLoop: -MOVQ (DI), DX -XORQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntXorSliceLoop -popcntXorSliceEnd: -MOVQ AX, ret+48(FP) -RET diff --git a/vendor/github.com/bits-and-blooms/bitset/popcnt_generic.go b/vendor/github.com/bits-and-blooms/bitset/popcnt_generic.go deleted file mode 100644 index 9e0ad464e0..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/popcnt_generic.go +++ /dev/null @@ -1,25 +0,0 @@ -//go:build !go1.9 && (!amd64 || appengine) -// +build !go1.9 -// +build !amd64 appengine - -package bitset - -func popcntSlice(s []uint64) uint64 { - return popcntSliceGo(s) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - return popcntMaskSliceGo(s, m) -} - -func popcntAndSlice(s, m []uint64) uint64 { - return popcntAndSliceGo(s, m) -} - -func popcntOrSlice(s, m []uint64) uint64 { - return popcntOrSliceGo(s, m) -} - -func popcntXorSlice(s, m []uint64) uint64 { - return popcntXorSliceGo(s, m) -} diff --git a/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_18.go b/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_18.go deleted file mode 100644 index 12336e76af..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_18.go +++ /dev/null @@ -1,15 +0,0 @@ -//go:build !go1.9 -// +build !go1.9 - -package bitset - -var deBruijn = [...]byte{ - 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, - 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, - 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, - 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, -} - -func trailingZeroes64(v uint64) uint { - return uint(deBruijn[((v&-v)*0x03f79d71b4ca8b09)>>58]) -} diff --git a/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_19.go b/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_19.go deleted file mode 100644 index cfb0a84091..0000000000 --- a/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_19.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build go1.9 -// +build go1.9 - -package bitset - -import "math/bits" - -func trailingZeroes64(v uint64) uint { - return uint(bits.TrailingZeros64(v)) -} diff --git a/vendor/github.com/longhorn/backing-image-manager/pkg/util/bitmap.go b/vendor/github.com/longhorn/backing-image-manager/pkg/util/bitmap.go deleted file mode 100644 index 1d8fc8ef28..0000000000 --- a/vendor/github.com/longhorn/backing-image-manager/pkg/util/bitmap.go +++ /dev/null @@ -1,86 +0,0 @@ -package util - -import ( - "fmt" - "sync" - - "github.com/RoaringBitmap/roaring" -) - -type Bitmap struct { - base int32 - size int32 - data *roaring.Bitmap - lock *sync.Mutex -} - -// NewBitmap allocate a bitmap range from [start, end], notice the end is included -func NewBitmap(start, end int32) *Bitmap { - size := end - start + 1 - data := roaring.New() - if size > 0 { - data.AddRange(0, uint64(size)) - } - return &Bitmap{ - base: start, - size: size, - data: data, - lock: &sync.Mutex{}, - } -} - -func (b *Bitmap) AllocateRange(count int32) (int32, int32, error) { - b.lock.Lock() - defer b.lock.Unlock() - - if count <= 0 { - return 0, 0, fmt.Errorf("invalid request for non-positive counts: %v", count) - } - i := b.data.Iterator() - bStart := int32(0) - for bStart <= b.size { - last := int32(-1) - remains := count - for i.HasNext() && remains > 0 { - // first element - if last < 0 { - last = int32(i.Next()) - bStart = last - remains-- - continue - } - next := int32(i.Next()) - // failed to find the available range - if next-last > 1 { - break - } - last = next - remains-- - } - if remains == 0 { - break - } - if !i.HasNext() { - return 0, 0, fmt.Errorf("cannot find an empty port range") - } - } - bEnd := bStart + count - 1 - b.data.RemoveRange(uint64(bStart), uint64(bEnd)+1) - return b.base + bStart, b.base + bEnd, nil -} - -func (b *Bitmap) ReleaseRange(start, end int32) error { - b.lock.Lock() - defer b.lock.Unlock() - - if start == end && end == 0 { - return nil - } - bStart := start - b.base - bEnd := end - b.base - if bStart < 0 || bEnd >= b.size { - return fmt.Errorf("exceed range: %v-%v (%v-%v)", start, end, bStart, bEnd) - } - b.data.AddRange(uint64(bStart), uint64(bEnd)+1) - return nil -} diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/types/types.go b/vendor/github.com/longhorn/longhorn-engine/pkg/types/types.go index 3e02bbb7e0..83197a71e4 100644 --- a/vendor/github.com/longhorn/longhorn-engine/pkg/types/types.go +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/types/types.go @@ -109,7 +109,14 @@ type Backend interface { } type BackendFactory interface { - Create(volumeName, address string, dataServerProtocol DataServerProtocol, engineReplicaTimeout time.Duration) (Backend, error) + Create(volumeName, address string, dataServerProtocol DataServerProtocol, + sharedTimeouts SharedTimeouts) (Backend, error) +} + +type SharedTimeouts interface { + Increment() + Decrement() + CheckAndDecrement(duration time.Duration) time.Duration } type Controller interface { diff --git a/vendor/github.com/longhorn/longhorn-engine/pkg/util/shared_timeouts.go b/vendor/github.com/longhorn/longhorn-engine/pkg/util/shared_timeouts.go new file mode 100644 index 0000000000..c609d1cec0 --- /dev/null +++ b/vendor/github.com/longhorn/longhorn-engine/pkg/util/shared_timeouts.go @@ -0,0 +1,63 @@ +package util + +import ( + "sync" + "time" +) + +// SharedTimeouts has the following use case: +// - Multiple goroutines may need to time out eventually. +// - Only the goroutines themselves know if the conditions for a timeout have been met. +// - It is fine for some of the goroutines to time out quickly. +// - The last goroutine should time out more slowly. +// SharedTimeouts implements the types.SharedTimeouts instead of directly defining the concrete type to avoid an import +// loop. +type SharedTimeouts struct { + mutex sync.RWMutex + longTimeout time.Duration + shortTimeout time.Duration + numConsumers int +} + +func NewSharedTimeouts(shortTimeout, longTimeout time.Duration) *SharedTimeouts { + return &SharedTimeouts{ + longTimeout: longTimeout, + shortTimeout: shortTimeout, + } +} + +func (t *SharedTimeouts) Increment() { + t.mutex.Lock() + defer t.mutex.Unlock() + t.numConsumers++ +} + +func (t *SharedTimeouts) Decrement() { + t.mutex.Lock() + defer t.mutex.Unlock() + t.numConsumers-- +} + +// CheckAndDecrement checks if duration exceeds longTimeout or shortTimeout, returns the timeout exceeded (if +// applicable) and decrements numConsumers. +// - shortTimeout is only considered exceeded if there is still one other consumer to wait for longTimeout. +// - The caller MUST take whatever action is required for a timeout if a value > 0 is returned. +func (t *SharedTimeouts) CheckAndDecrement(duration time.Duration) time.Duration { + if duration > t.longTimeout { + t.mutex.Lock() + defer t.mutex.Unlock() + t.numConsumers-- + return t.longTimeout + } + + if duration > t.shortTimeout { + t.mutex.Lock() + defer t.mutex.Unlock() + if t.numConsumers > 1 { + t.numConsumers-- + return t.shortTimeout + } + } + + return 0 +} diff --git a/vendor/github.com/mschoch/smat/.gitignore b/vendor/github.com/mschoch/smat/.gitignore deleted file mode 100644 index eee880759d..0000000000 --- a/vendor/github.com/mschoch/smat/.gitignore +++ /dev/null @@ -1,14 +0,0 @@ -#* -*.sublime-* -*~ -.#* -.project -.settings -**/.idea/ -**/*.iml -/examples/bolt/boltsmat-fuzz.zip -/examples/bolt/workdir/ -.DS_Store -coverage.out -*.test -tags diff --git a/vendor/github.com/mschoch/smat/.travis.yml b/vendor/github.com/mschoch/smat/.travis.yml deleted file mode 100644 index 3c9c346365..0000000000 --- a/vendor/github.com/mschoch/smat/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -sudo: false -language: go -go: -- 1.6 -script: -- go get golang.org/x/tools/cmd/cover -- go get github.com/mattn/goveralls -- go get github.com/kisielk/errcheck -- go test -v -race -- go vet -- errcheck ./... -- go test -coverprofile=profile.out -covermode=count -- goveralls -service=travis-ci -coverprofile=profile.out -repotoken $COVERALLS -notifications: - email: - - marty.schoch@gmail.com diff --git a/vendor/github.com/mschoch/smat/LICENSE b/vendor/github.com/mschoch/smat/LICENSE deleted file mode 100644 index 7a4a3ea242..0000000000 --- a/vendor/github.com/mschoch/smat/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/vendor/github.com/mschoch/smat/README.md b/vendor/github.com/mschoch/smat/README.md deleted file mode 100644 index f5ca1c5440..0000000000 --- a/vendor/github.com/mschoch/smat/README.md +++ /dev/null @@ -1,166 +0,0 @@ -# smat – State Machine Assisted Testing - -The concept is simple, describe valid uses of your library as states and actions. States describe which actions are possible, and with what probability they should occur. Actions mutate the context and transition to another state. - -By doing this, two things are possible: - -1. Use [go-fuzz](https://github.com/dvyukov/go-fuzz) to find/test interesting sequences of operations on your library. - -2. Automate longevity testing of your application by performing long sequences of valid operations. - -**NOTE**: both of these can also incorporate validation logic (not just failure detection by building validation into the state machine) - -## Status - -The API is still not stable. This is brand new and we'll probably change things we don't like... - -[![Build Status](https://travis-ci.org/mschoch/smat.svg?branch=master)](https://travis-ci.org/mschoch/smat) -[![Coverage Status](https://coveralls.io/repos/github/mschoch/smat/badge.svg?branch=master)](https://coveralls.io/github/mschoch/smat?branch=master) -[![GoDoc](https://godoc.org/github.com/mschoch/smat?status.svg)](https://godoc.org/github.com/mschoch/smat) -[![codebeat badge](https://codebeat.co/badges/c3ff6180-a241-4128-97f0-fa6bf6f48752)](https://codebeat.co/projects/github-com-mschoch-smat) -[![Go Report Card](https://goreportcard.com/badge/github.com/mschoch/smat)](https://goreportcard.com/report/github.com/mschoch/smat) - -## License - -Apache 2.0 - -## How do I use it? - -### smat.Context - -Choose a structure to keep track of any state. You pass in an instance of this when you start, and it will be passed to every action when it executes. The actions may mutate this context. - -For example, consider a database library, once you open a database handle, you need to use it inside of the other actions. So you might use a structure like: - -``` -type context struct { - db *DB -} -``` - -### smat.State - -A state represents a state that your application/library can be in, and the probabilities thats certain actions should be taken. - -For example, consider a database library, in a state where the database is open, there many things you can do. Let's consider just two right now, you can set a value, or you can delete a value. - -``` -func dbOpen(next byte) smat.ActionID { - return smat.PercentExecute(next, - smat.PercentAction{50, setValue}, - smat.PercentAction{50, deleteValue}, - ) -} -``` - -This says that in the open state, there are two valid actions, 50% of the time you should set a value and 50% of the time you should delete a value. **NOTE**: these percentages are just for characterizing the test workload. - -### smat.Action - -Actions are functions that do some work, optionally mutate the context, and indicate the next state to transition to. Below we see an example action to set value in a database. - -``` -func setValueFunc(ctx smat.Context) (next smat.State, err error) { - // type assert to our custom context type - context := ctx.(*context) - // perform the operation - err = context.db.Set("k", "v") - if err != nil { - return nil, err - } - // return the new state - return dbOpen, nil -} -``` - -### smat.ActionID and smat.ActionMap - -Actions are just functions, and since we can't compare functions in Go, we need to introduce an external identifier for them. This allows us to build a bi-directional mapping which we'll take advantage of later. - -``` -const ( - setup smat.ActionID = iota - teardown - setValue - deleteValue -) - -var actionMap = smat.ActionMap{ - setup: setupFunc, - teardown: teardownFunc, - setValue: setValueFunc, - deleteValue: deleteValueFunc, -} -``` - -### smat.ActionSeq - -A common way that many users think about a library is as a sequence of actions to be performed. Using the ActionID's that we've already seen we can build up sequences of operations. - -``` - actionSeq := smat.ActionSeq{ - open, - setValue, - setValue, - setValue, - } -``` - -Notice that we build these actions using the constants we defined above, and because of this we can have a bi-directional mapping between a stream of bytes (driving the state machine) and a sequence of actions to be performed. - -## Fuzzing - -We've built a lot of pieces, lets wire it up to go-fuzz. - -``` -func Fuzz(data []byte) int { - return smat.Fuzz(&context{}, setup, teardown, actionMap, data) -} -``` - -* The first argument is an instance of context structure. -* The second argument is the ActionID of our setup function. The setup function does not consume any of the input stream and is used to initialize the context and determine the start state. -* The third argument is the teardown function. This will be called unconditionally to clean up any resources associated with the test. -* The fourth argument is the actionMap which maps all ActionIDs to Actions. -* The fifth argument is the data passed in from the go-fuzz application. - -### Generating Initial go-fuzz Corpus - -Earlier we mentioned the bi-directional mapping between Actions and the byte stream driving the state machine. We can now leverage this to build the inital go-fuzz corpus. - -Using the `ActinSeq`s we learned about earlier we can build up a list of them as: - - var actionSeqs = []smat.ActionSeq{...} - -Then, we can write them out to disk using: - -``` -for i, actionSeq := range actionSeqs { - byteSequence, err := actionSeq.ByteEncoding(&context{}, setup, teardown, actionMap) - if err != nil { - // handle error - } - os.MkdirAll("workdir/corpus", 0700) - ioutil.WriteFile(fmt.Sprintf("workdir/corpus/%d", i), byteSequence, 0600) -} -``` - -You can then either put this into a test case or a main application depending on your needs. - -## Longevity Testing - -Fuzzing is great, but most of your corpus is likely to be shorter meaningful sequences. And go-fuzz works to find shortest sequences that cause problems, but sometimes you actually want to explore longer sequences that appear to go-fuzz as not triggering additional code coverage. - -For these cases we have another helper you can use: - -``` - Longevity(ctx, setup, teardown, actionMap, 0, closeChan) -``` - -The first four arguments are the same, the last two are: -* random seed used to ensure repeatable tests -* closeChan (chan struct{}) - close this channel if you want the function to stop and return ErrClosed, otherwise it will run forever - -## Examples - -See the examples directory for a working example that tests some BoltDB functionality. diff --git a/vendor/github.com/mschoch/smat/actionseq.go b/vendor/github.com/mschoch/smat/actionseq.go deleted file mode 100644 index 6c8297f891..0000000000 --- a/vendor/github.com/mschoch/smat/actionseq.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2016 Marty Schoch - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the -// License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an "AS -// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language -// governing permissions and limitations under the License. - -package smat - -// ActionSeq represents a sequence of actions, used for populating a corpus -// of byte sequences for the corresponding fuzz tests -type ActionSeq []ActionID - -// ByteEncoding runs the FSM to produce a byte sequence to trigger the -// desired action -func (a ActionSeq) ByteEncoding(ctx Context, setup, teardown ActionID, actionMap ActionMap) ([]byte, error) { - setupFunc, teardownFunc, err := actionMap.findSetupTeardown(setup, teardown) - if err != nil { - return nil, err - } - state, err := setupFunc(ctx) - if err != nil { - return nil, err - } - defer func() { - _, _ = teardownFunc(ctx) - }() - - var rv []byte - for _, actionID := range a { - b, err := probeStateForAction(state, actionID) - if err != nil { - return nil, err - } - rv = append(rv, b) - action, ok := actionMap[actionID] - if !ok { - continue - } - state, err = action(ctx) - if err != nil { - return nil, err - } - } - return rv, nil -} - -func probeStateForAction(state State, actionID ActionID) (byte, error) { - for i := 0; i < 256; i++ { - nextActionID := state(byte(i)) - if nextActionID == actionID { - return byte(i), nil - } - } - return 0, ErrActionNotPossible -} diff --git a/vendor/github.com/mschoch/smat/smat.go b/vendor/github.com/mschoch/smat/smat.go deleted file mode 100644 index f6ea4975f2..0000000000 --- a/vendor/github.com/mschoch/smat/smat.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright (c) 2016 Marty Schoch - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the -// License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an "AS -// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language -// governing permissions and limitations under the License. - -package smat - -import ( - "bufio" - "bytes" - "fmt" - "io" - "io/ioutil" - "log" - "math/rand" -) - -// Logger is a configurable logger used by this package -// by default output is discarded -var Logger = log.New(ioutil.Discard, "smat ", log.LstdFlags) - -// Context is a container for any user state -type Context interface{} - -// State is a function which describes which action to perform in the event -// that a particular byte is seen -type State func(next byte) ActionID - -// PercentAction describes the frequency with which an action should occur -// for example: Action{Percent:10, Action:DonateMoney} means that 10% of -// the time you should donate money. -type PercentAction struct { - Percent int - Action ActionID -} - -// Action is any function which returns the next state to transition to -// it can optionally mutate the provided context object -// if any error occurs, it may return an error which will abort execution -type Action func(Context) (State, error) - -// ActionID is a unique identifier for an action -type ActionID int - -// NopAction does nothing and simply continues to the next input -var NopAction ActionID = -1 - -// ActionMap is a mapping form ActionID to Action -type ActionMap map[ActionID]Action - -func (a ActionMap) findSetupTeardown(setup, teardown ActionID) (Action, Action, error) { - setupFunc, ok := a[setup] - if !ok { - return nil, nil, ErrSetupMissing - } - teardownFunc, ok := a[teardown] - if !ok { - return nil, nil, ErrTeardownMissing - } - return setupFunc, teardownFunc, nil -} - -// Fuzz runs the fuzzing state machine with the provided context -// first, the setup action is executed unconditionally -// the start state is determined by this action -// actionMap is a lookup table for all actions -// the data byte slice determines all future state transitions -// finally, the teardown action is executed unconditionally for cleanup -func Fuzz(ctx Context, setup, teardown ActionID, actionMap ActionMap, data []byte) int { - reader := bytes.NewReader(data) - err := runReader(ctx, setup, teardown, actionMap, reader, nil) - if err != nil { - panic(err) - } - return 1 -} - -// Longevity runs the state machine with the provided context -// first, the setup action is executed unconditionally -// the start state is determined by this action -// actionMap is a lookup table for all actions -// random bytes are generated to determine all future state transitions -// finally, the teardown action is executed unconditionally for cleanup -func Longevity(ctx Context, setup, teardown ActionID, actionMap ActionMap, seed int64, closeChan chan struct{}) error { - source := rand.NewSource(seed) - return runReader(ctx, setup, teardown, actionMap, rand.New(source), closeChan) -} - -var ( - // ErrSetupMissing is returned when the setup action cannot be found - ErrSetupMissing = fmt.Errorf("setup action missing") - // ErrTeardownMissing is returned when the teardown action cannot be found - ErrTeardownMissing = fmt.Errorf("teardown action missing") - // ErrClosed is returned when the closeChan was closed to cancel the op - ErrClosed = fmt.Errorf("closed") - // ErrActionNotPossible is returned when an action is encountered in a - // FuzzCase that is not possible in the current state - ErrActionNotPossible = fmt.Errorf("action not possible in state") -) - -func runReader(ctx Context, setup, teardown ActionID, actionMap ActionMap, r io.Reader, closeChan chan struct{}) error { - setupFunc, teardownFunc, err := actionMap.findSetupTeardown(setup, teardown) - if err != nil { - return err - } - Logger.Printf("invoking setup action") - state, err := setupFunc(ctx) - if err != nil { - return err - } - defer func() { - Logger.Printf("invoking teardown action") - _, _ = teardownFunc(ctx) - }() - - reader := bufio.NewReader(r) - for next, err := reader.ReadByte(); err == nil; next, err = reader.ReadByte() { - select { - case <-closeChan: - return ErrClosed - default: - actionID := state(next) - action, ok := actionMap[actionID] - if !ok { - Logger.Printf("no such action defined, continuing") - continue - } - Logger.Printf("invoking action - %d", actionID) - state, err = action(ctx) - if err != nil { - Logger.Printf("it was action %d that returned err %v", actionID, err) - return err - } - } - } - return err -} - -// PercentExecute interprets the next byte as a random value and normalizes it -// to values 0-99, it then looks to see which action should be execued based -// on the action distributions -func PercentExecute(next byte, pas ...PercentAction) ActionID { - percent := int(99 * int(next) / 255) - - sofar := 0 - for _, pa := range pas { - sofar = sofar + pa.Percent - if percent < sofar { - return pa.Action - } - - } - return NopAction -} diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/config.go b/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/config.go index e3e1d452b0..d9b91a24b1 100644 --- a/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/config.go +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/config.go @@ -37,10 +37,14 @@ type Filter func(*InterceptorInfo) bool // config is a group of options for this instrumentation. type config struct { - Filter Filter - Propagators propagation.TextMapPropagator - TracerProvider trace.TracerProvider - MeterProvider metric.MeterProvider + Filter Filter + Propagators propagation.TextMapPropagator + TracerProvider trace.TracerProvider + MeterProvider metric.MeterProvider + SpanStartOptions []trace.SpanStartOption + + ReceivedEvent bool + SentEvent bool meter metric.Meter rpcServerDuration metric.Int64Histogram @@ -68,7 +72,10 @@ func newConfig(opts []Option) *config { metric.WithSchemaURL(semconv.SchemaURL), ) var err error - if c.rpcServerDuration, err = c.meter.Int64Histogram("rpc.server.duration", metric.WithUnit("ms")); err != nil { + c.rpcServerDuration, err = c.meter.Int64Histogram("rpc.server.duration", + metric.WithDescription("Measures the duration of inbound RPC."), + metric.WithUnit("ms")) + if err != nil { otel.Handle(err) } @@ -131,3 +138,50 @@ func (o meterProviderOption) apply(c *config) { func WithMeterProvider(mp metric.MeterProvider) Option { return meterProviderOption{mp: mp} } + +// Event type that can be recorded, see WithMessageEvents. +type Event int + +// Different types of events that can be recorded, see WithMessageEvents. +const ( + ReceivedEvents Event = iota + SentEvents +) + +type messageEventsProviderOption struct { + events []Event +} + +func (m messageEventsProviderOption) apply(c *config) { + for _, e := range m.events { + switch e { + case ReceivedEvents: + c.ReceivedEvent = true + case SentEvents: + c.SentEvent = true + } + } +} + +// WithMessageEvents configures the Handler to record the specified events +// (span.AddEvent) on spans. By default only summary attributes are added at the +// end of the request. +// +// Valid events are: +// - ReceivedEvents: Record the number of bytes read after every gRPC read operation. +// - SentEvents: Record the number of bytes written after every gRPC write operation. +func WithMessageEvents(events ...Event) Option { + return messageEventsProviderOption{events: events} +} + +type spanStartOption struct{ opts []trace.SpanStartOption } + +func (o spanStartOption) apply(c *config) { + c.SpanStartOptions = append(c.SpanStartOptions, o.opts...) +} + +// WithSpanOptions configures an additional set of +// trace.SpanOptions, which are applied to each new span. +func WithSpanOptions(opts ...trace.SpanStartOption) Option { + return spanStartOption{opts} +} diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/doc.go b/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/doc.go new file mode 100644 index 0000000000..a993e0fc92 --- /dev/null +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/doc.go @@ -0,0 +1,45 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package otelgrpc is the instrumentation library for [google.golang.org/grpc] + +For now you can instrument your program which use [google.golang.org/grpc] in two ways: + + - by [grpc.UnaryClientInterceptor], [grpc.UnaryServerInterceptor], [grpc.StreamClientInterceptor], [grpc.StreamServerInterceptor] + - by [stats.Handler] + +Notice: Do not use both interceptors and [stats.Handler] at the same time! If so, you will get duplicated spans and the parent/child relationships between spans will also be broken. + +We strongly still recommand you to use [stats.Handler], mainly for two reasons: + +Functional advantages: [stats.Handler] has more information for user to build more flexible and granular metric, for example + + - multiple different types of represent "data length": In [stats.InPayload], there exists "Length", "CompressedLength", "WireLength" to denote the size of uncompressed, compressed payload data, with or without framing data. But in interceptors, we can only got uncompressed data, and this feature is also removed due to performance problem. + + - more accurate timestamp: [stats.InPayload]'s "RecvTime" and [stats.OutPayload]'s "SentTime" records more accurate timestamp that server got and sent the message, the timestamp recorded by interceptors depends on the location of this interceptors in the total interceptor chain. + + - some other use cases: for example, catch failure of decoding message. + +Performance advantages: If too many interceptors are registered in a service, the interceptor chain can become too long, which increases the latency and processing time of the entire RPC call. + +[stats.Handler]: https://pkg.go.dev/google.golang.org/grpc/stats#Handler +[grpc.UnaryClientInterceptor]: https://pkg.go.dev/google.golang.org/grpc#UnaryClientInterceptor +[grpc.UnaryServerInterceptor]: https://pkg.go.dev/google.golang.org/grpc#UnaryServerInterceptor +[grpc.StreamClientInterceptor]: https://pkg.go.dev/google.golang.org/grpc#StreamClientInterceptor +[grpc.StreamServerInterceptor]: https://pkg.go.dev/google.golang.org/grpc#StreamServerInterceptor +[stats.OutPayload]: https://pkg.go.dev/google.golang.org/grpc/stats#OutPayload +[stats.InPayload]: https://pkg.go.dev/google.golang.org/grpc/stats#InPayload +*/ +package otelgrpc // import "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go b/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go index d4dc5de5a9..561154061f 100644 --- a/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go @@ -84,22 +84,31 @@ func UnaryClientInterceptor(opts ...Option) grpc.UnaryClientInterceptor { } name, attr := spanInfo(method, cc.Target()) - var span trace.Span - ctx, span = tracer.Start( + + startOpts := append([]trace.SpanStartOption{ + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attr...)}, + cfg.SpanStartOptions..., + ) + + ctx, span := tracer.Start( ctx, name, - trace.WithSpanKind(trace.SpanKindClient), - trace.WithAttributes(attr...), + startOpts..., ) defer span.End() ctx = inject(ctx, cfg.Propagators) - messageSent.Event(ctx, 1, req) + if cfg.SentEvent { + messageSent.Event(ctx, 1, req) + } err := invoker(ctx, method, req, reply, cc, callOpts...) - messageReceived.Event(ctx, 1, reply) + if cfg.ReceivedEvent { + messageReceived.Event(ctx, 1, reply) + } if err != nil { s, _ := status.FromError(err) @@ -135,6 +144,9 @@ type clientStream struct { eventsDone chan struct{} finished chan error + receivedEvent bool + sentEvent bool + receivedMessageID int sentMessageID int } @@ -152,7 +164,10 @@ func (w *clientStream) RecvMsg(m interface{}) error { w.sendStreamEvent(errorEvent, err) } else { w.receivedMessageID++ - messageReceived.Event(w.Context(), w.receivedMessageID, m) + + if w.receivedEvent { + messageReceived.Event(w.Context(), w.receivedMessageID, m) + } } return err @@ -162,7 +177,10 @@ func (w *clientStream) SendMsg(m interface{}) error { err := w.ClientStream.SendMsg(m) w.sentMessageID++ - messageSent.Event(w.Context(), w.sentMessageID, m) + + if w.sentEvent { + messageSent.Event(w.Context(), w.sentMessageID, m) + } if err != nil { w.sendStreamEvent(errorEvent, err) @@ -191,7 +209,7 @@ func (w *clientStream) CloseSend() error { return err } -func wrapClientStream(ctx context.Context, s grpc.ClientStream, desc *grpc.StreamDesc) *clientStream { +func wrapClientStream(ctx context.Context, s grpc.ClientStream, desc *grpc.StreamDesc, cfg *config) *clientStream { events := make(chan streamEvent) eventsDone := make(chan struct{}) finished := make(chan error) @@ -218,11 +236,13 @@ func wrapClientStream(ctx context.Context, s grpc.ClientStream, desc *grpc.Strea }() return &clientStream{ - ClientStream: s, - desc: desc, - events: events, - eventsDone: eventsDone, - finished: finished, + ClientStream: s, + desc: desc, + events: events, + eventsDone: eventsDone, + finished: finished, + receivedEvent: cfg.ReceivedEvent, + sentEvent: cfg.SentEvent, } } @@ -259,12 +279,17 @@ func StreamClientInterceptor(opts ...Option) grpc.StreamClientInterceptor { } name, attr := spanInfo(method, cc.Target()) - var span trace.Span - ctx, span = tracer.Start( + + startOpts := append([]trace.SpanStartOption{ + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attr...)}, + cfg.SpanStartOptions..., + ) + + ctx, span := tracer.Start( ctx, name, - trace.WithSpanKind(trace.SpanKindClient), - trace.WithAttributes(attr...), + startOpts..., ) ctx = inject(ctx, cfg.Propagators) @@ -277,7 +302,7 @@ func StreamClientInterceptor(opts ...Option) grpc.StreamClientInterceptor { span.End() return s, err } - stream := wrapClientStream(ctx, s, desc) + stream := wrapClientStream(ctx, s, desc, cfg) go func() { err := <-stream.finished @@ -321,17 +346,24 @@ func UnaryServerInterceptor(opts ...Option) grpc.UnaryServerInterceptor { } ctx = extract(ctx, cfg.Propagators) - name, attr := spanInfo(info.FullMethod, peerFromCtx(ctx)) + + startOpts := append([]trace.SpanStartOption{ + trace.WithSpanKind(trace.SpanKindServer), + trace.WithAttributes(attr...)}, + cfg.SpanStartOptions..., + ) + ctx, span := tracer.Start( trace.ContextWithRemoteSpanContext(ctx, trace.SpanContextFromContext(ctx)), name, - trace.WithSpanKind(trace.SpanKindServer), - trace.WithAttributes(attr...), + startOpts..., ) defer span.End() - messageReceived.Event(ctx, 1, req) + if cfg.ReceivedEvent { + messageReceived.Event(ctx, 1, req) + } var statusCode grpc_codes.Code defer func(t time.Time) { @@ -347,11 +379,15 @@ func UnaryServerInterceptor(opts ...Option) grpc.UnaryServerInterceptor { statusCode, msg := serverStatus(s) span.SetStatus(statusCode, msg) span.SetAttributes(statusCodeAttr(s.Code())) - messageSent.Event(ctx, 1, s.Proto()) + if cfg.SentEvent { + messageSent.Event(ctx, 1, s.Proto()) + } } else { statusCode = grpc_codes.OK span.SetAttributes(statusCodeAttr(grpc_codes.OK)) - messageSent.Event(ctx, 1, resp) + if cfg.SentEvent { + messageSent.Event(ctx, 1, resp) + } } return resp, err @@ -366,6 +402,9 @@ type serverStream struct { receivedMessageID int sentMessageID int + + receivedEvent bool + sentEvent bool } func (w *serverStream) Context() context.Context { @@ -377,7 +416,9 @@ func (w *serverStream) RecvMsg(m interface{}) error { if err == nil { w.receivedMessageID++ - messageReceived.Event(w.Context(), w.receivedMessageID, m) + if w.receivedEvent { + messageReceived.Event(w.Context(), w.receivedMessageID, m) + } } return err @@ -387,15 +428,19 @@ func (w *serverStream) SendMsg(m interface{}) error { err := w.ServerStream.SendMsg(m) w.sentMessageID++ - messageSent.Event(w.Context(), w.sentMessageID, m) + if w.sentEvent { + messageSent.Event(w.Context(), w.sentMessageID, m) + } return err } -func wrapServerStream(ctx context.Context, ss grpc.ServerStream) *serverStream { +func wrapServerStream(ctx context.Context, ss grpc.ServerStream, cfg *config) *serverStream { return &serverStream{ - ServerStream: ss, - ctx: ctx, + ServerStream: ss, + ctx: ctx, + receivedEvent: cfg.ReceivedEvent, + sentEvent: cfg.SentEvent, } } @@ -420,21 +465,26 @@ func StreamServerInterceptor(opts ...Option) grpc.StreamServerInterceptor { Type: StreamServer, } if cfg.Filter != nil && !cfg.Filter(i) { - return handler(srv, wrapServerStream(ctx, ss)) + return handler(srv, wrapServerStream(ctx, ss, cfg)) } ctx = extract(ctx, cfg.Propagators) - name, attr := spanInfo(info.FullMethod, peerFromCtx(ctx)) + + startOpts := append([]trace.SpanStartOption{ + trace.WithSpanKind(trace.SpanKindServer), + trace.WithAttributes(attr...)}, + cfg.SpanStartOptions..., + ) + ctx, span := tracer.Start( trace.ContextWithRemoteSpanContext(ctx, trace.SpanContextFromContext(ctx)), name, - trace.WithSpanKind(trace.SpanKindServer), - trace.WithAttributes(attr...), + startOpts..., ) defer span.End() - err := handler(srv, wrapServerStream(ctx, ss)) + err := handler(srv, wrapServerStream(ctx, ss, cfg)) if err != nil { s, _ := status.FromError(err) statusCode, msg := serverStatus(s) @@ -451,10 +501,13 @@ func StreamServerInterceptor(opts ...Option) grpc.StreamServerInterceptor { // spanInfo returns a span name and all appropriate attributes from the gRPC // method and peer address. func spanInfo(fullMethod, peerAddress string) (string, []attribute.KeyValue) { - attrs := []attribute.KeyValue{RPCSystemGRPC} name, mAttrs := internal.ParseFullMethod(fullMethod) + peerAttrs := peerAttr(peerAddress) + + attrs := make([]attribute.KeyValue, 0, 1+len(mAttrs)+len(peerAttrs)) + attrs = append(attrs, RPCSystemGRPC) attrs = append(attrs, mAttrs...) - attrs = append(attrs, peerAttr(peerAddress)...) + attrs = append(attrs, peerAttrs...) return name, attrs } @@ -462,7 +515,7 @@ func spanInfo(fullMethod, peerAddress string) (string, []attribute.KeyValue) { func peerAttr(addr string) []attribute.KeyValue { host, p, err := net.SplitHostPort(addr) if err != nil { - return []attribute.KeyValue(nil) + return nil } if host == "" { @@ -470,7 +523,7 @@ func peerAttr(addr string) []attribute.KeyValue { } port, err := strconv.Atoi(p) if err != nil { - return []attribute.KeyValue(nil) + return nil } var attr []attribute.KeyValue diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/internal/parse.go b/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/internal/parse.go index ae160d5875..cf32a9e978 100644 --- a/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/internal/parse.go +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/internal/parse.go @@ -24,13 +24,21 @@ import ( // ParseFullMethod returns a span name following the OpenTelemetry semantic // conventions as well as all applicable span attribute.KeyValue attributes based // on a gRPC's FullMethod. +// +// Parsing is consistent with grpc-go implementation: +// https://github.com/grpc/grpc-go/blob/v1.57.0/internal/grpcutil/method.go#L26-L39 func ParseFullMethod(fullMethod string) (string, []attribute.KeyValue) { - name := strings.TrimLeft(fullMethod, "/") - service, method, found := strings.Cut(name, "/") - if !found { + if !strings.HasPrefix(fullMethod, "/") { + // Invalid format, does not follow `/package.service/method`. + return fullMethod, nil + } + name := fullMethod[1:] + pos := strings.LastIndex(name, "/") + if pos < 0 { // Invalid format, does not follow `/package.service/method`. return name, nil } + service, method := name[:pos], name[pos+1:] var attrs []attribute.KeyValue if service != "" { diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/stats_handler.go b/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/stats_handler.go new file mode 100644 index 0000000000..c64a53443b --- /dev/null +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/stats_handler.go @@ -0,0 +1,187 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package otelgrpc // import "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + +import ( + "context" + "sync/atomic" + + grpc_codes "google.golang.org/grpc/codes" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" + + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/internal" + "go.opentelemetry.io/otel/codes" + semconv "go.opentelemetry.io/otel/semconv/v1.17.0" + "go.opentelemetry.io/otel/trace" +) + +type gRPCContextKey struct{} + +type gRPCContext struct { + messagesReceived int64 + messagesSent int64 +} + +// NewServerHandler creates a stats.Handler for gRPC server. +func NewServerHandler(opts ...Option) stats.Handler { + h := &serverHandler{ + config: newConfig(opts), + } + + h.tracer = h.config.TracerProvider.Tracer( + instrumentationName, + trace.WithInstrumentationVersion(SemVersion()), + ) + return h +} + +type serverHandler struct { + *config + tracer trace.Tracer +} + +// TagRPC can attach some information to the given context. +func (h *serverHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { + ctx = extract(ctx, h.config.Propagators) + + name, attrs := internal.ParseFullMethod(info.FullMethodName) + attrs = append(attrs, RPCSystemGRPC) + ctx, _ = h.tracer.Start( + trace.ContextWithRemoteSpanContext(ctx, trace.SpanContextFromContext(ctx)), + name, + trace.WithSpanKind(trace.SpanKindServer), + trace.WithAttributes(attrs...), + ) + + gctx := gRPCContext{} + return context.WithValue(ctx, gRPCContextKey{}, &gctx) +} + +// HandleRPC processes the RPC stats. +func (h *serverHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) { + handleRPC(ctx, rs) +} + +// TagConn can attach some information to the given context. +func (h *serverHandler) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context { + span := trace.SpanFromContext(ctx) + attrs := peerAttr(peerFromCtx(ctx)) + span.SetAttributes(attrs...) + return ctx +} + +// HandleConn processes the Conn stats. +func (h *serverHandler) HandleConn(ctx context.Context, info stats.ConnStats) { +} + +// NewClientHandler creates a stats.Handler for gRPC client. +func NewClientHandler(opts ...Option) stats.Handler { + h := &clientHandler{ + config: newConfig(opts), + } + + h.tracer = h.config.TracerProvider.Tracer( + instrumentationName, + trace.WithInstrumentationVersion(SemVersion()), + ) + + return h +} + +type clientHandler struct { + *config + tracer trace.Tracer +} + +// TagRPC can attach some information to the given context. +func (h *clientHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { + name, attrs := internal.ParseFullMethod(info.FullMethodName) + attrs = append(attrs, RPCSystemGRPC) + ctx, _ = h.tracer.Start( + ctx, + name, + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attrs...), + ) + + gctx := gRPCContext{} + + return inject(context.WithValue(ctx, gRPCContextKey{}, &gctx), h.config.Propagators) +} + +// HandleRPC processes the RPC stats. +func (h *clientHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) { + handleRPC(ctx, rs) +} + +// TagConn can attach some information to the given context. +func (h *clientHandler) TagConn(ctx context.Context, cti *stats.ConnTagInfo) context.Context { + span := trace.SpanFromContext(ctx) + attrs := peerAttr(cti.RemoteAddr.String()) + span.SetAttributes(attrs...) + return ctx +} + +// HandleConn processes the Conn stats. +func (h *clientHandler) HandleConn(context.Context, stats.ConnStats) { + // no-op +} + +func handleRPC(ctx context.Context, rs stats.RPCStats) { + span := trace.SpanFromContext(ctx) + gctx, _ := ctx.Value(gRPCContextKey{}).(*gRPCContext) + var messageId int64 + + switch rs := rs.(type) { + case *stats.Begin: + case *stats.InPayload: + if gctx != nil { + messageId = atomic.AddInt64(&gctx.messagesReceived, 1) + } + span.AddEvent("message", + trace.WithAttributes( + semconv.MessageTypeReceived, + semconv.MessageIDKey.Int64(messageId), + semconv.MessageCompressedSizeKey.Int(rs.CompressedLength), + semconv.MessageUncompressedSizeKey.Int(rs.Length), + ), + ) + case *stats.OutPayload: + if gctx != nil { + messageId = atomic.AddInt64(&gctx.messagesSent, 1) + } + + span.AddEvent("message", + trace.WithAttributes( + semconv.MessageTypeSent, + semconv.MessageIDKey.Int64(messageId), + semconv.MessageCompressedSizeKey.Int(rs.CompressedLength), + semconv.MessageUncompressedSizeKey.Int(rs.Length), + ), + ) + case *stats.End: + if rs.Error != nil { + s, _ := status.FromError(rs.Error) + span.SetStatus(codes.Error, s.Message()) + span.SetAttributes(statusCodeAttr(s.Code())) + } else { + span.SetAttributes(statusCodeAttr(grpc_codes.OK)) + } + span.End() + default: + return + } +} diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/version.go b/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/version.go index 1fc5e3365d..7a8ecebf04 100644 --- a/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/version.go +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/version.go @@ -16,7 +16,7 @@ package otelgrpc // import "go.opentelemetry.io/contrib/instrumentation/google.g // Version is the current release version of the gRPC instrumentation. func Version() string { - return "0.42.0" + return "0.45.0" // This string is updated by the pre_release.sh script during release } diff --git a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/version.go b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/version.go index 8f3f53a958..6eace875cf 100644 --- a/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/version.go +++ b/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/version.go @@ -16,7 +16,7 @@ package otelhttp // import "go.opentelemetry.io/contrib/instrumentation/net/http // Version is the current release version of the otelhttp instrumentation. func Version() string { - return "0.44.0" + return "0.45.0" // This string is updated by the pre_release.sh script during release } diff --git a/vendor/modules.txt b/vendor/modules.txt index 628a4337de..1709a50230 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -12,10 +12,6 @@ github.com/Microsoft/go-winio/pkg/guid # github.com/NYTimes/gziphandler v1.1.1 ## explicit; go 1.11 github.com/NYTimes/gziphandler -# github.com/RoaringBitmap/roaring v1.9.4 -## explicit; go 1.14 -github.com/RoaringBitmap/roaring -github.com/RoaringBitmap/roaring/internal # github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df ## explicit; go 1.18 github.com/antlr/antlr4/runtime/Go/antlr/v4 @@ -28,9 +24,6 @@ github.com/avast/retry-go # github.com/beorn7/perks v1.0.1 ## explicit; go 1.11 github.com/beorn7/perks/quantile -# github.com/bits-and-blooms/bitset v1.12.0 -## explicit; go 1.16 -github.com/bits-and-blooms/bitset # github.com/blang/semver/v4 v4.0.0 ## explicit; go 1.14 github.com/blang/semver/v4 @@ -223,8 +216,8 @@ github.com/kubernetes-csi/csi-lib-utils/protosanitizer # github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de ## explicit github.com/liggitt/tabwriter -# github.com/longhorn/backing-image-manager v1.7.0-rc1 -## explicit; go 1.22.0 +# github.com/longhorn/backing-image-manager v1.7.0-dev.0.20240823042906-1ae3d5073f60 +## explicit; go 1.22.2 github.com/longhorn/backing-image-manager/api github.com/longhorn/backing-image-manager/pkg/client github.com/longhorn/backing-image-manager/pkg/meta @@ -262,7 +255,7 @@ github.com/longhorn/go-iscsi-helper/util # github.com/longhorn/go-spdk-helper v0.0.0-20240820144231-33c0873802ff ## explicit; go 1.22.0 github.com/longhorn/go-spdk-helper/pkg/types -# github.com/longhorn/longhorn-engine v1.7.0-rc3 +# github.com/longhorn/longhorn-engine v1.7.0-dev.0.20240824053610-9d2b194f765f ## explicit; go 1.22.2 github.com/longhorn/longhorn-engine/pkg/interceptor github.com/longhorn/longhorn-engine/pkg/meta @@ -314,9 +307,6 @@ github.com/modern-go/concurrent # github.com/modern-go/reflect2 v1.0.2 ## explicit; go 1.12 github.com/modern-go/reflect2 -# github.com/mschoch/smat v0.2.0 -## explicit; go 1.13 -github.com/mschoch/smat # github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 ## explicit github.com/munnerz/goautoneg @@ -486,11 +476,11 @@ go.etcd.io/etcd/client/v3 go.etcd.io/etcd/client/v3/credentials go.etcd.io/etcd/client/v3/internal/endpoint go.etcd.io/etcd/client/v3/internal/resolver -# go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 +# go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 ## explicit; go 1.19 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/internal -# go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 +# go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 ## explicit; go 1.19 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/internal/semconvutil