From 85b09cb71db96bf78fe1e6b777454ef2a910a1d8 Mon Sep 17 00:00:00 2001 From: Chad Currie Date: Fri, 8 Dec 2023 00:59:41 +1300 Subject: [PATCH 1/3] Index float as single rather than a double --- src/Examine.Core/FieldDefinitionTypes.cs | 14 ++- src/Examine.Lucene/Indexing/FloatType.cs | 114 ++++++++++++++++++ .../ValueTypeFactoryCollection.cs | 8 +- 3 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 src/Examine.Lucene/Indexing/FloatType.cs diff --git a/src/Examine.Core/FieldDefinitionTypes.cs b/src/Examine.Core/FieldDefinitionTypes.cs index 5fe13cb3..3ed57c30 100644 --- a/src/Examine.Core/FieldDefinitionTypes.cs +++ b/src/Examine.Core/FieldDefinitionTypes.cs @@ -13,10 +13,22 @@ public static class FieldDefinitionTypes public const string Integer = "int"; /// - /// Will be indexed as a float + /// Will be indexed as a single /// + [Obsolete("To remove in Examine V5. In Examine V3, this was indexed as a Double field. Use FloatV2")] public const string Float = "float"; + /// + /// Will be indexed as a float + /// + public const string FloatV2 = "float"; + + /// + /// Will be indexed as a single + /// + [Obsolete("To remove in Examine V5. Use FloatV2")] + public const string Single = "single"; + /// /// Will be indexed as a double /// diff --git a/src/Examine.Lucene/Indexing/FloatType.cs b/src/Examine.Lucene/Indexing/FloatType.cs new file mode 100644 index 00000000..b7fd6a84 --- /dev/null +++ b/src/Examine.Lucene/Indexing/FloatType.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using Examine.Lucene.Providers; +using Examine.Lucene.Search; +using Examine.Search; +using Lucene.Net.Documents; +using Lucene.Net.Facet; +using Lucene.Net.Facet.SortedSet; +using Lucene.Net.Search; +using Microsoft.Extensions.Logging; +using static Lucene.Net.Queries.Function.ValueSources.MultiFunction; + +namespace Examine.Lucene.Indexing +{ + /// + /// Represents a float/single + /// + public class FloatType : IndexFieldRangeValueType, IIndexFacetValueType + { + private readonly bool _isFacetable; +#pragma warning disable IDE0032 // Use auto property + private readonly bool _taxonomyIndex; +#pragma warning restore IDE0032 // Use auto property + + /// + public FloatType(string fieldName, bool isFacetable, bool taxonomyIndex, ILoggerFactory logger, bool store) + : base(fieldName, logger, store) + { + _isFacetable = isFacetable; + _taxonomyIndex = taxonomyIndex; + } + + /// + [Obsolete("To be removed in Examine V5")] +#pragma warning disable RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads + public FloatType(string fieldName, ILoggerFactory logger, bool store = true) +#pragma warning restore RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads + : base(fieldName, logger, store) + { + _isFacetable = false; + } + + /// + /// Can be sorted by the normal field name + /// + public override string SortableFieldName => FieldName; + + /// + public bool IsTaxonomyFaceted => _taxonomyIndex; + + /// + public override void AddValue(Document doc, object? value) + { + // Support setting taxonomy path + if (_isFacetable && _taxonomyIndex && value is object[] objArr && objArr != null && objArr.Length == 2) + { + if (!TryConvert(objArr[0], out float parsedVal)) + { + return; + } + + if (!TryConvert(objArr[1], out string[]? parsedPathVal)) + { + return; + } + + doc.Add(new SingleField(FieldName, parsedVal, Store ? Field.Store.YES : Field.Store.NO)); + + doc.Add(new FacetField(FieldName, parsedPathVal)); + doc.Add(new SingleDocValuesField(FieldName, parsedVal)); + return; + } + base.AddValue(doc, value); + } + + /// + protected override void AddSingleValue(Document doc, object value) + { + if (!TryConvert(value, out float parsedVal)) + { + return; + } + + doc.Add(new SingleField(FieldName, parsedVal, Store ? Field.Store.YES : Field.Store.NO)); + + if (_isFacetable && _taxonomyIndex) + { + doc.Add(new FacetField(FieldName, parsedVal.ToString())); + doc.Add(new SingleDocValuesField(FieldName, parsedVal)); + } + else if (_isFacetable && !_taxonomyIndex) + { + doc.Add(new SortedSetDocValuesFacetField(FieldName, parsedVal.ToString())); + doc.Add(new SingleDocValuesField(FieldName, parsedVal)); + } + } + + /// + public override Query? GetQuery(string query) => !TryConvert(query, out float parsedVal) ? null : GetQuery(parsedVal, parsedVal); + + /// + public override Query GetQuery(float? lower, float? upper, bool lowerInclusive = true, bool upperInclusive = true) + { + return NumericRangeQuery.NewDoubleRange(FieldName, + lower ?? float.MinValue, + upper ?? float.MaxValue, lowerInclusive, upperInclusive); + } + + /// + public virtual IEnumerable> ExtractFacets(IFacetExtractionContext facetExtractionContext, IFacetField field) + => field.ExtractFacets(facetExtractionContext); + } + +} diff --git a/src/Examine.Lucene/ValueTypeFactoryCollection.cs b/src/Examine.Lucene/ValueTypeFactoryCollection.cs index dc8f955b..9952f890 100644 --- a/src/Examine.Lucene/ValueTypeFactoryCollection.cs +++ b/src/Examine.Lucene/ValueTypeFactoryCollection.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; +using System.Runtime.InteropServices; using Examine.Lucene.Analyzers; using Examine.Lucene.Indexing; using Lucene.Net.Analysis; @@ -75,7 +76,8 @@ private static IReadOnlyDictionary> G { {"number", name => new Int32Type(name, loggerFactory)}, {FieldDefinitionTypes.Integer, name => new Int32Type(name, loggerFactory)}, - {FieldDefinitionTypes.Float, name => new SingleType(name, loggerFactory)}, + {FieldDefinitionTypes.Float, name => new FloatType(name, loggerFactory)}, + {FieldDefinitionTypes.Single, name => new SingleType(name, loggerFactory)}, {FieldDefinitionTypes.Double, name => new DoubleType(name, loggerFactory)}, {FieldDefinitionTypes.Long, name => new Int64Type(name, loggerFactory)}, {"date", name => new DateTimeType(name, loggerFactory, DateResolution.MILLISECOND)}, @@ -91,7 +93,7 @@ private static IReadOnlyDictionary> G {FieldDefinitionTypes.InvariantCultureIgnoreCase, name => new GenericAnalyzerFieldValueType(name, loggerFactory, new CultureInvariantWhitespaceAnalyzer())}, {FieldDefinitionTypes.EmailAddress, name => new GenericAnalyzerFieldValueType(name, loggerFactory, new EmailAddressAnalyzer())}, {FieldDefinitionTypes.FacetInteger, name => new Int32Type(name, true,false,loggerFactory, true)}, - {FieldDefinitionTypes.FacetFloat, name => new SingleType(name, true, false, loggerFactory, true)}, + {FieldDefinitionTypes.FacetFloat, name => new FloatType(name, true, false, loggerFactory, true)}, {FieldDefinitionTypes.FacetDouble, name => new DoubleType(name,true, false, loggerFactory, true)}, {FieldDefinitionTypes.FacetLong, name => new Int64Type(name, true, false, loggerFactory, true)}, {FieldDefinitionTypes.FacetDateTime, name => new DateTimeType(name, true, true, false, loggerFactory, DateResolution.MILLISECOND)}, @@ -103,7 +105,7 @@ private static IReadOnlyDictionary> G {FieldDefinitionTypes.FacetFullText, name => new FullTextType(name, loggerFactory, true, false, false, defaultAnalyzer ?? new CultureInvariantStandardAnalyzer())}, {FieldDefinitionTypes.FacetFullTextSortable, name => new FullTextType(name, loggerFactory, true, false,true, defaultAnalyzer ?? new CultureInvariantStandardAnalyzer())}, {FieldDefinitionTypes.FacetTaxonomyInteger, name => new Int32Type(name,true,true, loggerFactory, true)}, - {FieldDefinitionTypes.FacetTaxonomyFloat, name => new SingleType(name,isFacetable: true, taxonomyIndex: true, loggerFactory, true)}, + {FieldDefinitionTypes.FacetTaxonomyFloat, name => new FloatType(name,isFacetable: true, taxonomyIndex: true, loggerFactory, true)}, {FieldDefinitionTypes.FacetTaxonomyDouble, name => new DoubleType(name, true, true, loggerFactory, true)}, {FieldDefinitionTypes.FacetTaxonomyLong, name => new Int64Type(name, isFacetable: true, taxonomyIndex: true, loggerFactory, true)}, {FieldDefinitionTypes.FacetTaxonomyDateTime, name => new DateTimeType(name,true, true, taxonomyIndex : true, loggerFactory, DateResolution.MILLISECOND)}, From dd7063608748b66729235f60219195dd733a6d2a Mon Sep 17 00:00:00 2001 From: Chad Currie Date: Fri, 8 Dec 2023 01:02:26 +1300 Subject: [PATCH 2/3] float --- src/Examine.Core/FieldDefinitionTypes.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Examine.Core/FieldDefinitionTypes.cs b/src/Examine.Core/FieldDefinitionTypes.cs index 3ed57c30..a6f00b27 100644 --- a/src/Examine.Core/FieldDefinitionTypes.cs +++ b/src/Examine.Core/FieldDefinitionTypes.cs @@ -15,18 +15,12 @@ public static class FieldDefinitionTypes /// /// Will be indexed as a single /// - [Obsolete("To remove in Examine V5. In Examine V3, this was indexed as a Double field. Use FloatV2")] public const string Float = "float"; - /// - /// Will be indexed as a float - /// - public const string FloatV2 = "float"; - /// /// Will be indexed as a single /// - [Obsolete("To remove in Examine V5. Use FloatV2")] + [Obsolete("To remove in Examine V5. Use Float")] public const string Single = "single"; /// From f3420f6e432a770e6fb2056adcda8096a29a9cfe Mon Sep 17 00:00:00 2001 From: Chad Currie Date: Fri, 8 Dec 2023 01:14:45 +1300 Subject: [PATCH 3/3] Single range --- src/Examine.Lucene/Indexing/FloatType.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Examine.Lucene/Indexing/FloatType.cs b/src/Examine.Lucene/Indexing/FloatType.cs index b7fd6a84..079f8e33 100644 --- a/src/Examine.Lucene/Indexing/FloatType.cs +++ b/src/Examine.Lucene/Indexing/FloatType.cs @@ -101,7 +101,7 @@ protected override void AddSingleValue(Document doc, object value) /// public override Query GetQuery(float? lower, float? upper, bool lowerInclusive = true, bool upperInclusive = true) { - return NumericRangeQuery.NewDoubleRange(FieldName, + return NumericRangeQuery.NewSingleRange(FieldName, lower ?? float.MinValue, upper ?? float.MaxValue, lowerInclusive, upperInclusive); }