From a367eabaa1d60490508e760fb1dfebe2f82921ce Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 26 Jun 2024 13:58:25 -0600 Subject: [PATCH 01/16] Fixing SyncedFileSystemDirectoryFactory when the main index is corrupted. --- .../SyncedFileSystemDirectoryFactory.cs | 68 ++++++++++++++++--- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs index e59a3586..242f58dd 100644 --- a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs @@ -24,6 +24,7 @@ public class SyncedFileSystemDirectoryFactory : FileSystemDirectoryFactory { private readonly DirectoryInfo _localDir; private readonly ILoggerFactory _loggerFactory; + private readonly ILogger _logger; private ExamineReplicator _replicator; public SyncedFileSystemDirectoryFactory( @@ -35,6 +36,7 @@ public SyncedFileSystemDirectoryFactory( { _localDir = localDir; _loggerFactory = loggerFactory; + _logger = _loggerFactory.CreateLogger(); } protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock) @@ -42,24 +44,55 @@ protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool force var path = Path.Combine(_localDir.FullName, luceneIndex.Name); var localLuceneIndexFolder = new DirectoryInfo(path); - Directory mainDir = base.CreateDirectory(luceneIndex, forceUnlock); + var mainDir = base.CreateDirectory(luceneIndex, forceUnlock); + + var checker = new CheckIndex(mainDir); + // TODO: We can redirect the logging output + // checker.InfoStream = + var status = checker.DoCheckIndex(); + + if (!status.Clean) + { + _logger.LogInformation("Checked main director index and it is not clean, attempting to fix {IndexName}...", luceneIndex.Name); + + try + { + checker.FixIndex(status); + _logger.LogInformation("Index {IndexName} fixed.", luceneIndex.Name); + } + catch (Exception ex) + { + _logger.LogError(ex, "{IndexName} index could not be fixed, it will be deleted.", luceneIndex.Name); + } + } + else + { + _logger.LogInformation("Checked main director index {IndexName} and it is clean.", luceneIndex.Name); + } // used by the replicator, will be a short lived directory for each synced revision and deleted when finished. var tempDir = new DirectoryInfo(Path.Combine(_localDir.FullName, "Rep", Guid.NewGuid().ToString("N"))); if (DirectoryReader.IndexExists(mainDir)) { + IndexWriter indexWriter; + // when the lucene directory is going to be created, we'll sync from main storage to local // storage before any index/writer is opened. - using (var tempMainIndexWriter = new IndexWriter( - mainDir, - new IndexWriterConfig( - LuceneInfo.CurrentVersion, - new StandardAnalyzer(LuceneInfo.CurrentVersion)) - { - OpenMode = OpenMode.APPEND, - IndexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()) - })) + + try + { + indexWriter = GetIndexWriter(mainDir, OpenMode.APPEND); + } + catch (Exception ex) + { + // Index is corrupted, typically this will be FileNotFoundException + _logger.LogError(ex, "{IndexName} index is corrupt, a new one will be created", luceneIndex.Name); + + indexWriter = GetIndexWriter(mainDir, OpenMode.CREATE); + } + + using (var tempMainIndexWriter = indexWriter) using (var tempMainIndex = new LuceneIndex(_loggerFactory, luceneIndex.Name, new TempOptions(), tempMainIndexWriter)) using (var tempLocalDirectory = FSDirectory.Open(localLuceneIndexFolder, LockFactory.GetLockFactory(localLuceneIndexFolder))) using (var replicator = new ExamineReplicator(_loggerFactory, tempMainIndex, tempLocalDirectory, tempDir)) @@ -100,6 +133,21 @@ protected override void Dispose(bool disposing) } } + private IndexWriter GetIndexWriter(Directory mainDir, OpenMode openMode) + { + var indexWriter = new IndexWriter( + mainDir, + new IndexWriterConfig( + LuceneInfo.CurrentVersion, + new StandardAnalyzer(LuceneInfo.CurrentVersion)) + { + OpenMode = openMode, + IndexDeletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()) + }); + + return indexWriter; + } + private class TempOptions : IOptionsMonitor { public LuceneDirectoryIndexOptions CurrentValue => new LuceneDirectoryIndexOptions(); From c15b7c8f679de9bf65d6e59f3fbcf076d5b63e0d Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 26 Jun 2024 14:11:55 -0600 Subject: [PATCH 02/16] Adds more checks and logging --- .../SyncedFileSystemDirectoryFactory.cs | 54 +++++++++++++------ 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs index 242f58dd..d50afbc1 100644 --- a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs @@ -1,7 +1,6 @@ using System; using System.IO; -using System.Threading; using Examine.Lucene.Providers; using Lucene.Net.Analysis.Standard; using Lucene.Net.Index; @@ -46,28 +45,49 @@ protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool force var mainDir = base.CreateDirectory(luceneIndex, forceUnlock); - var checker = new CheckIndex(mainDir); - // TODO: We can redirect the logging output - // checker.InfoStream = - var status = checker.DoCheckIndex(); - - if (!status.Clean) + using (var writer = new StringWriter()) { - _logger.LogInformation("Checked main director index and it is not clean, attempting to fix {IndexName}...", luceneIndex.Name); + var checker = new CheckIndex(mainDir) + { + // Redirect the logging output of the checker + InfoStream = writer + }; - try + var status = checker.DoCheckIndex(); + writer.Flush(); + + _logger.LogDebug("{IndexName} health check report {IndexReport}", luceneIndex.Name, writer.ToString()); + + if (status.MissingSegments) { - checker.FixIndex(status); - _logger.LogInformation("Index {IndexName} fixed.", luceneIndex.Name); + _logger.LogError("{IndexName} index is missing segments, it will be deleted.", luceneIndex.Name); } - catch (Exception ex) + else if (!status.Clean) { - _logger.LogError(ex, "{IndexName} index could not be fixed, it will be deleted.", luceneIndex.Name); + _logger.LogWarning("Checked main director index and it is not clean, attempting to fix {IndexName}. {DocumentsLost} documents will be lost.", luceneIndex.Name, status.TotLoseDocCount); + + try + { + checker.FixIndex(status); + + if (!status.Clean) + { + _logger.LogError("{IndexName} index could not be fixed, it will be deleted.", luceneIndex.Name); + } + else + { + _logger.LogInformation("Index {IndexName} fixed. {DocumentsLost} documents were lost.", luceneIndex.Name, status.TotLoseDocCount); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "{IndexName} index could not be fixed, it will be deleted.", luceneIndex.Name); + } + } + else + { + _logger.LogInformation("Checked main director index {IndexName} and it is clean.", luceneIndex.Name); } - } - else - { - _logger.LogInformation("Checked main director index {IndexName} and it is clean.", luceneIndex.Name); } // used by the replicator, will be a short lived directory for each synced revision and deleted when finished. From 41c22210a903f5d7703eb2612bfd5c5a30c42a30 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 26 Jun 2024 17:01:29 -0600 Subject: [PATCH 03/16] Adds test --- .../SyncedFileSystemDirectoryFactory.cs | 35 ++++++++- .../SyncedFileSystemDirectoryFactoryTests.cs | 78 +++++++++++++++++++ .../Examine.Lucene/ExamineReplicatorTests.cs | 2 +- src/Examine.Test/ExamineBaseTest.cs | 12 +-- 4 files changed, 118 insertions(+), 9 deletions(-) create mode 100644 src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs diff --git a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs index d50afbc1..8cf065f7 100644 --- a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs @@ -38,8 +38,10 @@ public SyncedFileSystemDirectoryFactory( _logger = _loggerFactory.CreateLogger(); } - protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock) + internal CreateResult TryCreateDirectory(LuceneIndex luceneIndex, bool forceUnlock, out Directory directory) { + var result = CreateResult.OpenedSuccessfully; + var path = Path.Combine(_localDir.FullName, luceneIndex.Name); var localLuceneIndexFolder = new DirectoryInfo(path); @@ -61,27 +63,33 @@ protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool force if (status.MissingSegments) { _logger.LogError("{IndexName} index is missing segments, it will be deleted.", luceneIndex.Name); + result = CreateResult.MissingSegments; } else if (!status.Clean) { _logger.LogWarning("Checked main director index and it is not clean, attempting to fix {IndexName}. {DocumentsLost} documents will be lost.", luceneIndex.Name, status.TotLoseDocCount); + result = CreateResult.NotClean; try { checker.FixIndex(status); + status = checker.DoCheckIndex(); if (!status.Clean) { _logger.LogError("{IndexName} index could not be fixed, it will be deleted.", luceneIndex.Name); + result |= CreateResult.NotFixed; } else { _logger.LogInformation("Index {IndexName} fixed. {DocumentsLost} documents were lost.", luceneIndex.Name, status.TotLoseDocCount); + result |= CreateResult.Fixed; } } catch (Exception ex) { _logger.LogError(ex, "{IndexName} index could not be fixed, it will be deleted.", luceneIndex.Name); + result |= CreateResult.ExceptionNotFixed; } } else @@ -103,6 +111,7 @@ protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool force try { indexWriter = GetIndexWriter(mainDir, OpenMode.APPEND); + result |= CreateResult.OpenedSuccessfully; } catch (Exception ex) { @@ -110,6 +119,7 @@ protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool force _logger.LogError(ex, "{IndexName} index is corrupt, a new one will be created", luceneIndex.Name); indexWriter = GetIndexWriter(mainDir, OpenMode.CREATE); + result |= CreateResult.CorruptCreatedNew; } using (var tempMainIndexWriter = indexWriter) @@ -141,7 +151,28 @@ protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool force // Start replicating back to main _replicator.StartIndexReplicationOnSchedule(1000); - return localLuceneDir; + directory = localLuceneDir; + + return result; + } + + [Flags] + internal enum CreateResult + { + Init = 0, + MissingSegments = 1, + NotClean = 2, + Fixed = 4, + NotFixed = 8, + ExceptionNotFixed = 16, + CorruptCreatedNew = 32, + OpenedSuccessfully = 64 + } + + protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock) + { + _ = TryCreateDirectory(luceneIndex, forceUnlock, out var directory); + return directory; } protected override void Dispose(bool disposing) diff --git a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs new file mode 100644 index 00000000..9a5ed639 --- /dev/null +++ b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs @@ -0,0 +1,78 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Examine.Lucene; +using Examine.Lucene.Analyzers; +using Examine.Lucene.Directories; +using Examine.Lucene.Providers; +using Lucene.Net.Index; +using Lucene.Net.Store; +using Microsoft.Extensions.Options; +using Moq; +using NUnit.Framework; + +namespace Examine.Test.Examine.Lucene.Directories +{ + [TestFixture] + public class SyncedFileSystemDirectoryFactoryTests : ExamineBaseTest + { + [Test] + public void Given_ExistingCorruptIndex_When_CreatingDirectory_Then_IndexFixed() + { + var mainPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + + try + { + using (var mainDir = FSDirectory.Open(Path.Combine(mainPath, TestIndex.TestIndexName))) + { + using (var writer = new IndexWriter(mainDir, new IndexWriterConfig(LuceneInfo.CurrentVersion, new CultureInvariantStandardAnalyzer()))) + using (var indexer = GetTestIndex(writer)) + { + using (indexer.WithThreadingMode(IndexThreadingMode.Synchronous)) + { + indexer.IndexItem(new ValueSet(1.ToString(), "content", + new Dictionary> + { + {"item1", new List(new[] {"value1"})}, + {"item2", new List(new[] {"value2"})} + })); + } + } + + Assert.IsTrue(DirectoryReader.IndexExists(mainDir)); + + // Get an index (non segments file) and delete it (corrupt index) + var indexFile = mainDir.Directory.GetFiles().Where(x => !x.Name.Contains("segment", StringComparison.OrdinalIgnoreCase)).First(); + File.Delete(indexFile.FullName); + } + + using var syncedDirFactory = new SyncedFileSystemDirectoryFactory( + new DirectoryInfo(tempPath), + new DirectoryInfo(mainPath), + new DefaultLockFactory(), + LoggerFactory); + + using var index = new LuceneIndex( + LoggerFactory, + TestIndex.TestIndexName, + Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions + { + DirectoryFactory = syncedDirFactory, + })); + + var result = syncedDirFactory.TryCreateDirectory(index, false, out var dir); + + Assert.IsTrue(result.HasFlag(SyncedFileSystemDirectoryFactory.CreateResult.NotClean)); + Assert.IsTrue(result.HasFlag(SyncedFileSystemDirectoryFactory.CreateResult.Fixed)); + Assert.IsTrue(result.HasFlag(SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully)); + } + finally + { + System.IO.Directory.Delete(mainPath, true); + System.IO.Directory.Delete(tempPath, true); + } + } + } +} diff --git a/src/Examine.Test/Examine.Lucene/ExamineReplicatorTests.cs b/src/Examine.Test/Examine.Lucene/ExamineReplicatorTests.cs index 4852bda2..00845ee0 100644 --- a/src/Examine.Test/Examine.Lucene/ExamineReplicatorTests.cs +++ b/src/Examine.Test/Examine.Lucene/ExamineReplicatorTests.cs @@ -13,7 +13,7 @@ namespace Examine.Test.Examine.Lucene.Sync public class ExamineReplicatorTests : ExamineBaseTest { private ILoggerFactory GetLoggerFactory() - => LoggerFactory.Create(x => x.AddConsole().SetMinimumLevel(LogLevel.Debug)); + => Microsoft.Extensions.Logging.LoggerFactory.Create(x => x.AddConsole().SetMinimumLevel(LogLevel.Debug)); [Test] public void GivenAMainIndex_WhenReplicatedLocally_TheLocalIndexIsPopulated() diff --git a/src/Examine.Test/ExamineBaseTest.cs b/src/Examine.Test/ExamineBaseTest.cs index 2d7b7261..baf341f3 100644 --- a/src/Examine.Test/ExamineBaseTest.cs +++ b/src/Examine.Test/ExamineBaseTest.cs @@ -13,21 +13,21 @@ namespace Examine.Test { public abstract class ExamineBaseTest { - private ILoggerFactory _loggerFactory; + protected ILoggerFactory LoggerFactory { get; private set; } [SetUp] public virtual void Setup() { - _loggerFactory = LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug)); - _loggerFactory.CreateLogger(typeof(ExamineBaseTest)).LogDebug("Initializing test"); + LoggerFactory = Microsoft.Extensions.Logging.LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug)); + LoggerFactory.CreateLogger(typeof(ExamineBaseTest)).LogDebug("Initializing test"); } [TearDown] - public virtual void TearDown() => _loggerFactory.Dispose(); + public virtual void TearDown() => LoggerFactory.Dispose(); public TestIndex GetTestIndex(Directory d, Analyzer analyzer, FieldDefinitionCollection fieldDefinitions = null, IndexDeletionPolicy indexDeletionPolicy = null, IReadOnlyDictionary indexValueTypesFactory = null) => new TestIndex( - _loggerFactory, + LoggerFactory, Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions { FieldDefinitions = fieldDefinitions, @@ -39,7 +39,7 @@ public TestIndex GetTestIndex(Directory d, Analyzer analyzer, FieldDefinitionCol public TestIndex GetTestIndex(IndexWriter writer) => new TestIndex( - _loggerFactory, + LoggerFactory, Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneIndexOptions()), writer); } From 2a3bcd2781260e6f21f50732f8bfc3d36d8ca52d Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 26 Jun 2024 17:30:05 -0600 Subject: [PATCH 04/16] Adds test case --- .../SyncedFileSystemDirectoryFactoryTests.cs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs index 9a5ed639..9baefcd7 100644 --- a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs +++ b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs @@ -17,8 +17,12 @@ namespace Examine.Test.Examine.Lucene.Directories [TestFixture] public class SyncedFileSystemDirectoryFactoryTests : ExamineBaseTest { + [TestCase(false, SyncedFileSystemDirectoryFactory.CreateResult.NotClean | SyncedFileSystemDirectoryFactory.CreateResult.Fixed | SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully)] + [TestCase(true, SyncedFileSystemDirectoryFactory.CreateResult.MissingSegments | SyncedFileSystemDirectoryFactory.CreateResult.CorruptCreatedNew)] [Test] - public void Given_ExistingCorruptIndex_When_CreatingDirectory_Then_IndexFixed() + public void Given_ExistingCorruptIndex_When_CreatingDirectory_Then_IndexCreatedOrOpened( + bool removeSegments, + Enum expected) { var mainPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); @@ -44,7 +48,12 @@ public void Given_ExistingCorruptIndex_When_CreatingDirectory_Then_IndexFixed() Assert.IsTrue(DirectoryReader.IndexExists(mainDir)); // Get an index (non segments file) and delete it (corrupt index) - var indexFile = mainDir.Directory.GetFiles().Where(x => !x.Name.Contains("segment", StringComparison.OrdinalIgnoreCase)).First(); + var indexFile = mainDir.Directory.GetFiles() + .Where(x => removeSegments + ? x.Name.Contains("segments_", StringComparison.OrdinalIgnoreCase) + : !x.Name.Contains("segments", StringComparison.OrdinalIgnoreCase)) + .First(); + File.Delete(indexFile.FullName); } @@ -64,9 +73,7 @@ public void Given_ExistingCorruptIndex_When_CreatingDirectory_Then_IndexFixed() var result = syncedDirFactory.TryCreateDirectory(index, false, out var dir); - Assert.IsTrue(result.HasFlag(SyncedFileSystemDirectoryFactory.CreateResult.NotClean)); - Assert.IsTrue(result.HasFlag(SyncedFileSystemDirectoryFactory.CreateResult.Fixed)); - Assert.IsTrue(result.HasFlag(SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully)); + Assert.IsTrue(result.HasFlag(expected)); } finally { From 162c46862df7eb0214301aa347bc4c326f11d8c4 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 26 Jun 2024 17:32:44 -0600 Subject: [PATCH 05/16] missed file --- .../SyncedFileSystemDirectoryFactory.cs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs index 8cf065f7..82f05a43 100644 --- a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs @@ -110,8 +110,20 @@ internal CreateResult TryCreateDirectory(LuceneIndex luceneIndex, bool forceUnlo try { - indexWriter = GetIndexWriter(mainDir, OpenMode.APPEND); - result |= CreateResult.OpenedSuccessfully; + var openMode = result == CreateResult.Init || result.HasFlag(CreateResult.Fixed) + ? OpenMode.APPEND + : OpenMode.CREATE; + + indexWriter = GetIndexWriter(mainDir, openMode); + + if (openMode == OpenMode.APPEND) + { + result |= CreateResult.OpenedSuccessfully; + } + else + { + result |= CreateResult.CorruptCreatedNew; + } } catch (Exception ex) { From 190b08ece34dc3eda9d200687fa2da0c1edc5c91 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 26 Jun 2024 17:35:34 -0600 Subject: [PATCH 06/16] Another test case --- .../SyncedFileSystemDirectoryFactoryTests.cs | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs index 9baefcd7..443e3f89 100644 --- a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs +++ b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs @@ -17,10 +17,12 @@ namespace Examine.Test.Examine.Lucene.Directories [TestFixture] public class SyncedFileSystemDirectoryFactoryTests : ExamineBaseTest { - [TestCase(false, SyncedFileSystemDirectoryFactory.CreateResult.NotClean | SyncedFileSystemDirectoryFactory.CreateResult.Fixed | SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully)] - [TestCase(true, SyncedFileSystemDirectoryFactory.CreateResult.MissingSegments | SyncedFileSystemDirectoryFactory.CreateResult.CorruptCreatedNew)] + [TestCase(true, false, SyncedFileSystemDirectoryFactory.CreateResult.NotClean | SyncedFileSystemDirectoryFactory.CreateResult.Fixed | SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully)] + [TestCase(true, true, SyncedFileSystemDirectoryFactory.CreateResult.MissingSegments | SyncedFileSystemDirectoryFactory.CreateResult.CorruptCreatedNew)] + [TestCase(false, false, SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully)] [Test] public void Given_ExistingCorruptIndex_When_CreatingDirectory_Then_IndexCreatedOrOpened( + bool corruptIndex, bool removeSegments, Enum expected) { @@ -47,14 +49,17 @@ public void Given_ExistingCorruptIndex_When_CreatingDirectory_Then_IndexCreatedO Assert.IsTrue(DirectoryReader.IndexExists(mainDir)); - // Get an index (non segments file) and delete it (corrupt index) - var indexFile = mainDir.Directory.GetFiles() - .Where(x => removeSegments - ? x.Name.Contains("segments_", StringComparison.OrdinalIgnoreCase) - : !x.Name.Contains("segments", StringComparison.OrdinalIgnoreCase)) - .First(); + if (corruptIndex) + { + // Get an index (non segments file) and delete it (corrupt index) + var indexFile = mainDir.Directory.GetFiles() + .Where(x => removeSegments + ? x.Name.Contains("segments_", StringComparison.OrdinalIgnoreCase) + : !x.Name.Contains("segments", StringComparison.OrdinalIgnoreCase)) + .First(); - File.Delete(indexFile.FullName); + File.Delete(indexFile.FullName); + } } using var syncedDirFactory = new SyncedFileSystemDirectoryFactory( From 75f5a70aa6e1491801d861f629c5094bad4061f2 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 27 Jun 2024 12:02:13 -0600 Subject: [PATCH 07/16] Adds functionality to sync from local to main if main is corrupt with tests --- .../SyncedFileSystemDirectoryFactory.cs | 249 +++++++++++------- src/Examine.Lucene/ExamineReplicator.cs | 11 + .../SyncedFileSystemDirectoryFactoryTests.cs | 129 ++++++--- 3 files changed, 271 insertions(+), 118 deletions(-) diff --git a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs index 82f05a43..4bf1fd20 100644 --- a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs @@ -22,6 +22,7 @@ namespace Examine.Lucene.Directories public class SyncedFileSystemDirectoryFactory : FileSystemDirectoryFactory { private readonly DirectoryInfo _localDir; + private readonly DirectoryInfo _mainDir; private readonly ILoggerFactory _loggerFactory; private readonly ILogger _logger; private ExamineReplicator _replicator; @@ -34,126 +35,78 @@ public SyncedFileSystemDirectoryFactory( : base(mainDir, lockFactory) { _localDir = localDir; + _mainDir = mainDir; _loggerFactory = loggerFactory; _logger = _loggerFactory.CreateLogger(); } internal CreateResult TryCreateDirectory(LuceneIndex luceneIndex, bool forceUnlock, out Directory directory) { - var result = CreateResult.OpenedSuccessfully; + var mainPath = Path.Combine(_mainDir.FullName, luceneIndex.Name); + var mainLuceneIndexFolder = new DirectoryInfo(mainPath); - var path = Path.Combine(_localDir.FullName, luceneIndex.Name); - var localLuceneIndexFolder = new DirectoryInfo(path); + var localPath = Path.Combine(_localDir.FullName, luceneIndex.Name); + var localLuceneIndexFolder = new DirectoryInfo(localPath); - var mainDir = base.CreateDirectory(luceneIndex, forceUnlock); + // used by the replicator, will be a short lived directory for each synced revision and deleted when finished. + var tempDir = new DirectoryInfo(Path.Combine(_localDir.FullName, "Rep", Guid.NewGuid().ToString("N"))); - using (var writer = new StringWriter()) - { - var checker = new CheckIndex(mainDir) - { - // Redirect the logging output of the checker - InfoStream = writer - }; + var mainLuceneDir = base.CreateDirectory(luceneIndex, forceUnlock); + var localLuceneDir = FSDirectory.Open( + localLuceneIndexFolder, + LockFactory.GetLockFactory(localLuceneIndexFolder)); - var status = checker.DoCheckIndex(); - writer.Flush(); + var mainIndexExists = DirectoryReader.IndexExists(mainLuceneDir); + var localIndexExists = DirectoryReader.IndexExists(localLuceneDir); - _logger.LogDebug("{IndexName} health check report {IndexReport}", luceneIndex.Name, writer.ToString()); + var mainResult = CreateResult.Init; - if (status.MissingSegments) - { - _logger.LogError("{IndexName} index is missing segments, it will be deleted.", luceneIndex.Name); - result = CreateResult.MissingSegments; - } - else if (!status.Clean) - { - _logger.LogWarning("Checked main director index and it is not clean, attempting to fix {IndexName}. {DocumentsLost} documents will be lost.", luceneIndex.Name, status.TotLoseDocCount); - result = CreateResult.NotClean; + if (mainIndexExists) + { + mainResult = CheckIndexHealthAndFix(mainLuceneDir, luceneIndex.Name); + } - try - { - checker.FixIndex(status); - status = checker.DoCheckIndex(); + // the main index is/was unhealthy or missing, lets check the local index if it exists + if (localIndexExists && (!mainIndexExists || mainResult.HasFlag(CreateResult.NotClean) || mainResult.HasFlag(CreateResult.MissingSegments))) + { + var localResult = CheckIndexHealthAndFix(localLuceneDir, luceneIndex.Name); - if (!status.Clean) - { - _logger.LogError("{IndexName} index could not be fixed, it will be deleted.", luceneIndex.Name); - result |= CreateResult.NotFixed; - } - else + if (localResult == CreateResult.Init) + { + // it was read successfully, we can sync back to main + localResult |= TryGetIndexWriter(OpenMode.APPEND, localLuceneDir, false, luceneIndex.Name, out var indexWriter); + using (indexWriter) + { + if (localResult.HasFlag(CreateResult.OpenedSuccessfully)) { - _logger.LogInformation("Index {IndexName} fixed. {DocumentsLost} documents were lost.", luceneIndex.Name, status.TotLoseDocCount); - result |= CreateResult.Fixed; + SyncIndex(indexWriter, true, luceneIndex.Name, mainLuceneIndexFolder, tempDir); + mainResult |= CreateResult.SyncedFromLocal; } } - catch (Exception ex) - { - _logger.LogError(ex, "{IndexName} index could not be fixed, it will be deleted.", luceneIndex.Name); - result |= CreateResult.ExceptionNotFixed; - } - } - else - { - _logger.LogInformation("Checked main director index {IndexName} and it is clean.", luceneIndex.Name); } } - // used by the replicator, will be a short lived directory for each synced revision and deleted when finished. - var tempDir = new DirectoryInfo(Path.Combine(_localDir.FullName, "Rep", Guid.NewGuid().ToString("N"))); - - if (DirectoryReader.IndexExists(mainDir)) + if (mainIndexExists) { - IndexWriter indexWriter; - // when the lucene directory is going to be created, we'll sync from main storage to local // storage before any index/writer is opened. - try - { - var openMode = result == CreateResult.Init || result.HasFlag(CreateResult.Fixed) + var openMode = mainResult == CreateResult.Init || mainResult.HasFlag(CreateResult.Fixed) || mainResult.HasFlag(CreateResult.SyncedFromLocal) ? OpenMode.APPEND : OpenMode.CREATE; - indexWriter = GetIndexWriter(mainDir, openMode); - - if (openMode == OpenMode.APPEND) - { - result |= CreateResult.OpenedSuccessfully; - } - else - { - result |= CreateResult.CorruptCreatedNew; - } - } - catch (Exception ex) - { - // Index is corrupted, typically this will be FileNotFoundException - _logger.LogError(ex, "{IndexName} index is corrupt, a new one will be created", luceneIndex.Name); - - indexWriter = GetIndexWriter(mainDir, OpenMode.CREATE); - result |= CreateResult.CorruptCreatedNew; - } - - using (var tempMainIndexWriter = indexWriter) - using (var tempMainIndex = new LuceneIndex(_loggerFactory, luceneIndex.Name, new TempOptions(), tempMainIndexWriter)) - using (var tempLocalDirectory = FSDirectory.Open(localLuceneIndexFolder, LockFactory.GetLockFactory(localLuceneIndexFolder))) - using (var replicator = new ExamineReplicator(_loggerFactory, tempMainIndex, tempLocalDirectory, tempDir)) + mainResult |= TryGetIndexWriter(openMode, mainLuceneDir, true, luceneIndex.Name, out var indexWriter); + using (indexWriter) { - if (forceUnlock) + if (!mainResult.HasFlag(CreateResult.SyncedFromLocal)) { - IndexWriter.Unlock(tempLocalDirectory); + SyncIndex(indexWriter, forceUnlock, luceneIndex.Name, localLuceneIndexFolder, tempDir); } - - // replicate locally. - replicator.ReplicateIndex(); } } // now create the replicator that will copy from local to main on schedule - _replicator = new ExamineReplicator(_loggerFactory, luceneIndex, mainDir, tempDir); - var localLuceneDir = FSDirectory.Open( - localLuceneIndexFolder, - LockFactory.GetLockFactory(localLuceneIndexFolder)); + _replicator = new ExamineReplicator(_loggerFactory, luceneIndex, mainLuceneDir, tempDir); if (forceUnlock) { @@ -165,7 +118,7 @@ internal CreateResult TryCreateDirectory(LuceneIndex luceneIndex, bool forceUnlo directory = localLuceneDir; - return result; + return mainResult; } [Flags] @@ -178,7 +131,8 @@ internal enum CreateResult NotFixed = 8, ExceptionNotFixed = 16, CorruptCreatedNew = 32, - OpenedSuccessfully = 64 + OpenedSuccessfully = 64, + SyncedFromLocal = 128 } protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool forceUnlock) @@ -196,6 +150,125 @@ protected override void Dispose(bool disposing) } } + private CreateResult TryGetIndexWriter( + OpenMode openMode, + Directory luceneDirectory, + bool createNewIfCorrupt, + string indexName, + out IndexWriter indexWriter) + { + try + { + indexWriter = GetIndexWriter(luceneDirectory, openMode); + + if (openMode == OpenMode.APPEND) + { + return CreateResult.OpenedSuccessfully; + } + else + { + return CreateResult.CorruptCreatedNew; + } + } + catch (Exception ex) + { + if (createNewIfCorrupt) + { + // Index is corrupted, typically this will be FileNotFoundException + _logger.LogError(ex, "{IndexName} index is corrupt, a new one will be created", indexName); + + indexWriter = GetIndexWriter(luceneDirectory, OpenMode.CREATE); + } + else + { + indexWriter = null; + } + + return CreateResult.CorruptCreatedNew; + } + } + + private void SyncIndex(IndexWriter sourceIndexWriter, bool forceUnlock, string indexName, DirectoryInfo destinationDirectory, DirectoryInfo tempDir) + { + // First, we need to clear the main index. If for some reason it is at the same revision, the syncing won't do anything. + if (destinationDirectory.Exists) + { + foreach (var file in destinationDirectory.EnumerateFiles()) + { + file.Delete(); + } + } + + using (var sourceIndex = new LuceneIndex(_loggerFactory, indexName, new TempOptions(), sourceIndexWriter)) + using (var destinationLuceneDirectory = FSDirectory.Open(destinationDirectory, LockFactory.GetLockFactory(destinationDirectory))) + using (var replicator = new ExamineReplicator(_loggerFactory, sourceIndex, destinationLuceneDirectory, tempDir)) + { + if (forceUnlock) + { + IndexWriter.Unlock(destinationLuceneDirectory); + } + + // replicate locally. + replicator.ReplicateIndex(); + } + } + + private CreateResult CheckIndexHealthAndFix(Directory luceneDir, string indexName) + { + using var writer = new StringWriter(); + var result = CreateResult.Init; + + var checker = new CheckIndex(luceneDir) + { + // Redirect the logging output of the checker + InfoStream = writer + }; + + var status = checker.DoCheckIndex(); + writer.Flush(); + + _logger.LogDebug("{IndexName} health check report {IndexReport}", indexName, writer.ToString()); + + if (status.MissingSegments) + { + _logger.LogError("{IndexName} index is missing segments, it will be deleted.", indexName); + result = CreateResult.MissingSegments; + } + else if (!status.Clean) + { + _logger.LogWarning("Checked main index and it is not clean, attempting to fix {IndexName}. {DocumentsLost} documents will be lost.", indexName, status.TotLoseDocCount); + result = CreateResult.NotClean; + + try + { + checker.FixIndex(status); + status = checker.DoCheckIndex(); + + if (!status.Clean) + { + _logger.LogError("{IndexName} index could not be fixed, it will be deleted.", indexName); + result |= CreateResult.NotFixed; + } + else + { + _logger.LogInformation("Index {IndexName} fixed. {DocumentsLost} documents were lost.", indexName, status.TotLoseDocCount); + result |= CreateResult.Fixed; + } + } + catch (Exception ex) + { + _logger.LogError(ex, "{IndexName} index could not be fixed, it will be deleted.", indexName); + result |= CreateResult.ExceptionNotFixed; + } + } + else + { + _logger.LogInformation("Checked main index {IndexName} and it is clean.", indexName); + } + + return result; + } + private IndexWriter GetIndexWriter(Directory mainDir, OpenMode openMode) { var indexWriter = new IndexWriter( diff --git a/src/Examine.Lucene/ExamineReplicator.cs b/src/Examine.Lucene/ExamineReplicator.cs index 247e2843..d3590a83 100644 --- a/src/Examine.Lucene/ExamineReplicator.cs +++ b/src/Examine.Lucene/ExamineReplicator.cs @@ -74,6 +74,11 @@ public void ReplicateIndex() throw new InvalidOperationException("The destination directory is locked"); } + _logger.LogInformation( + "Replicating index from {SourceIndex} to {DestinationIndex}", + _sourceIndex.GetLuceneDirectory(), + _destinationDirectory); + IndexRevision rev; try { @@ -82,11 +87,17 @@ public void ReplicateIndex() catch (InvalidOperationException) { // will occur if there is nothing to sync + _logger.LogInformation("There was nothing to replicate to {DestinationIndex}", _destinationDirectory); return; } _replicator.Publish(rev); _localReplicationClient.UpdateNow(); + + _logger.LogInformation( + "Replication from index {SourceIndex} to {DestinationIndex} complete.", + _sourceIndex.GetLuceneDirectory(), + _destinationDirectory); } public void StartIndexReplicationOnSchedule(int milliseconds) diff --git a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs index 443e3f89..cf8a2316 100644 --- a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs +++ b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.Options; using Moq; using NUnit.Framework; +using Directory = Lucene.Net.Store.Directory; namespace Examine.Test.Examine.Lucene.Directories { @@ -31,36 +32,7 @@ public void Given_ExistingCorruptIndex_When_CreatingDirectory_Then_IndexCreatedO try { - using (var mainDir = FSDirectory.Open(Path.Combine(mainPath, TestIndex.TestIndexName))) - { - using (var writer = new IndexWriter(mainDir, new IndexWriterConfig(LuceneInfo.CurrentVersion, new CultureInvariantStandardAnalyzer()))) - using (var indexer = GetTestIndex(writer)) - { - using (indexer.WithThreadingMode(IndexThreadingMode.Synchronous)) - { - indexer.IndexItem(new ValueSet(1.ToString(), "content", - new Dictionary> - { - {"item1", new List(new[] {"value1"})}, - {"item2", new List(new[] {"value2"})} - })); - } - } - - Assert.IsTrue(DirectoryReader.IndexExists(mainDir)); - - if (corruptIndex) - { - // Get an index (non segments file) and delete it (corrupt index) - var indexFile = mainDir.Directory.GetFiles() - .Where(x => removeSegments - ? x.Name.Contains("segments_", StringComparison.OrdinalIgnoreCase) - : !x.Name.Contains("segments", StringComparison.OrdinalIgnoreCase)) - .First(); - - File.Delete(indexFile.FullName); - } - } + CreateIndex(mainPath, corruptIndex, removeSegments); using var syncedDirFactory = new SyncedFileSystemDirectoryFactory( new DirectoryInfo(tempPath), @@ -86,5 +58,102 @@ public void Given_ExistingCorruptIndex_When_CreatingDirectory_Then_IndexCreatedO System.IO.Directory.Delete(tempPath, true); } } + + [Test] + public void Given_CorruptMainIndex_And_HealthyLocalIndex_When_CreatingDirectory_Then_LocalIndexSyncedToMain() + { + var mainPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + + try + { + // create unhealthy index + CreateIndex(mainPath, true, false); + + // create healthy index + CreateIndex(tempPath, false, false); + + using (var syncedDirFactory = new SyncedFileSystemDirectoryFactory( + new DirectoryInfo(tempPath), + new DirectoryInfo(mainPath), + new DefaultLockFactory(), + LoggerFactory)) + { + using var index = new LuceneIndex( + LoggerFactory, + TestIndex.TestIndexName, + Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions + { + DirectoryFactory = syncedDirFactory, + })); + + var result = syncedDirFactory.TryCreateDirectory(index, false, out var dir); + + Assert.IsTrue(result.HasFlag(SyncedFileSystemDirectoryFactory.CreateResult.SyncedFromLocal)); + } + + // Ensure the docs are there in main + using var mainIndex = new LuceneIndex( + LoggerFactory, + TestIndex.TestIndexName, + Mock.Of>(x => x.Get(TestIndex.TestIndexName) == new LuceneDirectoryIndexOptions + { + DirectoryFactory = new GenericDirectoryFactory(_ => FSDirectory.Open(Path.Combine(mainPath, TestIndex.TestIndexName))), + })); + + var searchResults = mainIndex.Searcher.CreateQuery().All().Execute(); + Assert.AreEqual(2, searchResults.TotalItemCount); + } + finally + { + System.IO.Directory.Delete(mainPath, true); + System.IO.Directory.Delete(tempPath, true); + } + } + + private void CreateIndex(string rootPath, bool corruptIndex, bool removeSegments) + { + using var luceneDir = FSDirectory.Open(Path.Combine(rootPath, TestIndex.TestIndexName)); + + using (var writer = new IndexWriter(luceneDir, new IndexWriterConfig(LuceneInfo.CurrentVersion, new CultureInvariantStandardAnalyzer()))) + using (var indexer = GetTestIndex(writer)) + using (indexer.WithThreadingMode(IndexThreadingMode.Synchronous)) + { + indexer.IndexItems(new[] + { + new ValueSet(1.ToString(), "content", + new Dictionary> + { + {"item1", new List(new[] {"value1"})}, + {"item2", new List(new[] {"value2"})} + }), + new ValueSet(2.ToString(), "content", + new Dictionary> + { + {"item1", new List(new[] {"value3"})}, + {"item2", new List(new[] {"value4"})} + }), + }); + } + + Assert.IsTrue(DirectoryReader.IndexExists(luceneDir)); + + if (corruptIndex) + { + CorruptIndex(luceneDir.Directory, removeSegments); + } + } + + private void CorruptIndex(DirectoryInfo dir, bool removeSegments) + { + // Get an index (non segments file) and delete it (corrupt index) + var indexFile = dir.GetFiles() + .Where(x => removeSegments + ? x.Name.Contains("segments_", StringComparison.OrdinalIgnoreCase) + : !x.Name.Contains("segments", StringComparison.OrdinalIgnoreCase)) + .First(); + + File.Delete(indexFile.FullName); + } } } From 3cb55ea2e28d77f8e13237c5e9afab0464c673a6 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 27 Jun 2024 12:08:31 -0600 Subject: [PATCH 08/16] Makes it optional whether to attempt to fix the indexes, default is false --- .../SyncedFileSystemDirectoryFactory.cs | 54 ++++++++++++------- .../SyncedFileSystemDirectoryFactoryTests.cs | 6 ++- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs index 4bf1fd20..5474be0d 100644 --- a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs @@ -24,6 +24,7 @@ public class SyncedFileSystemDirectoryFactory : FileSystemDirectoryFactory private readonly DirectoryInfo _localDir; private readonly DirectoryInfo _mainDir; private readonly ILoggerFactory _loggerFactory; + private readonly bool _tryFixMainIndexIfCorrupt; private readonly ILogger _logger; private ExamineReplicator _replicator; @@ -32,11 +33,22 @@ public SyncedFileSystemDirectoryFactory( DirectoryInfo mainDir, ILockFactory lockFactory, ILoggerFactory loggerFactory) + : this(localDir, mainDir, lockFactory, loggerFactory, false) + { + } + + public SyncedFileSystemDirectoryFactory( + DirectoryInfo localDir, + DirectoryInfo mainDir, + ILockFactory lockFactory, + ILoggerFactory loggerFactory, + bool tryFixMainIndexIfCorrupt) : base(mainDir, lockFactory) { _localDir = localDir; _mainDir = mainDir; _loggerFactory = loggerFactory; + _tryFixMainIndexIfCorrupt = tryFixMainIndexIfCorrupt; _logger = _loggerFactory.CreateLogger(); } @@ -63,13 +75,13 @@ internal CreateResult TryCreateDirectory(LuceneIndex luceneIndex, bool forceUnlo if (mainIndexExists) { - mainResult = CheckIndexHealthAndFix(mainLuceneDir, luceneIndex.Name); + mainResult = CheckIndexHealthAndFix(mainLuceneDir, luceneIndex.Name, _tryFixMainIndexIfCorrupt); } // the main index is/was unhealthy or missing, lets check the local index if it exists if (localIndexExists && (!mainIndexExists || mainResult.HasFlag(CreateResult.NotClean) || mainResult.HasFlag(CreateResult.MissingSegments))) { - var localResult = CheckIndexHealthAndFix(localLuceneDir, luceneIndex.Name); + var localResult = CheckIndexHealthAndFix(localLuceneDir, luceneIndex.Name, false); if (localResult == CreateResult.Init) { @@ -213,7 +225,10 @@ private void SyncIndex(IndexWriter sourceIndexWriter, bool forceUnlock, string i } } - private CreateResult CheckIndexHealthAndFix(Directory luceneDir, string indexName) + private CreateResult CheckIndexHealthAndFix( + Directory luceneDir, + string indexName, + bool doFix) { using var writer = new StringWriter(); var result = CreateResult.Init; @@ -239,27 +254,30 @@ private CreateResult CheckIndexHealthAndFix(Directory luceneDir, string indexNam _logger.LogWarning("Checked main index and it is not clean, attempting to fix {IndexName}. {DocumentsLost} documents will be lost.", indexName, status.TotLoseDocCount); result = CreateResult.NotClean; - try + if (doFix) { - checker.FixIndex(status); - status = checker.DoCheckIndex(); - - if (!status.Clean) + try { - _logger.LogError("{IndexName} index could not be fixed, it will be deleted.", indexName); - result |= CreateResult.NotFixed; + checker.FixIndex(status); + status = checker.DoCheckIndex(); + + if (!status.Clean) + { + _logger.LogError("{IndexName} index could not be fixed, it will be deleted.", indexName); + result |= CreateResult.NotFixed; + } + else + { + _logger.LogInformation("Index {IndexName} fixed. {DocumentsLost} documents were lost.", indexName, status.TotLoseDocCount); + result |= CreateResult.Fixed; + } } - else + catch (Exception ex) { - _logger.LogInformation("Index {IndexName} fixed. {DocumentsLost} documents were lost.", indexName, status.TotLoseDocCount); - result |= CreateResult.Fixed; + _logger.LogError(ex, "{IndexName} index could not be fixed, it will be deleted.", indexName); + result |= CreateResult.ExceptionNotFixed; } } - catch (Exception ex) - { - _logger.LogError(ex, "{IndexName} index could not be fixed, it will be deleted.", indexName); - result |= CreateResult.ExceptionNotFixed; - } } else { diff --git a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs index cf8a2316..aea2f5a5 100644 --- a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs +++ b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs @@ -38,7 +38,8 @@ public void Given_ExistingCorruptIndex_When_CreatingDirectory_Then_IndexCreatedO new DirectoryInfo(tempPath), new DirectoryInfo(mainPath), new DefaultLockFactory(), - LoggerFactory); + LoggerFactory, + true); using var index = new LuceneIndex( LoggerFactory, @@ -77,7 +78,8 @@ public void Given_CorruptMainIndex_And_HealthyLocalIndex_When_CreatingDirectory_ new DirectoryInfo(tempPath), new DirectoryInfo(mainPath), new DefaultLockFactory(), - LoggerFactory)) + LoggerFactory, + true)) { using var index = new LuceneIndex( LoggerFactory, From f1857c60427f319fb99d2ccc1be00792bbfe7d70 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 17 Jul 2024 13:02:14 -0600 Subject: [PATCH 09/16] mostly just analysis changes --- src/Examine.Core/FieldDefinition.cs | 15 +++++++++++---- src/Examine.Core/IndexOperation.cs | 4 ++-- src/Examine.Core/PublicAPI.Unshipped.txt | 1 + src/Examine.Core/Search/ExamineValue.cs | 5 ++--- src/Examine.Core/Search/SortableField.cs | 6 +++--- src/Examine.Core/ValueSetValidationResult.cs | 2 +- .../Directories/FileSystemDirectoryFactory.cs | 1 + src/Examine.Lucene/Providers/LuceneIndex.cs | 6 +++--- src/Examine.Lucene/PublicAPI.Unshipped.txt | 1 + .../Search/LuceneSearchQueryBase.cs | 8 ++++++++ src/Examine.Test/Examine.Test.csproj | 2 +- 11 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/Examine.Core/FieldDefinition.cs b/src/Examine.Core/FieldDefinition.cs index 1513d108..18ee688e 100644 --- a/src/Examine.Core/FieldDefinition.cs +++ b/src/Examine.Core/FieldDefinition.cs @@ -5,7 +5,7 @@ namespace Examine /// /// Defines a field to be indexed /// - public struct FieldDefinition : IEquatable + public readonly struct FieldDefinition : IEquatable { /// /// Constructor @@ -14,8 +14,14 @@ public struct FieldDefinition : IEquatable /// public FieldDefinition(string name, string type) { - if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Value cannot be null or whitespace.", nameof(name)); - if (string.IsNullOrWhiteSpace(type)) throw new ArgumentException("Value cannot be null or whitespace.", nameof(type)); + if (string.IsNullOrWhiteSpace(name)) + { + throw new ArgumentException("Value cannot be null or whitespace.", nameof(name)); + } + if (string.IsNullOrWhiteSpace(type)) + { + throw new ArgumentException("Value cannot be null or whitespace.", nameof(type)); + } Name = name; Type = type; } @@ -34,7 +40,8 @@ public FieldDefinition(string name, string type) public override bool Equals(object obj) { - if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(null, obj)) + return false; return obj is FieldDefinition definition && Equals(definition); } diff --git a/src/Examine.Core/IndexOperation.cs b/src/Examine.Core/IndexOperation.cs index 7ddbd4ea..5d7122c8 100644 --- a/src/Examine.Core/IndexOperation.cs +++ b/src/Examine.Core/IndexOperation.cs @@ -3,7 +3,7 @@ namespace Examine /// /// Represents an indexing operation (either add/remove) /// - public struct IndexOperation + public readonly struct IndexOperation { /// /// Initializes a new instance of the class. @@ -27,4 +27,4 @@ public IndexOperation(ValueSet valueSet, IndexOperationType operation) /// public IndexOperationType Operation { get; } } -} \ No newline at end of file +} diff --git a/src/Examine.Core/PublicAPI.Unshipped.txt b/src/Examine.Core/PublicAPI.Unshipped.txt index e69de29b..3b367992 100644 --- a/src/Examine.Core/PublicAPI.Unshipped.txt +++ b/src/Examine.Core/PublicAPI.Unshipped.txt @@ -0,0 +1 @@ +static Examine.SearchExtensions.Escape(this string s, float boost) -> Examine.Search.IExamineValue \ No newline at end of file diff --git a/src/Examine.Core/Search/ExamineValue.cs b/src/Examine.Core/Search/ExamineValue.cs index c49eeac0..660eaac6 100644 --- a/src/Examine.Core/Search/ExamineValue.cs +++ b/src/Examine.Core/Search/ExamineValue.cs @@ -1,8 +1,8 @@ -using Examine.Search; +using Examine.Search; namespace Examine.Search { - public struct ExamineValue : IExamineValue + public readonly struct ExamineValue : IExamineValue { public ExamineValue(Examineness vagueness, string value) : this(vagueness, value, 1) @@ -21,6 +21,5 @@ public ExamineValue(Examineness vagueness, string value, float level) public string Value { get; } public float Level { get; } - } } diff --git a/src/Examine.Core/Search/SortableField.cs b/src/Examine.Core/Search/SortableField.cs index 2ef14efa..14665a03 100644 --- a/src/Examine.Core/Search/SortableField.cs +++ b/src/Examine.Core/Search/SortableField.cs @@ -1,9 +1,9 @@ -namespace Examine.Search +namespace Examine.Search { /// /// Represents a field used to sort results /// - public struct SortableField + public readonly struct SortableField { /// /// The field name to sort by @@ -36,4 +36,4 @@ public SortableField(string fieldName, SortType sortType) SortType = sortType; } } -} \ No newline at end of file +} diff --git a/src/Examine.Core/ValueSetValidationResult.cs b/src/Examine.Core/ValueSetValidationResult.cs index 14834612..0f138a71 100644 --- a/src/Examine.Core/ValueSetValidationResult.cs +++ b/src/Examine.Core/ValueSetValidationResult.cs @@ -1,6 +1,6 @@ namespace Examine { - public struct ValueSetValidationResult + public readonly struct ValueSetValidationResult { public ValueSetValidationResult(ValueSetValidationStatus status, ValueSet valueSet) { diff --git a/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs index 2da989af..72ae5ca8 100644 --- a/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/FileSystemDirectoryFactory.cs @@ -28,6 +28,7 @@ protected override Directory CreateDirectory(LuceneIndex luceneIndex, bool force { IndexWriter.Unlock(dir); } + return dir; } } diff --git a/src/Examine.Lucene/Providers/LuceneIndex.cs b/src/Examine.Lucene/Providers/LuceneIndex.cs index e9f0fcfa..af400534 100644 --- a/src/Examine.Lucene/Providers/LuceneIndex.cs +++ b/src/Examine.Lucene/Providers/LuceneIndex.cs @@ -89,12 +89,13 @@ internal LuceneIndex( #endregion + private static readonly string[] s_possibleSuffixes = new[] { "Index", "Indexer" }; private readonly LuceneIndexOptions _options; private PerFieldAnalyzerWrapper _fieldAnalyzer; private ControlledRealTimeReopenThread _nrtReopenThread; private readonly ILogger _logger; private readonly Lazy _directory; - private FileStream _logOutput; + private readonly FileStream _logOutput; private bool _disposedValue; private readonly IndexCommiter _committer; @@ -1029,9 +1030,8 @@ public TrackingIndexWriter IndexWriter private LuceneSearcher CreateSearcher() { - var possibleSuffixes = new[] { "Index", "Indexer" }; var name = Name; - foreach (var suffix in possibleSuffixes) + foreach (var suffix in s_possibleSuffixes) { //trim the "Indexer" / "Index" suffix if it exists if (!name.EndsWith(suffix)) diff --git a/src/Examine.Lucene/PublicAPI.Unshipped.txt b/src/Examine.Lucene/PublicAPI.Unshipped.txt index b31f1482..23436976 100644 --- a/src/Examine.Lucene/PublicAPI.Unshipped.txt +++ b/src/Examine.Lucene/PublicAPI.Unshipped.txt @@ -1 +1,2 @@ +Examine.Lucene.Directories.SyncedFileSystemDirectoryFactory.SyncedFileSystemDirectoryFactory(System.IO.DirectoryInfo localDir, System.IO.DirectoryInfo mainDir, Examine.Lucene.Directories.ILockFactory lockFactory, Microsoft.Extensions.Logging.ILoggerFactory loggerFactory, bool tryFixMainIndexIfCorrupt) -> void virtual Examine.Lucene.Providers.LuceneIndex.UpdateLuceneDocument(Lucene.Net.Index.Term term, Lucene.Net.Documents.Document doc) -> long? \ No newline at end of file diff --git a/src/Examine.Lucene/Search/LuceneSearchQueryBase.cs b/src/Examine.Lucene/Search/LuceneSearchQueryBase.cs index 437c5139..a5dc75c8 100644 --- a/src/Examine.Lucene/Search/LuceneSearchQueryBase.cs +++ b/src/Examine.Lucene/Search/LuceneSearchQueryBase.cs @@ -286,11 +286,19 @@ protected internal LuceneBooleanOperationBase IdInternal(string id, Occur occurr protected virtual Query GetFieldInternalQuery(string fieldName, IExamineValue fieldValue, bool useQueryParser) { if (string.IsNullOrEmpty(fieldName)) + { throw new ArgumentException($"'{nameof(fieldName)}' cannot be null or empty", nameof(fieldName)); + } + if (fieldValue is null) + { throw new ArgumentNullException(nameof(fieldValue)); + } + if (string.IsNullOrEmpty(fieldValue.Value)) + { throw new ArgumentException($"'{nameof(fieldName)}' cannot be null or empty", nameof(fieldName)); + } Query queryToAdd; diff --git a/src/Examine.Test/Examine.Test.csproj b/src/Examine.Test/Examine.Test.csproj index 51e2436a..008b128b 100644 --- a/src/Examine.Test/Examine.Test.csproj +++ b/src/Examine.Test/Examine.Test.csproj @@ -1,4 +1,4 @@ - + Library From d85b1e1108e8df1168b59ec050b692b1e3f8af03 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 18 Jul 2024 09:18:23 -0600 Subject: [PATCH 10/16] update test output --- .../Directories/SyncedFileSystemDirectoryFactoryTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs index aea2f5a5..5a850dad 100644 --- a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs +++ b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs @@ -51,7 +51,7 @@ public void Given_ExistingCorruptIndex_When_CreatingDirectory_Then_IndexCreatedO var result = syncedDirFactory.TryCreateDirectory(index, false, out var dir); - Assert.IsTrue(result.HasFlag(expected)); + Assert.IsTrue(result.HasFlag(expected), $"{result} does not have flag {expected}"); } finally { From 5a5c5f252d523cb651a7f401f0e0c00e5fe6de19 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 18 Jul 2024 10:12:52 -0600 Subject: [PATCH 11/16] trying to get tests to work --- .../Directories/SyncedFileSystemDirectoryFactoryTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs index 5a850dad..1ed78b5e 100644 --- a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs +++ b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs @@ -19,8 +19,8 @@ namespace Examine.Test.Examine.Lucene.Directories public class SyncedFileSystemDirectoryFactoryTests : ExamineBaseTest { [TestCase(true, false, SyncedFileSystemDirectoryFactory.CreateResult.NotClean | SyncedFileSystemDirectoryFactory.CreateResult.Fixed | SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully)] - [TestCase(true, true, SyncedFileSystemDirectoryFactory.CreateResult.MissingSegments | SyncedFileSystemDirectoryFactory.CreateResult.CorruptCreatedNew)] - [TestCase(false, false, SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully)] + [TestCase(true, true, SyncedFileSystemDirectoryFactory.CreateResult.MissingSegments | SyncedFileSystemDirectoryFactory.CreateResult.CorruptCreatedNew, Ignore = "testing")] + [TestCase(false, false, SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully, Ignore = "testing")] [Test] public void Given_ExistingCorruptIndex_When_CreatingDirectory_Then_IndexCreatedOrOpened( bool corruptIndex, From 7c17e5d4a6ab79c9b4f83333cbe21cb4db0cfef2 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 18 Jul 2024 17:33:39 -0600 Subject: [PATCH 12/16] adds logging --- .../SyncedFileSystemDirectoryFactoryTests.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs index 1ed78b5e..c51477a6 100644 --- a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs +++ b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs @@ -8,6 +8,7 @@ using Examine.Lucene.Providers; using Lucene.Net.Index; using Lucene.Net.Store; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Moq; using NUnit.Framework; @@ -18,6 +19,13 @@ namespace Examine.Test.Examine.Lucene.Directories [TestFixture] public class SyncedFileSystemDirectoryFactoryTests : ExamineBaseTest { + private readonly ILogger _logger; + + public SyncedFileSystemDirectoryFactoryTests() + { + _logger = LoggerFactory.CreateLogger(); + } + [TestCase(true, false, SyncedFileSystemDirectoryFactory.CreateResult.NotClean | SyncedFileSystemDirectoryFactory.CreateResult.Fixed | SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully)] [TestCase(true, true, SyncedFileSystemDirectoryFactory.CreateResult.MissingSegments | SyncedFileSystemDirectoryFactory.CreateResult.CorruptCreatedNew, Ignore = "testing")] [TestCase(false, false, SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully, Ignore = "testing")] @@ -115,7 +123,10 @@ public void Given_CorruptMainIndex_And_HealthyLocalIndex_When_CreatingDirectory_ private void CreateIndex(string rootPath, bool corruptIndex, bool removeSegments) { - using var luceneDir = FSDirectory.Open(Path.Combine(rootPath, TestIndex.TestIndexName)); + var indexPath = Path.Combine(rootPath, TestIndex.TestIndexName); + _logger.LogInformation("Creating index at " + indexPath); + + using var luceneDir = FSDirectory.Open(indexPath); using (var writer = new IndexWriter(luceneDir, new IndexWriterConfig(LuceneInfo.CurrentVersion, new CultureInvariantStandardAnalyzer()))) using (var indexer = GetTestIndex(writer)) @@ -138,6 +149,7 @@ private void CreateIndex(string rootPath, bool corruptIndex, bool removeSegments }); } + _logger.LogInformation("Created index at " + luceneDir.Directory); Assert.IsTrue(DirectoryReader.IndexExists(luceneDir)); if (corruptIndex) @@ -155,6 +167,7 @@ private void CorruptIndex(DirectoryInfo dir, bool removeSegments) : !x.Name.Contains("segments", StringComparison.OrdinalIgnoreCase)) .First(); + Console.WriteLine($"Deleting {indexFile.FullName}"); File.Delete(indexFile.FullName); } } From 328dcd8f13aa54d662056c04b43111d8ce486426 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 19 Jul 2024 08:20:14 -0600 Subject: [PATCH 13/16] try again --- .../SyncedFileSystemDirectoryFactoryTests.cs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs index c51477a6..5059703e 100644 --- a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs +++ b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs @@ -19,13 +19,6 @@ namespace Examine.Test.Examine.Lucene.Directories [TestFixture] public class SyncedFileSystemDirectoryFactoryTests : ExamineBaseTest { - private readonly ILogger _logger; - - public SyncedFileSystemDirectoryFactoryTests() - { - _logger = LoggerFactory.CreateLogger(); - } - [TestCase(true, false, SyncedFileSystemDirectoryFactory.CreateResult.NotClean | SyncedFileSystemDirectoryFactory.CreateResult.Fixed | SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully)] [TestCase(true, true, SyncedFileSystemDirectoryFactory.CreateResult.MissingSegments | SyncedFileSystemDirectoryFactory.CreateResult.CorruptCreatedNew, Ignore = "testing")] [TestCase(false, false, SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully, Ignore = "testing")] @@ -123,8 +116,10 @@ public void Given_CorruptMainIndex_And_HealthyLocalIndex_When_CreatingDirectory_ private void CreateIndex(string rootPath, bool corruptIndex, bool removeSegments) { + var logger = LoggerFactory.CreateLogger(); + var indexPath = Path.Combine(rootPath, TestIndex.TestIndexName); - _logger.LogInformation("Creating index at " + indexPath); + logger.LogInformation("Creating index at " + indexPath); using var luceneDir = FSDirectory.Open(indexPath); @@ -149,7 +144,7 @@ private void CreateIndex(string rootPath, bool corruptIndex, bool removeSegments }); } - _logger.LogInformation("Created index at " + luceneDir.Directory); + logger.LogInformation("Created index at " + luceneDir.Directory); Assert.IsTrue(DirectoryReader.IndexExists(luceneDir)); if (corruptIndex) From 5fb1fd14048770f46db0fe518a13d3b7d48a761e Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 19 Jul 2024 08:44:10 -0600 Subject: [PATCH 14/16] try again --- .../Directories/SyncedFileSystemDirectoryFactoryTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs index 5059703e..28f35660 100644 --- a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs +++ b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs @@ -119,7 +119,7 @@ private void CreateIndex(string rootPath, bool corruptIndex, bool removeSegments var logger = LoggerFactory.CreateLogger(); var indexPath = Path.Combine(rootPath, TestIndex.TestIndexName); - logger.LogInformation("Creating index at " + indexPath); + logger.LogInformation($"Creating index at {indexPath} with options: corruptIndex: {corruptIndex}, removeSegments: {removeSegments}"); using var luceneDir = FSDirectory.Open(indexPath); @@ -149,11 +149,11 @@ private void CreateIndex(string rootPath, bool corruptIndex, bool removeSegments if (corruptIndex) { - CorruptIndex(luceneDir.Directory, removeSegments); + CorruptIndex(luceneDir.Directory, removeSegments, logger); } } - private void CorruptIndex(DirectoryInfo dir, bool removeSegments) + private void CorruptIndex(DirectoryInfo dir, bool removeSegments, ILogger logger) { // Get an index (non segments file) and delete it (corrupt index) var indexFile = dir.GetFiles() @@ -162,7 +162,7 @@ private void CorruptIndex(DirectoryInfo dir, bool removeSegments) : !x.Name.Contains("segments", StringComparison.OrdinalIgnoreCase)) .First(); - Console.WriteLine($"Deleting {indexFile.FullName}"); + logger.LogInformation($"Deleting {indexFile.FullName}"); File.Delete(indexFile.FullName); } } From 5129af2c03a7bfc3610e63aea1f1ed0e26833c38 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 19 Jul 2024 08:49:10 -0600 Subject: [PATCH 15/16] try again --- .../Directories/SyncedFileSystemDirectoryFactoryTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs index 28f35660..24b2cff5 100644 --- a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs +++ b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs @@ -17,6 +17,7 @@ namespace Examine.Test.Examine.Lucene.Directories { [TestFixture] + [NonParallelizable] public class SyncedFileSystemDirectoryFactoryTests : ExamineBaseTest { [TestCase(true, false, SyncedFileSystemDirectoryFactory.CreateResult.NotClean | SyncedFileSystemDirectoryFactory.CreateResult.Fixed | SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully)] From 7c255306b32478e95a5116267f3c0f23962a4503 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 31 Jul 2024 09:36:24 -0600 Subject: [PATCH 16/16] Fix tests --- .../SyncedFileSystemDirectoryFactory.cs | 2 +- .../SyncedFileSystemDirectoryFactoryTests.cs | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs index 5474be0d..2d653875 100644 --- a/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs +++ b/src/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactory.cs @@ -246,7 +246,7 @@ private CreateResult CheckIndexHealthAndFix( if (status.MissingSegments) { - _logger.LogError("{IndexName} index is missing segments, it will be deleted.", indexName); + _logger.LogWarning("{IndexName} index is missing segments, it will be deleted.", indexName); result = CreateResult.MissingSegments; } else if (!status.Clean) diff --git a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs index 24b2cff5..e124c784 100644 --- a/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs +++ b/src/Examine.Test/Examine.Lucene/Directories/SyncedFileSystemDirectoryFactoryTests.cs @@ -6,6 +6,7 @@ using Examine.Lucene.Analyzers; using Examine.Lucene.Directories; using Examine.Lucene.Providers; +using Lucene.Net.Codecs.Lucene46; using Lucene.Net.Index; using Lucene.Net.Store; using Microsoft.Extensions.Logging; @@ -21,8 +22,8 @@ namespace Examine.Test.Examine.Lucene.Directories public class SyncedFileSystemDirectoryFactoryTests : ExamineBaseTest { [TestCase(true, false, SyncedFileSystemDirectoryFactory.CreateResult.NotClean | SyncedFileSystemDirectoryFactory.CreateResult.Fixed | SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully)] - [TestCase(true, true, SyncedFileSystemDirectoryFactory.CreateResult.MissingSegments | SyncedFileSystemDirectoryFactory.CreateResult.CorruptCreatedNew, Ignore = "testing")] - [TestCase(false, false, SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully, Ignore = "testing")] + [TestCase(true, true, SyncedFileSystemDirectoryFactory.CreateResult.MissingSegments | SyncedFileSystemDirectoryFactory.CreateResult.CorruptCreatedNew)] + [TestCase(false, false, SyncedFileSystemDirectoryFactory.CreateResult.OpenedSuccessfully)] [Test] public void Given_ExistingCorruptIndex_When_CreatingDirectory_Then_IndexCreatedOrOpened( bool corruptIndex, @@ -94,7 +95,7 @@ public void Given_CorruptMainIndex_And_HealthyLocalIndex_When_CreatingDirectory_ var result = syncedDirFactory.TryCreateDirectory(index, false, out var dir); Assert.IsTrue(result.HasFlag(SyncedFileSystemDirectoryFactory.CreateResult.SyncedFromLocal)); - } + } // Ensure the docs are there in main using var mainIndex = new LuceneIndex( @@ -156,11 +157,16 @@ private void CreateIndex(string rootPath, bool corruptIndex, bool removeSegments private void CorruptIndex(DirectoryInfo dir, bool removeSegments, ILogger logger) { + // index file extensions (no segments, no gen) + var indexFileExtensions = IndexFileNames.INDEX_EXTENSIONS + .Except(new[] { IndexFileNames.GEN_EXTENSION }) + .ToArray(); + // Get an index (non segments file) and delete it (corrupt index) var indexFile = dir.GetFiles() .Where(x => removeSegments - ? x.Name.Contains("segments_", StringComparison.OrdinalIgnoreCase) - : !x.Name.Contains("segments", StringComparison.OrdinalIgnoreCase)) + ? x.Extension.Contains(Lucene46SegmentInfoFormat.SI_EXTENSION, StringComparison.OrdinalIgnoreCase) + : indexFileExtensions.Any(e => IndexFileNames.MatchesExtension(x.Extension, e))) .First(); logger.LogInformation($"Deleting {indexFile.FullName}");