DYN-6038 Lucene search IMP (DynamoDS#14428)

* improvements

* Simply initialization logic

* Skip indexing as part of DynamoModel if the index files already exist

* Remove DynamoModel reference and use singleton

* clean up

* improvements and package manager search

* Use single constructor and other code clean up

* Update...

* Update

* Dispose all Lucene objects in the correct order

* Adding DynamoModel back to test regressions

* Null check

* Update

* IndexWriter dispose sequence only for non-RAM mode

Author: QilongTang

src/DynamoCore/Models/DynamoModel.cs

@@ -921,16 +921,7 @@ protected DynamoModel(IStartConfiguration config)
 
             CustomNodeManager = new CustomNodeManager(NodeFactory, MigrationManager, LibraryServices);
 
-            LuceneSearch.LuceneUtilityNodeSearch = new LuceneSearchUtility(this);
-
-            if (IsTestMode)
-            {
-                LuceneUtility.InitializeLuceneConfig(string.Empty, LuceneSearchUtility.LuceneStorage.RAM);
-            }
-            else
-            {
-                LuceneUtility.InitializeLuceneConfig(LuceneConfig.NodesIndexingDirectory);
-            }
+            LuceneSearch.LuceneUtilityNodeSearch = new LuceneSearchUtility(this, LuceneSearchUtility.DefaultNodeIndexStartConfig);
 
             InitializeCustomNodeManager();
 
@@ -1013,10 +1004,15 @@ protected DynamoModel(IStartConfiguration config)
             TraceReconciliationProcessor = this;
 
             State = DynamoModelState.StartedUIless;
-            // This event should only be raised at the end of this method.
-            DynamoReady(new ReadyParams(this));
             // Write index to disk only once
             LuceneUtility.CommitWriterChanges();
+            //Disposed writer if it is in file system mode so that the index files can be used by other processes (potentially a second Dynamo session)
+            if (LuceneUtility.startConfig.StorageType == LuceneSearchUtility.LuceneStorage.FILE_SYSTEM)
+            {
+                LuceneUtility.DisposeWriter();
+            }
+            // This event should only be raised at the end of this method.
+            DynamoReady(new ReadyParams(this));
         }
 
         private void SearchModel_ItemProduced(NodeModel node)
@@ -1405,12 +1401,8 @@ public void Dispose()
                 PreferenceSettings.MessageLogged -= LogMessage;
             }
 
-            //The writer have to be disposed at DynamoModel level due that we could index package-nodes as new packages are installed
-            LuceneUtility.DisposeWriter();
-
             // Lucene disposals (just if LuceneNET was initialized)
-            LuceneUtility.indexDir?.Dispose();
-            LuceneUtility.dirReader?.Dispose();
+            LuceneUtility.DisposeAll();
 
 #if DEBUG
             CurrentWorkspace.NodeAdded -= CrashOnDemand.CurrentWorkspace_NodeAdded;
@@ -1474,7 +1466,7 @@ private void InitializeCustomNodeManager()
                 var iDoc = LuceneUtility.InitializeIndexDocumentForNodes();
                 if (searchElement != null)
                 {
-                    AddNodeTypeToSearchIndex(searchElement, iDoc);
+                    LuceneUtility.AddNodeTypeToSearchIndex(searchElement, iDoc);
                 }
 
                 Action<CustomNodeInfo> infoUpdatedHandler = null;
@@ -1543,7 +1535,7 @@ private void InitializeIncludedNodes()
 
             var cnbNode = new CodeBlockNodeSearchElement(cbnData, LibraryServices);
             SearchModel?.Add(cnbNode);
-            AddNodeTypeToSearchIndex(cnbNode, iDoc);
+            LuceneUtility.AddNodeTypeToSearchIndex(cnbNode, iDoc);
 
             var symbolSearchElement = new NodeModelSearchElement(symbolData)
             {
@@ -1562,10 +1554,10 @@ private void InitializeIncludedNodes()
             };
 
             SearchModel?.Add(symbolSearchElement);
-            AddNodeTypeToSearchIndex(symbolSearchElement, iDoc);
+            LuceneUtility.AddNodeTypeToSearchIndex(symbolSearchElement, iDoc);
 
             SearchModel?.Add(outputSearchElement);
-            AddNodeTypeToSearchIndex(outputSearchElement, iDoc);
+            LuceneUtility.AddNodeTypeToSearchIndex(outputSearchElement, iDoc);
 
         }
 
@@ -1719,7 +1711,7 @@ private void LoadNodeModels(List<TypeLoadData> nodes, bool isPackageMember)
                     // TODO: get search element some other way
                     if (ele != null)
                     {
-                        AddNodeTypeToSearchIndex(ele, iDoc);
+                        LuceneUtility.AddNodeTypeToSearchIndex(ele, iDoc);
                     }
                 }
                 catch (Exception e)
@@ -3219,24 +3211,6 @@ private NodeModelSearchElement AddNodeTypeToSearch(TypeLoadData typeLoadData)
             return node;
         }
 
-        /// <summary>
-        /// Add node information to Lucene index
-        /// </summary>
-        /// <param name="node">node info that will be indexed</param>
-        /// <param name="doc">Lucene document in which the node info will be indexed</param>
-        internal void AddNodeTypeToSearchIndex(NodeSearchElement node, Document doc)
-        {
-            if (LuceneUtility.addedFields == null) return;
-
-            LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.FullCategoryName), node.FullCategoryName);
-            LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Name), node.Name);
-            LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Description), node.Description);
-            if (node.SearchKeywords.Count > 0) LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.SearchKeywords), node.SearchKeywords.Aggregate((x, y) => x + " " + y), true, true);
-            LuceneUtility.SetDocumentFieldValue(doc, nameof(LuceneConfig.NodeFieldsEnum.Parameters), node.Parameters ?? string.Empty);
-
-            LuceneUtility.writer?.AddDocument(doc);
-        }
-
         /// <summary>
         /// Remove node information from Lucene indexing.
         /// </summary>
@@ -3296,7 +3270,7 @@ private void AddZeroTouchNodeToSearch(FunctionDescriptor functionDescriptor, Doc
             {
                 var ele = new ZeroTouchSearchElement(functionDescriptor);
                 SearchModel?.Add(ele);
-                AddNodeTypeToSearchIndex(ele, iDoc);
+                LuceneUtility.AddNodeTypeToSearchIndex(ele, iDoc);
             }
         }
 

src/DynamoCore/Search/NodeSearchModel.cs

@@ -10,6 +10,7 @@
 using Dynamo.Utilities;
 using DynamoUtilities;
 using Lucene.Net.Documents;
+using Lucene.Net.Index;
 using Lucene.Net.QueryParsers.Classic;
 using Lucene.Net.Search;
 
@@ -236,9 +237,14 @@ internal IEnumerable<NodeSearchElement> Search(string search, LuceneSearchUtilit
             if (luceneSearchUtility != null)
             {
                 //The DirectoryReader and IndexSearcher have to be assigned after commiting indexing changes and before executing the Searcher.Search() method, otherwise new indexed info won't be reflected
-                luceneSearchUtility.dirReader = luceneSearchUtility.writer?.GetReader(applyAllDeletes: true);
-                if (luceneSearchUtility.dirReader == null) return null;
-
+                if (luceneSearchUtility.writer != null)
+                {
+                    luceneSearchUtility.dirReader = luceneSearchUtility.writer.GetReader(applyAllDeletes: true);
+                }
+                else
+                {
+                    luceneSearchUtility.dirReader = DirectoryReader.Open(luceneSearchUtility.indexDir);
+                }
                 luceneSearchUtility.Searcher = new IndexSearcher(luceneSearchUtility.dirReader);
 
                 string searchTerm = search.Trim();