Adjust index writer options and commit scheduler timer

Author: Shazwazza

src/Examine.Lucene/LoggingInfoStream.cs

@@ -5,7 +5,10 @@ namespace Examine.Lucene
 {
     internal class LoggingInfoStream<T> : InfoStream
     {
-        public LoggingInfoStream(ILogger<T> logger) => Logger = logger;
+        public LoggingInfoStream(ILogger<T> logger)
+        {
+            Logger = logger;
+        }
 
         public ILogger<T> Logger { get; }
 

src/Examine.Lucene/Providers/LuceneIndex.cs

@@ -354,7 +354,7 @@ public void EnsureIndex(bool forceOverwrite)
                             //In this case we need to initialize a writer and continue as normal.
                             //Since we are already inside the writer lock and it is null, we are allowed to 
                             // make this call with out using GetIndexWriter() to do the initialization.
-                            _writer ??= CreateIndexWriterInternal();
+                            _writer ??= CreateIndexWriterWithLockCheck();
 
                             //We're forcing an overwrite, 
                             // this means that we need to cancel all operations currently in place,
@@ -416,18 +416,7 @@ private void CreateNewIndex(Directory dir)
                 }
 
                 //create the writer (this will overwrite old index files)
-                var writerConfig = new IndexWriterConfig(LuceneInfo.CurrentVersion, FieldAnalyzer)
-                {
-                    OpenMode = OpenMode.CREATE,
-                    MergeScheduler = new ErrorLoggingConcurrentMergeScheduler(Name,
-                        (s, e) => OnIndexingError(new IndexingErrorEventArgs(this, s, "-1", e)))
-                };
-
-                // TODO: With NRT, we should apparently use this but there is no real implementation of it!?
-                // https://stackoverflow.com/questions/12271614/lucene-net-indexwriter-setmergedsegmentwarmer
-                //writerConfig.SetMergedSegmentWarmer(new SimpleMergedSegmentWarmer())
-
-                writer = new IndexWriter(dir, writerConfig);
+                writer = CreateIndexWriterWithOpenMode(dir, OpenMode.CREATE);
 
                 // Required to remove old index files which can be problematic
                 // if they remain in the index folder when replication is attempted.
@@ -793,7 +782,7 @@ private class IndexCommiter : DisposableObjectSlim
             private DateTime _timestamp;
             private Timer _timer;
             private readonly object _locker = new object();
-            private const int WaitMilliseconds = 2000;
+            private const int WaitMilliseconds = 1000;
 
             /// <summary>
             /// The maximum time period that will elapse until we must commit (5 mins)
@@ -926,20 +915,21 @@ private bool ProcessQueueItem(IndexOperation item)
         /// Used to create an index writer - this is called in GetIndexWriter (and therefore, GetIndexWriter should not be overridden)
         /// </summary>
         /// <returns></returns>
-        private TrackingIndexWriter CreateIndexWriterInternal()
+        private TrackingIndexWriter CreateIndexWriterWithLockCheck()
         {
             var dir = GetLuceneDirectory();
 
-            // Unfortunatley if the appdomain is taken down this will remain locked, so we can 
+            // Unfortunately if the appdomain is taken down this will remain locked, so we can 
             // ensure that it's unlocked here in that case.
             try
             {
                 if (IsLocked(dir))
                 {
-                    if (_logger.IsEnabled(LogLevel.Debug))
+                    if (_logger.IsEnabled(LogLevel.Information))
                     {
                         _logger.LogDebug("Forcing index {IndexName} to be unlocked since it was left in a locked state", Name);
                     }
+
                     //unlock it!
                     Unlock(dir);
                 }
@@ -962,46 +952,7 @@ private TrackingIndexWriter CreateIndexWriterInternal()
         /// </summary>
         /// <param name="d"></param>
         /// <returns></returns>
-        protected virtual IndexWriter CreateIndexWriter(Directory d)
-        {
-            if (d == null)
-            {
-                throw new ArgumentNullException(nameof(d));
-            }
-
-            var writer = new IndexWriter(d, new IndexWriterConfig(LuceneInfo.CurrentVersion, FieldAnalyzer)
-            {
-                IndexDeletionPolicy = _options.IndexDeletionPolicy ?? new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()),
-#if FULLDEBUG
-
-            //If we want to enable logging of lucene output....
-            //It is also possible to set a default InfoStream on the static IndexWriter class
-            InfoStream =
-
-            _logOutput?.Close();
-            if (LuceneIndexFolder != null)
-            {
-                try
-                {
-                    System.IO.Directory.CreateDirectory(LuceneIndexFolder.FullName);
-                    _logOutput = new FileStream(Path.Combine(LuceneIndexFolder.FullName, DateTime.UtcNow.ToString("yyyy-MM-dd") + ".log"), FileMode.Append);
-           
-            
-                }
-                catch (Exception ex)
-                {
-                    //if an exception is thrown here we won't worry about it, it will mean we cannot create the log file
-                }
-            }
-
-#endif
-
-                MergeScheduler = new ErrorLoggingConcurrentMergeScheduler(Name,
-                    (s, e) => OnIndexingError(new IndexingErrorEventArgs(this, s, "-1", e)))
-            });
-
-            return writer;
-        }
+        protected virtual IndexWriter CreateIndexWriter(Directory d) => CreateIndexWriterWithOpenMode(d, OpenMode.CREATE_OR_APPEND);
 
         /// <summary>
         /// Gets the TrackingIndexWriter for the current directory
@@ -1025,7 +976,7 @@ public TrackingIndexWriter IndexWriter
                     Monitor.Enter(_writerLocker);
                     try
                     {
-                        _writer ??= CreateIndexWriterInternal();
+                        _writer ??= CreateIndexWriterWithLockCheck();
                     }
                     finally
                     {
@@ -1042,6 +993,65 @@ public TrackingIndexWriter IndexWriter
 
         #region Private
 
+        private IndexWriter CreateIndexWriterWithOpenMode(Directory d, OpenMode openMode)
+        {
+            if (d == null)
+            {
+                throw new ArgumentNullException(nameof(d));
+            }
+            var writerConfig = new IndexWriterConfig(LuceneInfo.CurrentVersion, FieldAnalyzer)
+            {
+                IndexDeletionPolicy = _options.IndexDeletionPolicy ?? new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()),
+                OpenMode = openMode,
+
+                //https://solr.apache.org/guide/solr/latest/configuration-guide/index-segments-merging.html#mergepolicyfactory
+                MergePolicy = new TieredMergePolicy
+                {
+                    MaxMergeAtOnce = 10,
+                    SegmentsPerTier = 10,
+                    ForceMergeDeletesPctAllowed = 10.0f,
+                    MaxMergedSegmentMB = 5000
+                },
+#if FULLDEBUG
+
+            //If we want to enable logging of lucene output....
+            //It is also possible to set a default InfoStream on the static IndexWriter class
+            InfoStream =
+
+            _logOutput?.Close();
+            if (LuceneIndexFolder != null)
+            {
+                try
+                {
+                    System.IO.Directory.CreateDirectory(LuceneIndexFolder.FullName);
+                    _logOutput = new FileStream(Path.Combine(LuceneIndexFolder.FullName, DateTime.UtcNow.ToString("yyyy-MM-dd") + ".log"), FileMode.Append);
+           
+            
+                }
+                catch (Exception ex)
+                {
+                    //if an exception is thrown here we won't worry about it, it will mean we cannot create the log file
+                }
+            }
+
+#endif
+
+                MergeScheduler = new ErrorLoggingConcurrentMergeScheduler(Name,
+                    (s, e) => OnIndexingError(new IndexingErrorEventArgs(this, s, "-1", e)))
+            };
+
+            if (_options.NrtEnabled)
+            {
+                // With NRT, we should apparently use this but there is no real implementation of it!?
+                // https://stackoverflow.com/questions/12271614/lucene-net-indexwriter-setmergedsegmentwarmer
+                writerConfig.MergedSegmentWarmer = new SimpleMergedSegmentWarmer(new LoggingInfoStream<LuceneIndex>(_logger));
+            }
+
+            var writer = new IndexWriter(d, writerConfig);
+
+            return writer;
+        }
+
         private LuceneSearcher CreateSearcher()
         {
             var name = Name;