Fix external log stream init timestamp (#318)

Author: chenziliang

src/Storages/ExternalStream/Log/FileLog.cpp

@@ -9,10 +9,10 @@
 #include <Storages/IStorage.h>
 #include <Storages/SelectQueryInfo.h>
 #include <Storages/Streaming/storageUtil.h>
+#include <Common/logger_useful.h>
 
 #include <boost/algorithm/string.hpp>
 #include <re2/re2.h>
-#include <Poco/Logger.h>
 
 namespace DB
 {
@@ -148,6 +148,15 @@ FileLogSource::FileContainer FileLog::searchForCandidates()
             candidates.erase(candidates.begin());
     }
 
+    LOG_INFO(
+        log,
+        "Using {} regexes: {} to search log_dir={} with start_timestamp={}, found {} candidates",
+        file_regexes.size(),
+        settings->log_files.value,
+        settings->log_dir.value,
+        start_timestamp,
+        candidates.size());
+
     if (!candidates.empty())
         /// Update start timestamp for next scan
         start_timestamp = candidates.rbegin()->first;

src/Storages/ExternalStream/Log/FileLog.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "FileLogSource.h"
+#include <Storages/ExternalStream/Log/FileLogSource.h>
 
 #include <Storages/ExternalStream/ExternalStreamSettings.h>
 #include <Storages/ExternalStream/StorageExternalStreamImpl.h>
@@ -55,7 +55,7 @@ class FileLog final : public StorageExternalStreamImpl
     std::unique_ptr<re2::RE2> timestamp_regex;
     std::unique_ptr<re2::RE2> linebreaker_regex;
 
-    Int64 start_timestamp;
+    Int64 start_timestamp = 0;
 
     Poco::Logger * log;
 };

src/Storages/ExternalStream/Log/FileLogSource.cpp

@@ -1,6 +1,6 @@
-#include "FileLogSource.h"
-#include "FileLog.h"
-#include "fileLastModifiedTime.h"
+#include <Storages/ExternalStream/Log/FileLog.h>
+#include <Storages/ExternalStream/Log/FileLogSource.h>
+#include <Storages/ExternalStream/Log/fileLastModifiedTime.h>
 
 #include <Storages/ExternalStream/BreakLines.h>
 #include <base/ClockUtils.h>
@@ -15,10 +15,10 @@ namespace DB
 {
 namespace ErrorCodes
 {
-    extern const int FILE_DOESNT_EXIST;
-    extern const int CANNOT_OPEN_FILE;
-    extern const int CANNOT_FSTAT;
-    extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR;
+extern const int FILE_DOESNT_EXIST;
+extern const int CANNOT_OPEN_FILE;
+extern const int CANNOT_FSTAT;
+extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR;
 }
 
 FileLogSource::FileLogSource(
@@ -75,7 +75,6 @@ Chunk FileLogSource::generate()
             if (buffer.empty())
                 buffer.resize(4 * 1024 * 1024);
 
-
             if (!handleCurrentFile())
                 return head_chunk.clone();
         }
@@ -183,6 +182,8 @@ void FileLogSource::checkNewFiles()
     }
 
     auto new_files{file_log->searchForCandidates()};
+    last_check = MonotonicMilliseconds::now();
+
     if (new_files.empty())
     {
         /// FIXME, notification