bumps version, writes popvcf.gz when using popvcf encoding

Author: hannespetur

.github/workflows/cancel.yaml

@@ -10,6 +10,7 @@ example_test.vcf
 *.log
 .nfs*
 *.bak
+*.git*.sh
 
 # ggtags
 /GPATH

.gitignore

@@ -7,7 +7,7 @@ include(CheckIncludeFileCXX)
 # The version number
 set (graphtyper_VERSION_MAJOR 2)
 set (graphtyper_VERSION_MINOR 7)
-set (graphtyper_VERSION_PATCH 3)
+set (graphtyper_VERSION_PATCH 4)
 
 # Graphtyper's headers
 include_directories("${CMAKE_CURRENT_SOURCE_DIR}/include")
@@ -302,6 +302,15 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/catch/single_include/)
 enable_testing(true)
 add_subdirectory(test)
 
+############################################################################
+#  ARCHIVE
+############################################################################
+add_custom_target(archive
+    COMMAND sh -c "bash .git-archive-all.sh --format tar.gz --prefix graphtyper-v${graphtyper_VERSION}/ ${CMAKE_CURRENT_BINARY_DIR}/graphtyper-v${graphtyper_VERSION}.tar.gz"
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMENT "Generating an archive which includes submodules."
+    VERBATIM)
+
 #############################################################################
 ##  CLANG-FORMAT
 #############################################################################

CMakeLists.txt

@@ -25,7 +25,7 @@ class BGZF_stream
 public:
   std::ostringstream ss;
   std::string buffer_in;
-  std::string buffer_out;
+  std::vector<char> buffer_out;
 
   BGZF_stream() = default;
   ~BGZF_stream(); // custom
@@ -70,12 +70,11 @@ inline void BGZF_stream::flush()
   if (Options::const_instance()->is_on_final_output && Options::const_instance()->encoding == 'p')
   {
     assert(buffer_in.size() >= ed.i);
-    ed.bytes_read = buffer_in.size();
     popvcf::encode_buffer(buffer_out, buffer_in, ed);
 
     if (fp == nullptr)
     {
-      std::cout << buffer_out; // Write uncompressed to stdout
+      std::copy(buffer_out.begin(), buffer_out.end(), std::ostream_iterator<char>(std::cout));
     }
     else if (buffer_out.size() > 0)
     {
@@ -94,7 +93,6 @@ inline void BGZF_stream::flush()
       }
     }
 
-    buffer_in.resize(ed.i);
     buffer_out.resize(0);
   }
   else

include/graphtyper/utilities/bgzf_stream.hpp

@@ -107,4 +107,14 @@ void print_log(log_severity const severity, args_t &&... args)
   *log_singleton->sink << '\n';
 }
 
+#ifdef NDEBUG // Release build
+#  define print_debug(...) ((void)0)
+#else // not NDEBUG (=> debug build)
+#  define print_debug(...) print_log(gyper::log_severity::debug, __VA_ARGS__)
+#endif // NDEBUG
+
+#define print_info(...) print_log(gyper::log_severity::info, __VA_ARGS__)
+#define print_warning(...) print_log(gyper::log_severity::warning, __VA_ARGS__)
+#define print_error(...) print_log(gyper::log_severity::error, __VA_ARGS__)
+
 } // namespace gyper

include/graphtyper/utilities/logging.hpp

@@ -37,6 +37,7 @@ class Options
   bool uncompressed_sample_names{false};
   char encoding{'v'};             // v VCF, p popVCF
   bool is_on_final_output{false}; // Set as true before writing out final output
+  int bgzf_compression_level{9};
 
   /****
    * FILTERING OPTIONS

include/graphtyper/utilities/options.hpp

@@ -12,81 +12,83 @@
 
 namespace popvcf
 {
-//! Buffer size when encoding
-long constexpr ENC_BUFFER_SIZE{4 * 65536};
-
-//! Data type of an array buffer
-using Tarray_buf = std::array<char, ENC_BUFFER_SIZE>; //!< Buffer type
-
 class EncodeData
 {
 public:
-  std::size_t bytes_read{0};
-  std::size_t field{0};         // current vcf field
-  std::size_t b{0};             // begin index in buffer_in
-  std::size_t i{b};             // index in buffer_in
-  bool header_line{true};       //!< True iff in header line
-  bool no_previous_line{false}; //! Set to skip using previous line
-
-  std::string prev_contig{};
-  int64_t prev_pos{0};
-  uint32_t prev_num_unique_fields{};
+  std::size_t field{0};   //!< current vcf field.
+  std::size_t in_size{0}; //!< Size of inut buffer.
+  std::size_t b{0};       //!< begin index in buffer_in
+  std::size_t i{b};       //!< index in buffer_in
+  bool header_line{true}; //!< True iff in header line
+
+  /* Data fields from previous line. */
   std::vector<std::string> prev_unique_fields{};
   std::vector<uint32_t> prev_field2uid{};
   phmap::flat_hash_map<std::string, uint32_t> prev_map_to_unique_fields{};
 
+  /* Data fields from current line. */
   std::string contig{};
   int64_t pos{0};
-  uint32_t num_unique_fields{};
+  int32_t n_alt{-1};
   std::vector<std::string> unique_fields{};
   std::vector<uint32_t> field2uid{};
   phmap::flat_hash_map<std::string, uint32_t> map_to_unique_fields{};
 
-  inline void clear_line()
-  {
-    field = 0; // reset field index
+  /* Data fields for the next line. */
+  std::string next_contig{};
+  int64_t next_pos{0};
 
-    if (not no_previous_line)
+  inline void clear_line(std::string && next_contig, int64_t next_pos, int32_t next_n_alt)
+  {
+    if (next_contig != contig || (next_pos / 10000) != (pos / 10000))
+    {
+      /// Previous line is not available, clear values
+      prev_unique_fields.resize(0);
+      prev_field2uid.resize(0);
+      prev_map_to_unique_fields.clear();
+    }
+    else if (next_n_alt == n_alt)
     {
-      std::swap(prev_contig, contig);
-      prev_pos = pos;
-      prev_num_unique_fields = num_unique_fields;
+      /// Only swap out from this line if we have the same amount of alts
       std::swap(prev_unique_fields, unique_fields);
       std::swap(prev_field2uid, field2uid);
       std::swap(prev_map_to_unique_fields, map_to_unique_fields);
     }
 
-    contig.resize(0);
-    pos = 0;
-    num_unique_fields = 0;
+    /// Clear data from this line for the next
+    contig = std::move(next_contig);
+    pos = next_pos;
+    n_alt = next_n_alt;
     unique_fields.resize(0);
     field2uid.resize(0);
-    map_to_unique_fields.clear(); // clear map every line
+    map_to_unique_fields.clear();
   }
 };
 
-template <typename Tint, typename Tbuffer_out>
-inline void to_chars(Tint char_val, Tbuffer_out & buffer_out)
+template <typename Tbuffer_in>
+inline void set_input_size(Tbuffer_in & buffer_in, EncodeData & ed)
 {
-  while (char_val >= CHAR_SET_SIZE)
-  {
-    auto rem = char_val % CHAR_SET_SIZE;
-    char_val = char_val / CHAR_SET_SIZE;
-    buffer_out.push_back(int_to_ascii(rem));
-  }
+  ed.in_size = buffer_in.size();
+}
 
-  assert(char_val < CHAR_SET_SIZE);
-  buffer_out.push_back(int_to_ascii(char_val));
+template <>
+inline void set_input_size(Tenc_array_buf & /*buffer_in*/, EncodeData & /*ed*/)
+{
+  // Do nothing.
+  // NOTE: dd.in_size must be set prior to calling decode_buffer in arrays
 }
 
 //! Encodes an input buffer. Output is written in \a buffer_out.
 template <typename Tbuffer_out, typename Tbuffer_in>
 inline void encode_buffer(Tbuffer_out & buffer_out, Tbuffer_in & buffer_in, EncodeData & ed)
 {
-  buffer_out.reserve(ENC_BUFFER_SIZE / 2);
+  set_input_size(buffer_in, ed);
+  buffer_out.reserve(ENC_BUFFER_SIZE);
   std::size_t constexpr N_FIELDS_SITE_DATA{9}; // how many fields of the VCF contains site data
+  std::string next_contig{};
+  int64_t next_pos{0};
 
-  while (ed.i < ed.bytes_read)
+  while (ed.i < ed.in_size)
   {
     char const b_in = buffer_in[ed.i];
 
@@ -96,35 +98,31 @@ inline void encode_buffer(Tbuffer_out & buffer_out, Tbuffer_in & buffer_in, Enco
       continue; // we are in a vcf field
     }
 
-    if (ed.field == 0)
+    if (ed.field == 0) /*CHROM field*/
     {
       // check if in header line and store contig
-      if (buffer_in[ed.b] == '#')
-      {
-        ed.header_line = true;
-      }
-      else
-      {
-        ed.header_line = false;
-        ed.contig = std::string(&buffer_in[ed.b], ed.i - ed.b);
-      }
+      ed.header_line = buffer_in[ed.b] == '#'; // check if in header line
+
+      if (not ed.header_line)
+        next_contig.assign(&buffer_in[ed.b], ed.i - ed.b);
     }
-    else if (ed.header_line == false && ed.field == 1)
+    else if (not ed.header_line)
     {
-      std::from_chars(&buffer_in[ed.b], &buffer_in[ed.i], ed.pos);
-
-      if (ed.contig != ed.prev_contig || (ed.pos / 10000) != (ed.prev_pos / 10000))
+      if (ed.field == 1) /*POS field*/
+      {
+        std::from_chars(&buffer_in[ed.b], &buffer_in[ed.i], next_pos);
+      }
+      else if (ed.field == 4) /*ALT field*/
       {
-        // previous line is not available
-        ed.prev_num_unique_fields = 0;
-        ed.prev_map_to_unique_fields.clear();
+        int32_t next_n_alt = std::count(&buffer_in[ed.b], &buffer_in[ed.i], ',');
+        ed.clear_line(std::move(next_contig), next_pos, next_n_alt);
       }
     }
 
     if (ed.header_line || ed.field < N_FIELDS_SITE_DATA)
     {
-      ++ed.i; // adds '\t' or '\n'
-      std::copy(&buffer_in[ed.b], &buffer_in[ed.i], std::back_inserter(buffer_out));
+      ++ed.i; // adds '\t' or '\n' and then insert the field to the output buffer
+      buffer_out.insert(buffer_out.end(), &buffer_in[ed.b], &buffer_in[ed.i]);
     }
     else
     {
@@ -135,25 +133,25 @@ inline void encode_buffer(Tbuffer_out & buffer_out, Tbuffer_in & buffer_in, Enco
       auto insert_it = ed.map_to_unique_fields.insert(
         std::pair<std::string, uint32_t>(std::piecewise_construct,
                                          std::forward_as_tuple(&buffer_in[ed.b], ed.i - ed.b),
-                                         std::forward_as_tuple(ed.num_unique_fields)));
+                                         std::forward_as_tuple(ed.unique_fields.size())));
 
       long const field_idx = ed.field - N_FIELDS_SITE_DATA;
       assert(field_idx == static_cast<long>(ed.field2uid.size()));
 
       if (insert_it.second == true)
       {
         ed.field2uid.push_back(ed.unique_fields.size());
-        ++ed.num_unique_fields; // unique field
         ed.unique_fields.emplace_back(&buffer_in[ed.b], ed.i - ed.b);
 
-        assert(ed.num_unique_fields == static_cast<long>(ed.unique_fields.size()));
-
         if (field_idx < static_cast<long>(ed.prev_field2uid.size()) &&
             ed.prev_unique_fields[ed.prev_field2uid[field_idx]] == ed.unique_fields[insert_it.first->second])
         {
           /* Case 0: unique and same as above. */
           buffer_out.push_back('$');
-          buffer_out.push_back(buffer_in[ed.i]);
+
+          if (b_in == '\n') /* never skip newline */
+            buffer_out.push_back('\n');
+
           ++ed.i;
         }
         else
@@ -164,8 +162,8 @@ inline void encode_buffer(Tbuffer_out & buffer_out, Tbuffer_in & buffer_in, Enco
           if (prev_find_it == ed.prev_map_to_unique_fields.end())
           {
             /* Case 1: Field is unique in the current line and is not in the previous line. */
-            ++ed.i;                                                                        // adds '\t' or '\n'
-            std::copy(&buffer_in[ed.b], &buffer_in[ed.i], std::back_inserter(buffer_out)); // just copy as is
+            ++ed.i; // adds '\t' or '\n'
+            buffer_out.insert(buffer_out.end(), &buffer_in[ed.b], &buffer_in[ed.i]);
           }
           else
           {
@@ -186,7 +184,10 @@ inline void encode_buffer(Tbuffer_out & buffer_out, Tbuffer_in & buffer_in, Enco
         {
           /* Case 3: Field is not unique and same has the field above. */
           buffer_out.push_back('&');
-          buffer_out.push_back(buffer_in[ed.i]);
+
+          if (b_in == '\n') /* never skip newline */
+            buffer_out.push_back('\n');
+
           ++ed.i;
         }
         else
@@ -207,7 +208,7 @@ inline void encode_buffer(Tbuffer_out & buffer_out, Tbuffer_in & buffer_in, Enco
 
     // check if we need to clear line or increment field
     if (b_in == '\n')
-      ed.clear_line();
+      ed.field = 0; // reset field index
     else
       ++ed.field;
   } // ends inner loop
@@ -216,7 +217,8 @@ inline void encode_buffer(Tbuffer_out & buffer_out, Tbuffer_in & buffer_in, Enco
   std::copy(&buffer_in[ed.b], &buffer_in[ed.i], &buffer_in[0]);
   ed.i = ed.i - ed.b;
   ed.b = 0;
-  ed.bytes_read = ed.i;
+  ed.in_size = ed.i;
+  resize_input_buffer(buffer_in, ed.i);
 }
 
 //! Encode a gzipped file and write to stdout
@@ -225,7 +227,6 @@ void encode_file(std::string const & input_fn,
                  std::string const & output_fn,
                  std::string const & output_mode,
                  bool const is_bgzf_output,
-                 int const compression_threads,
-                 bool const no_previous_line);
+                 int const compression_threads);
 
 } // namespace popvcf