fixes #192 - utf-16 handling for stream-based constructor

Author: d99kris

CMakeLists.txt

@@ -184,6 +184,9 @@ if(RAPIDCSV_BUILD_TESTS)
   add_unit_test(test098)
   add_unit_test(test099)
   add_unit_test(test100)
+  if(HAS_CODECVT)
+    add_unit_test(test101)
+  endif()
 
   # perf tests
   add_perf_test(ptest001)

src/rapidcsv.h

@@ -2,7 +2,7 @@
  * rapidcsv.h
  *
  * URL:      https://github.com/d99kris/rapidcsv
- * Version:  8.87
+ * Version:  8.88
  *
  * Copyright (C) 2017-2025 Kristofer Berggren
  * All rights reserved.
@@ -1532,24 +1532,28 @@ namespace rapidcsv
         mIsUtf16 = true;
         mIsLE = (bom2b == bomU16le);
 
-        std::wifstream wstream;
-        wstream.exceptions(std::wifstream::failbit | std::wifstream::badbit);
-        wstream.open(mPath, std::ios::binary);
-        if (mIsLE)
-        {
-          wstream.imbue(std::locale(wstream.getloc(),
-                                    new std::codecvt_utf16<wchar_t, 0x10ffff,
-                                                           static_cast<std::codecvt_mode>(std::consume_header |
-                                                                                          std::little_endian)>));
-        }
-        else
+        std::vector<char> buffer(static_cast<size_t>(length));
+        pStream.read(buffer.data(), length);
+
+        const std::wstring& utf16 = [&]()
         {
-          wstream.imbue(std::locale(wstream.getloc(),
-                                    new std::codecvt_utf16<wchar_t, 0x10ffff,
-                                                           std::consume_header>));
-        }
-        std::wstringstream wss;
-        wss << wstream.rdbuf();
+          if (mIsLE)
+          {
+            const std::codecvt_mode mode =
+              static_cast<std::codecvt_mode>(std::consume_header | std::little_endian);
+            std::wstring_convert<std::codecvt_utf16<wchar_t, 0x10ffff, mode>> utf16conv;
+            return utf16conv.from_bytes(buffer.data(), buffer.data() + length);
+          }
+          else
+          {
+            const std::codecvt_mode mode =
+              static_cast<std::codecvt_mode>(std::consume_header);
+            std::wstring_convert<std::codecvt_utf16<wchar_t, 0x10ffff, mode>> utf16conv;
+            return utf16conv.from_bytes(buffer.data(), buffer.data() + length);
+          }
+        }();
+
+        std::wstringstream wss(utf16);
         std::string utf8 = ToString(wss.str());
         std::stringstream ss(utf8);
         ParseCsv(ss, static_cast<std::streamsize>(utf8.size()));

tests/test101.cpp

@@ -0,0 +1,46 @@
+// test101.cpp - read UTF-16 LE stream
+
+#include <rapidcsv.h>
+#include "unittest.h"
+
+int main()
+{
+  int rv = 0;
+
+  const unsigned char u16le[] =
+  {
+    0xff, 0xfe,
+    0x2d, 0x00, 0x2c, 0x00, 0x41, 0x00, 0x2c, 0x00, 0x42, 0x00, 0x2c, 0x00,
+    0x43, 0x00, 0x0a, 0x00, 0x31, 0x00, 0x2c, 0x00, 0x33, 0x00, 0x2c, 0x00,
+    0x39, 0x00, 0x2c, 0x00, 0x38, 0x00, 0x31, 0x00, 0x0a, 0x00, 0x32, 0x00,
+    0x2c, 0x00, 0x34, 0x00, 0x2c, 0x00, 0x31, 0x00, 0x36, 0x00, 0x2c, 0x00,
+    0x32, 0x00, 0x35, 0x00, 0x36, 0x00, 0x0a, 0x00
+  };
+  const unsigned int u16le_len = 58;
+
+  std::string csv(reinterpret_cast<const char*>(u16le), u16le_len);
+  // "-,A,B,C\n"
+  // "1,3,9,81\n"
+  // "2,4,16,256\n"
+
+  try
+  {
+    // stream from string
+    std::istringstream sstream(csv);
+    rapidcsv::Document doc(sstream, rapidcsv::LabelParams(0, 0));
+    unittest::ExpectEqual(int, doc.GetCell<int>(0, 0), 3);
+    unittest::ExpectEqual(int, doc.GetCell<int>(1, 0), 9);
+    unittest::ExpectEqual(int, doc.GetCell<int>(2, 0), 81);
+
+    unittest::ExpectEqual(std::string, doc.GetCell<std::string>("A", "2"), "4");
+    unittest::ExpectEqual(std::string, doc.GetCell<std::string>("B", "2"), "16");
+    unittest::ExpectEqual(std::string, doc.GetCell<std::string>("C", "2"), "256");
+  }
+  catch (const std::exception& ex)
+  {
+    std::cout << ex.what() << std::endl;
+    rv = 1;
+  }
+
+  return rv;
+}