[CORE] Enable and improve buffered IO in LocalFile to reduce its cost by up to 90% (#1299)

Author: xezon

Core/GameEngine/Include/Common/GameDefines.h

@@ -47,3 +47,9 @@
 #ifndef ENABLE_CONFIGURABLE_SHROUD
 #define ENABLE_CONFIGURABLE_SHROUD (1) // When enabled, the GlobalData contains a field to turn on/off the shroud, otherwise shroud is always enabled
 #endif
+
+// Enable buffered IO in File System. Was disabled in retail game.
+// Buffered IO generally is much faster than unbuffered for small reads and writes.
+#ifndef USE_BUFFERED_IO
+#define USE_BUFFERED_IO (1)
+#endif

Core/GameEngine/Include/Common/LocalFile.h

@@ -54,12 +54,7 @@
 
 #include "Common/file.h"
 
-// srj sez: this was purely an experiment in optimization.
-// at the present time, it doesn't appear to be a good one.
-// but I am leaving the code in for now.
-// if still present in 2003, please nuke.
-#define NO_USE_BUFFERED_IO
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 #include <stdio.h>
 #endif
 
@@ -85,11 +80,12 @@ class LocalFile : public File
 	MEMORY_POOL_GLUE_ABC(LocalFile)		
 	private:
 
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
+		// srj sez: this was purely an experiment in optimization.
+		// at the present time, it doesn't appear to be a good one.
+		// TheSuperHackers @info It is a good optimization and will be
+		// significantly faster than unbuffered IO with small reads and writes.
 		FILE* m_file;
-
-		enum { BUF_SIZE = 32768 };
-		char m_vbuf[BUF_SIZE];	
 #else
 		int m_handle;											///< Local C file handle
 #endif

Core/GameEngine/Source/Common/System/LocalFile.cpp

@@ -107,7 +107,7 @@ static Int s_totalOpen = 0;
 //=================================================================
 
 LocalFile::LocalFile()
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 	: m_file(NULL)
 #else
 	: m_handle(-1)
@@ -127,7 +127,7 @@ LocalFile::LocalFile()
 
 LocalFile::~LocalFile()
 {
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 	if (m_file)
 	{
 		fclose(m_file);
@@ -167,61 +167,54 @@ Bool LocalFile::open( const Char *filename, Int access )
 	}
 
 	/* here we translate WSYS file access to the std C equivalent */
-#ifdef USE_BUFFERED_IO
-	char mode[32];
-	char* m = mode;
-
-	if (m_access & APPEND)
+#if USE_BUFFERED_IO
+
+	// r    open for reading (The file must exist)
+	// w    open for writing (creates file if it doesn't exist). Deletes content and overwrites the file.
+	// a    open for appending (creates file if it doesn't exist).
+	// r+   open for reading and writing (The file must exist).
+	// w+   open for reading and writing.
+	//      If file exists deletes content and overwrites the file, otherwise creates an empty new file.
+	// a+   open for reading and writing (append if file exists).
+
+	const Bool write     = (m_access & WRITE) != 0;
+	const Bool readwrite = (m_access & READWRITE) == READWRITE;
+	const Bool append    = (m_access & APPEND) != 0;
+	const Bool create    = (m_access & CREATE) != 0;
+	const Bool truncate  = (m_access & TRUNCATE) != 0;
+	const Bool binary    = (m_access & BINARY) != 0;
+
+	const Char *mode = NULL;
+
+	// Mode string selection (mimics _open flag combinations)
+	// TEXT is implicit for fopen if 'b' is not present
+	// READ is implicit here if not READWRITE or WRITE
+	if (readwrite)
 	{
-		DEBUG_CRASH(("not yet supported by buffered mode"));
-	}
-
-	if (m_access & TRUNCATE)
-	{
-		DEBUG_CRASH(("not yet supported by buffered mode"));
-	}
-
-	if((m_access & READWRITE ) == READWRITE )
-	{
-		if (m_access & CREATE)
-		{
-			*m++ = 'w';
-			*m++ = '+';
-		}
+		if (append)
+			mode = binary ? "a+b" : "a+";
+		else if (truncate || create)
+			mode = binary ? "w+b" : "w+";
 		else
-		{
-			*m++ = 'r';
-			*m++ = '+';
-		}
+			mode = binary ? "r+b" : "r+";
 	}
-	else if(m_access & WRITE)
-	{
-		*m++ = 'w';
-	}
-	else
-	{
-		*m++ = 'r';
-		DEBUG_ASSERTCRASH(!(m_access & TRUNCATE), ("cannot truncate with read-only"));
-	}
-
-	if (m_access & TEXT)
+	else if (write)
 	{
-		*m++ = 't';
+		if (append)
+			mode = binary ? "ab" : "a";
+		else
+			mode = binary ? "wb" : "w";
 	}
-	if (m_access & BINARY)
+	else // implicitly read-only
 	{
-		*m++ = 'b';
+		mode = binary ? "rb" : "r";
 	}
 
-	*m++ = 0;
-
 	m_file = fopen(filename, mode);
 	if (m_file == NULL)
 	{
 		goto error;
 	}
-	
-	//setvbuf(m_file, m_vbuf, _IOFBF, sizeof(BUF_SIZE));
 
 #else
 
@@ -257,7 +250,7 @@ Bool LocalFile::open( const Char *filename, Int access )
 		flags |= _O_WRONLY;
 		flags |= _O_CREAT;
 	}
-	else
+	else // implicitly read-only
 	{
 		flags |= _O_RDONLY;
 	}
@@ -318,15 +311,15 @@ Int LocalFile::read( void *buffer, Int bytes )
 
 	if (buffer == NULL) 
 	{
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 		fseek(m_file, bytes, SEEK_CUR);
 #else
 		_lseek(m_handle, bytes, SEEK_CUR);
 #endif
 		return bytes;
 	}
 
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 	Int ret = fread(buffer, 1, bytes, m_file);
 #else
 	Int ret = _read( m_handle, buffer, bytes );
@@ -347,7 +340,7 @@ Int LocalFile::write( const void *buffer, Int bytes )
 		return -1;
 	}
 
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 	Int ret = fwrite(buffer, 1, bytes, m_file);
 #else
 	Int ret = _write( m_handle, buffer, bytes );
@@ -366,21 +359,21 @@ Int LocalFile::seek( Int pos, seekMode mode)
 	switch( mode )
 	{
 		case START:
+			DEBUG_ASSERTCRASH(pos >= 0, ("LocalFile::seek - pos must be >= 0 when seeking from the beginning of the file"));
 			lmode = SEEK_SET;
 			break;
 		case CURRENT:
 			lmode = SEEK_CUR;
 			break;
 		case END:
-			DEBUG_ASSERTCRASH(pos <= 0, ("LocalFile::seek - pos should be <= 0 for a seek starting at the end of the file"));
 			lmode = SEEK_END;
 			break;
 		default:
-			// bad seek mode
+			DEBUG_CRASH(("LocalFile::seek - bad seek mode"));
 			return -1;
 	}
 
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 	Int ret = fseek(m_file, pos, lmode);
 	if (ret == 0)
 		return ftell(m_file);
@@ -406,7 +399,7 @@ Bool LocalFile::scanInt(Int &newInt)
 
 	// skip preceding non-numeric characters
 	do {
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 		val = fread(&c, 1, 1, m_file);
 #else
 		val = _read( m_handle, &c, 1);
@@ -419,7 +412,7 @@ Bool LocalFile::scanInt(Int &newInt)
 
 	do {
 		tempstr.concat(c);
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 		val = fread(&c, 1, 1, m_file);
 #else
 		val = _read( m_handle, &c, 1);
@@ -428,7 +421,7 @@ Bool LocalFile::scanInt(Int &newInt)
 
 	// put the last read char back, since we didn't use it.
 	if (val != 0) {
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 		fseek(m_file, -1, SEEK_CUR);
 #else
 		_lseek(m_handle, -1, SEEK_CUR);
@@ -454,7 +447,7 @@ Bool LocalFile::scanReal(Real &newReal)
 
 	// skip the preceding white space
 	do {
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 		val = fread(&c, 1, 1, m_file);
 #else
 		val = _read( m_handle, &c, 1);
@@ -470,15 +463,15 @@ Bool LocalFile::scanReal(Real &newReal)
 		if (c == '.') {
 			sawDec = TRUE;
 		}
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 		val = fread(&c, 1, 1, m_file);
 #else
 		val = _read(m_handle, &c, 1);
 #endif
 	} while ((val != 0) && (((c >= '0') && (c <= '9')) || ((c == '.') && !sawDec)));
 
 	if (val != 0) {
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 		fseek(m_file, -1, SEEK_CUR);
 #else
 		_lseek(m_handle, -1, SEEK_CUR);
@@ -503,7 +496,7 @@ Bool LocalFile::scanString(AsciiString &newString)
 
 	// skip the preceding whitespace
 	do {
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 		val = fread(&c, 1, 1, m_file);
 #else
 		val = _read(m_handle, &c, 1);
@@ -516,15 +509,15 @@ Bool LocalFile::scanString(AsciiString &newString)
 
 	do {
 		newString.concat(c);
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 		val = fread(&c, 1, 1, m_file);
 #else
 		val = _read(m_handle, &c, 1);
 #endif
 	} while ((val != 0) && (!isspace(c)));
 
 	if (val != 0) {
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 		fseek(m_file, -1, SEEK_CUR);
 #else
 		_lseek(m_handle, -1, SEEK_CUR);
@@ -547,13 +540,13 @@ void LocalFile::nextLine(Char *buf, Int bufSize)
 	// seek to the next new-line.
 	do {
 		if ((buf == NULL) || (i >= (bufSize-1))) {
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 			val = fread(&c, 1, 1, m_file);
 #else
 			val = _read(m_handle, &c, 1);
 #endif
 		} else {
-#ifdef USE_BUFFERED_IO
+#if USE_BUFFERED_IO
 			val = fread(buf + i, 1, 1, m_file);
 #else
 			val = _read(m_handle, buf + i, 1);