Skip to content

[CORE] Enable and improve buffered IO in LocalFile to reduce its cost by up to 90% #1299

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jul 18, 2025
Merged

[CORE] Enable and improve buffered IO in LocalFile to reduce its cost by up to 90% #1299

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
03c3596
[CORE] Fix and enable buffered IO in GameEngine File System
xezon Jul 17, 2025
155197e
Minor refactor
xezon Jul 17, 2025
77e76df
Remove obsolete comments
xezon Jul 17, 2025
49748c1
Revert uncommenting
xezon Jul 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Core/GameEngine/Include/Common/GameDefines.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,9 @@
#ifndef ENABLE_CONFIGURABLE_SHROUD
#define ENABLE_CONFIGURABLE_SHROUD (1) // When enabled, the GlobalData contains a field to turn on/off the shroud, otherwise shroud is always enabled
#endif

// Enable buffered IO in File System. Was disabled in retail game.
// Buffered IO generally is much faster than unbuffered for small reads and writes.
#ifndef USE_BUFFERED_IO
#define USE_BUFFERED_IO (1)
#endif
16 changes: 6 additions & 10 deletions Core/GameEngine/Include/Common/LocalFile.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,7 @@

#include "Common/file.h"

// srj sez: this was purely an experiment in optimization.
// at the present time, it doesn't appear to be a good one.
// but I am leaving the code in for now.
// if still present in 2003, please nuke.
#define NO_USE_BUFFERED_IO
#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
#include <stdio.h>
#endif

Expand All @@ -85,11 +80,12 @@ class LocalFile : public File
MEMORY_POOL_GLUE_ABC(LocalFile)
private:

#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
// srj sez: this was purely an experiment in optimization.
// at the present time, it doesn't appear to be a good one.
// TheSuperHackers @info It is a good optimization and will be
// significantly faster than unbuffered IO with small reads and writes.
FILE* m_file;

enum { BUF_SIZE = 32768 };
char m_vbuf[BUF_SIZE];
#else
int m_handle; ///< Local C file handle
#endif
Expand Down
117 changes: 55 additions & 62 deletions Core/GameEngine/Source/Common/System/LocalFile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ static Int s_totalOpen = 0;
//=================================================================

LocalFile::LocalFile()
#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
: m_file(NULL)
#else
: m_handle(-1)
Expand All @@ -127,7 +127,7 @@ LocalFile::LocalFile()

LocalFile::~LocalFile()
{
#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
if (m_file)
{
fclose(m_file);
Expand Down Expand Up @@ -167,61 +167,54 @@ Bool LocalFile::open( const Char *filename, Int access )
}

/* here we translate WSYS file access to the std C equivalent */
#ifdef USE_BUFFERED_IO
char mode[32];
char* m = mode;

if (m_access & APPEND)
#if USE_BUFFERED_IO

// r open for reading (The file must exist)
// w open for writing (creates file if it doesn't exist). Deletes content and overwrites the file.
// a open for appending (creates file if it doesn't exist).
// r+ open for reading and writing (The file must exist).
// w+ open for reading and writing.
// If file exists deletes content and overwrites the file, otherwise creates an empty new file.
// a+ open for reading and writing (append if file exists).

const Bool write = (m_access & WRITE) != 0;
const Bool readwrite = (m_access & READWRITE) == READWRITE;
const Bool append = (m_access & APPEND) != 0;
const Bool create = (m_access & CREATE) != 0;
const Bool truncate = (m_access & TRUNCATE) != 0;
const Bool binary = (m_access & BINARY) != 0;

const Char *mode = NULL;

// Mode string selection (mimics _open flag combinations)
// TEXT is implicit for fopen if 'b' is not present
// READ is implicit here if not READWRITE or WRITE
if (readwrite)
{
DEBUG_CRASH(("not yet supported by buffered mode"));
}

if (m_access & TRUNCATE)
{
DEBUG_CRASH(("not yet supported by buffered mode"));
}

if((m_access & READWRITE ) == READWRITE )
{
if (m_access & CREATE)
{
*m++ = 'w';
*m++ = '+';
}
if (append)
mode = binary ? "a+b" : "a+";
else if (truncate || create)
mode = binary ? "w+b" : "w+";
else
{
*m++ = 'r';
*m++ = '+';
}
mode = binary ? "r+b" : "r+";
}
else if(m_access & WRITE)
{
*m++ = 'w';
}
else
{
*m++ = 'r';
DEBUG_ASSERTCRASH(!(m_access & TRUNCATE), ("cannot truncate with read-only"));
}

if (m_access & TEXT)
else if (write)
{
*m++ = 't';
if (append)
mode = binary ? "ab" : "a";
else
mode = binary ? "wb" : "w";
}
if (m_access & BINARY)
else // implicitly read-only
{
*m++ = 'b';
mode = binary ? "rb" : "r";
}

*m++ = 0;

m_file = fopen(filename, mode);
if (m_file == NULL)
{
goto error;
}

//setvbuf(m_file, m_vbuf, _IOFBF, sizeof(BUF_SIZE));

#else

Expand Down Expand Up @@ -257,7 +250,7 @@ Bool LocalFile::open( const Char *filename, Int access )
flags |= _O_WRONLY;
flags |= _O_CREAT;
}
else
else // implicitly read-only
{
flags |= _O_RDONLY;
}
Expand Down Expand Up @@ -318,15 +311,15 @@ Int LocalFile::read( void *buffer, Int bytes )

if (buffer == NULL)
{
#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
fseek(m_file, bytes, SEEK_CUR);
#else
_lseek(m_handle, bytes, SEEK_CUR);
#endif
return bytes;
}

#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
Int ret = fread(buffer, 1, bytes, m_file);
#else
Int ret = _read( m_handle, buffer, bytes );
Expand All @@ -347,7 +340,7 @@ Int LocalFile::write( const void *buffer, Int bytes )
return -1;
}

#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
Int ret = fwrite(buffer, 1, bytes, m_file);
#else
Int ret = _write( m_handle, buffer, bytes );
Expand All @@ -366,21 +359,21 @@ Int LocalFile::seek( Int pos, seekMode mode)
switch( mode )
{
case START:
DEBUG_ASSERTCRASH(pos >= 0, ("LocalFile::seek - pos must be >= 0 when seeking from the beginning of the file"));
lmode = SEEK_SET;
break;
case CURRENT:
lmode = SEEK_CUR;
break;
case END:
DEBUG_ASSERTCRASH(pos <= 0, ("LocalFile::seek - pos should be <= 0 for a seek starting at the end of the file"));
lmode = SEEK_END;
break;
default:
// bad seek mode
DEBUG_CRASH(("LocalFile::seek - bad seek mode"));
return -1;
}

#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
Int ret = fseek(m_file, pos, lmode);
if (ret == 0)
return ftell(m_file);
Expand All @@ -406,7 +399,7 @@ Bool LocalFile::scanInt(Int &newInt)

// skip preceding non-numeric characters
do {
#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
val = fread(&c, 1, 1, m_file);
#else
val = _read( m_handle, &c, 1);
Expand All @@ -419,7 +412,7 @@ Bool LocalFile::scanInt(Int &newInt)

do {
tempstr.concat(c);
#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
val = fread(&c, 1, 1, m_file);
#else
val = _read( m_handle, &c, 1);
Expand All @@ -428,7 +421,7 @@ Bool LocalFile::scanInt(Int &newInt)

// put the last read char back, since we didn't use it.
if (val != 0) {
#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
fseek(m_file, -1, SEEK_CUR);
#else
_lseek(m_handle, -1, SEEK_CUR);
Expand All @@ -454,7 +447,7 @@ Bool LocalFile::scanReal(Real &newReal)

// skip the preceding white space
do {
#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
val = fread(&c, 1, 1, m_file);
#else
val = _read( m_handle, &c, 1);
Expand All @@ -470,15 +463,15 @@ Bool LocalFile::scanReal(Real &newReal)
if (c == '.') {
sawDec = TRUE;
}
#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
val = fread(&c, 1, 1, m_file);
#else
val = _read(m_handle, &c, 1);
#endif
} while ((val != 0) && (((c >= '0') && (c <= '9')) || ((c == '.') && !sawDec)));

if (val != 0) {
#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
fseek(m_file, -1, SEEK_CUR);
#else
_lseek(m_handle, -1, SEEK_CUR);
Expand All @@ -503,7 +496,7 @@ Bool LocalFile::scanString(AsciiString &newString)

// skip the preceding whitespace
do {
#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
val = fread(&c, 1, 1, m_file);
#else
val = _read(m_handle, &c, 1);
Expand All @@ -516,15 +509,15 @@ Bool LocalFile::scanString(AsciiString &newString)

do {
newString.concat(c);
#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
val = fread(&c, 1, 1, m_file);
#else
val = _read(m_handle, &c, 1);
#endif
} while ((val != 0) && (!isspace(c)));

if (val != 0) {
#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
fseek(m_file, -1, SEEK_CUR);
#else
_lseek(m_handle, -1, SEEK_CUR);
Expand All @@ -547,13 +540,13 @@ void LocalFile::nextLine(Char *buf, Int bufSize)
// seek to the next new-line.
do {
if ((buf == NULL) || (i >= (bufSize-1))) {
#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
val = fread(&c, 1, 1, m_file);
#else
val = _read(m_handle, &c, 1);
#endif
} else {
#ifdef USE_BUFFERED_IO
#if USE_BUFFERED_IO
val = fread(buf + i, 1, 1, m_file);
#else
val = _read(m_handle, buf + i, 1);
Expand Down
Loading