From 62bc43d13a1a3dd6cf09bf00d8ecd32bd0930a29 Mon Sep 17 00:00:00 2001 From: David Rose Date: Thu, 11 Sep 2008 01:37:00 +0000 Subject: [PATCH] better nl stuff, rename SIMPLE_THREADS_NO_IMPLICIT_YIELD --- dtool/Config.pp | 50 ++++--- dtool/LocalSetup.pp | 5 +- dtool/src/dtoolutil/filename.cxx | 10 +- dtool/src/dtoolutil/filename.h | 4 +- dtool/src/dtoolutil/pandaFileStream.cxx | 4 + dtool/src/dtoolutil/pandaFileStream.h | 10 +- dtool/src/dtoolutil/pandaFileStreamBuf.cxx | 144 ++++++++++++++++++--- dtool/src/dtoolutil/pandaFileStreamBuf.h | 28 +++- dtool/src/prc/configPageManager.cxx | 6 + dtool/src/prc/config_prc.cxx | 14 +- dtool/src/prc/config_prc.h | 4 + 11 files changed, 230 insertions(+), 49 deletions(-) diff --git a/dtool/Config.pp b/dtool/Config.pp index 934ac553a2..29c42642c7 100644 --- a/dtool/Config.pp +++ b/dtool/Config.pp @@ -704,31 +704,32 @@ // default is to use OS-provided threading constructs, which usually // allows for full multiprogramming support (i.e. the program can take // advantage of multiple CPU's). On the other hand, compiling in this -// full OS-provided support can impose some runtime overhead, making -// the application run slower on a single-CPU machine. To avoid this -// overhead, but still gain some of the basic functionality of threads -// (such as support for asynchronous model loads), define -// SIMPLE_THREADS true in addition to HAVE_THREADS. This will compile -// in a homespun cooperative threading implementation that runs -// strictly on one CPU, adding very little overhead over plain -// single-threaded code. +// full OS-provided support can impose some substantial runtime +// overhead, making the application run slower on a single-CPU +// machine. To avoid this overhead, but still gain some of the basic +// functionality of threads (such as support for asynchronous model +// loads), define SIMPLE_THREADS true in addition to HAVE_THREADS. +// This will compile in a homespun cooperative threading +// implementation that runs strictly on one CPU, adding very little +// overhead over plain single-threaded code. -// Enabling SIMPLE_THREADS is highly experimental at the present time. // Since SIMPLE_THREADS requires special support from the C runtime // library, it may not be available on all platforms and -// architectures. +// architectures. (However, it is known to be available on the big +// three: Linux, Win32, and OSX, with their most common +// architectures.) #define SIMPLE_THREADS -// If you are using SIMPLE_THREADS, the default is to consider an -// implicit context switch at every attempt to lock a mutex. This -// makes it less necessary to pepper your code with explicit calls to -// Thread::consider_yield(). However, you may want to restrict this -// behavior, and only allow context switches at explicit calls to -// Thread::yield(), consider_yield(), and sleep() (as well as calls to -// ConditionVar::wait()). This gives you absolute control over when -// the context switch happens, and makes mutexes unnecessary (mutexes -// will be compiled out). Define this true to build that way. -#define SIMPLE_THREADS_NO_IMPLICIT_YIELD +// If you are using SIMPLE_THREADS, you might further wish to disable +// mutexes altogether. In this mode, mutexes are compiled out (they +// become a no-op), and the only context switches happen at explicit +// calls to Thread::force_yield(), consider_yield(), and sleep(), as +// well as calls to ConditionVar::wait(), and certain I/O operations. +// This gives you control over when the context switch happens, and +// may make mutexes unnecessary, if you are somewhat careful in your +// code design. Disabling mutexes saves a tiny bit of runtime and +// memory overhead. +#define SIMPLE_THREADS_NO_MUTEX // Whether threading is defined or not, you might want to validate the // thread and synchronization operations. With threading enabled, @@ -746,6 +747,15 @@ // than CPU's. Even then, OS-based locking is probably better. #define MUTEX_SPINLOCK +// Define this to use the PandaFileStream interface for pifstream, +// pofstream, and pfstream. This is a customized file buffer that may +// have slightly better newline handling, but its primary benefit is +// that it supports SIMPLE_THREADS better by blocking just the active +// "thread" when I/O is delayed, instead of blocking the entire +// process. Normally, there's no reason to turn this off, unless you +// suspect a bug in Panda. +#define USE_PANDAFILESTREAM + // Do you want to build the PStats interface, for graphical run-time // performance statistics? This requires NET to be available. By // default, we don't build PStats when OPTIMIZE = 4, although this is diff --git a/dtool/LocalSetup.pp b/dtool/LocalSetup.pp index 163d0e75ca..91c6d85d7b 100644 --- a/dtool/LocalSetup.pp +++ b/dtool/LocalSetup.pp @@ -310,7 +310,7 @@ $[cdefine HAVE_THREADS] /* Define if we want to use fast, user-space simulated threads. */ $[cdefine SIMPLE_THREADS] -$[cdefine SIMPLE_THREADS_NO_IMPLICIT_YIELD] +$[cdefine SIMPLE_THREADS_NO_MUTEX] /* Define to enable deadlock detection, mutex recursion checks, etc. */ $[cdefine DEBUG_THREADS] @@ -318,6 +318,9 @@ $[cdefine DEBUG_THREADS] /* Define to implement mutexes and condition variables via a user-space spinlock. */ $[cdefine MUTEX_SPINLOCK] +/* Define to enable the PandaFileStream implementation of pfstream etc. */ +$[cdefine USE_PANDAFILESTREAM] + /* Define if we want to compile the net code. */ $[cdefine HAVE_NET] diff --git a/dtool/src/dtoolutil/filename.cxx b/dtool/src/dtoolutil/filename.cxx index 9b3631ec73..80860e0467 100644 --- a/dtool/src/dtoolutil/filename.cxx +++ b/dtool/src/dtoolutil/filename.cxx @@ -1913,7 +1913,7 @@ open_read_write(fstream &stream) const { return (!stream.fail()); } -/* +#ifdef USE_PANDAFILESTREAM //////////////////////////////////////////////////////////////////// // Function: Filename::open_read // Access: Published @@ -1945,7 +1945,9 @@ open_read(pifstream &stream) const { stream.open(os_specific.c_str(), open_mode); return (!stream.fail()); } +#endif // USE_PANDAFILESTREAM +#ifdef USE_PANDAFILESTREAM //////////////////////////////////////////////////////////////////// // Function: Filename::open_write // Access: Published @@ -2000,7 +2002,9 @@ open_write(pofstream &stream, bool truncate) const { return (!stream.fail()); } +#endif // USE_PANDAFILESTREAM +#ifdef USE_PANDAFILESTREAM //////////////////////////////////////////////////////////////////// // Function: Filename::open_append // Access: Published @@ -2037,7 +2041,9 @@ open_append(pofstream &stream) const { return (!stream.fail()); } +#endif // USE_PANDAFILESTREAM +#ifdef USE_PANDAFILESTREAM //////////////////////////////////////////////////////////////////// // Function: Filename::open_read_write // Access: Published @@ -2080,7 +2086,7 @@ open_read_write(pfstream &stream) const { return (!stream.fail()); } -*/ +#endif // USE_PANDAFILESTREAM //////////////////////////////////////////////////////////////////// // Function: Filename::touch diff --git a/dtool/src/dtoolutil/filename.h b/dtool/src/dtoolutil/filename.h index a00f8beebb..0a28148d31 100644 --- a/dtool/src/dtoolutil/filename.h +++ b/dtool/src/dtoolutil/filename.h @@ -180,12 +180,12 @@ PUBLISHED: bool open_append(ofstream &stream) const; bool open_read_write(fstream &stream) const; - /* +#ifdef USE_PANDAFILESTREAM bool open_read(pifstream &stream) const; bool open_write(pofstream &stream, bool truncate = true) const; bool open_append(pofstream &stream) const; bool open_read_write(pfstream &stream) const; - */ +#endif // USE_PANDAFILESTREAM bool chdir() const; bool touch() const; diff --git a/dtool/src/dtoolutil/pandaFileStream.cxx b/dtool/src/dtoolutil/pandaFileStream.cxx index ca0a2f4e40..9941e5fbb3 100644 --- a/dtool/src/dtoolutil/pandaFileStream.cxx +++ b/dtool/src/dtoolutil/pandaFileStream.cxx @@ -14,4 +14,8 @@ #include "pandaFileStream.h" +#ifdef USE_PANDAFILESTREAM + + +#endif // USE_PANDAFILESTREAM diff --git a/dtool/src/dtoolutil/pandaFileStream.h b/dtool/src/dtoolutil/pandaFileStream.h index 2649242c90..f1dbc5165d 100644 --- a/dtool/src/dtoolutil/pandaFileStream.h +++ b/dtool/src/dtoolutil/pandaFileStream.h @@ -16,6 +16,9 @@ #define PANDAFILESTREAM_H #include "dtoolbase.h" + +#ifdef USE_PANDAFILESTREAM + #include "pandaFileStreamBuf.h" //////////////////////////////////////////////////////////////////// @@ -86,15 +89,16 @@ private: #include "pandaFileStream.I" -// Not ready for prime time yet. -/* typedef IFileStream pifstream; typedef OFileStream pofstream; typedef FileStream pfstream; -*/ + +#else // USE_PANDAFILESTREAM typedef ifstream pifstream; typedef ofstream pofstream; typedef fstream pfstream; +#endif // USE_PANDAFILESTREAM + #endif diff --git a/dtool/src/dtoolutil/pandaFileStreamBuf.cxx b/dtool/src/dtoolutil/pandaFileStreamBuf.cxx index 0e859c5ab0..a07c2fe024 100644 --- a/dtool/src/dtoolutil/pandaFileStreamBuf.cxx +++ b/dtool/src/dtoolutil/pandaFileStreamBuf.cxx @@ -15,6 +15,8 @@ #include "pandaFileStreamBuf.h" #include "memoryHook.h" +#ifdef USE_PANDAFILESTREAM + #ifndef _WIN32 #include #include @@ -22,6 +24,8 @@ #include #endif // _WIN32 +PandaFileStreamBuf::NewlineMode PandaFileStreamBuf::_newline_mode = NM_native; + static const size_t file_buffer_size = 4096; //////////////////////////////////////////////////////////////////// @@ -507,17 +511,43 @@ write_chars(const char *start, size_t length) { // The file is opened in text mode. We have to encode newline // characters to the file. + NewlineMode this_newline_mode = _newline_mode; + if (this_newline_mode == NM_native) { #ifdef _WIN32 - // Windows requires a larger buffer here, since we are writing two - // newline characters for every one. - size_t buffer_length = length * 2; + this_newline_mode = NM_msdos; #else - size_t buffer_length = length; -#endif // _WIN32 + // Even the Mac uses Unix-style EOL characters these days. + this_newline_mode = NM_unix; +#endif + } + if (this_newline_mode == NM_binary) { + return write_chars_raw(start, length); + } + + size_t buffer_length = length; + if (this_newline_mode == NM_msdos) { + // Windows requires a larger buffer here, since we are writing two + // newline characters for every one. + buffer_length *= 2; + } char *buffer = (char *)alloca(buffer_length); - size_t write_length = encode_newlines(buffer, buffer_length, start, length); + size_t write_length; + switch (this_newline_mode) { + case NM_msdos: + write_length = encode_newlines_msdos(buffer, buffer_length, start, length); + break; + + case NM_mac: + write_length = encode_newlines_mac(buffer, buffer_length, start, length); + break; + + default: + write_length = encode_newlines_unix(buffer, buffer_length, start, length); + break; + } + if (write_length == write_chars_raw(buffer, write_length)) { // Success. Return the number of original characters. return length; @@ -770,10 +800,8 @@ decode_newlines(char *dest, size_t dest_length, return q - dest; } -#ifdef _WIN32 - //////////////////////////////////////////////////////////////////// -// Function: PandaFileStreamBuf::encode_newlines +// Function: PandaFileStreamBuf::encode_newlines_msdos // Access: Private // Description: Windows case: Converts a buffer from \n to \n\r. // @@ -783,8 +811,8 @@ decode_newlines(char *dest, size_t dest_length, // Returns the number of characters placed in dest. //////////////////////////////////////////////////////////////////// size_t PandaFileStreamBuf:: -encode_newlines(char *dest, size_t dest_length, - const char *source, size_t source_length) { +encode_newlines_msdos(char *dest, size_t dest_length, + const char *source, size_t source_length) { const char *p = source; // Read from p char *q = dest; // Write to q @@ -812,12 +840,10 @@ encode_newlines(char *dest, size_t dest_length, return q - dest; } -#else // _WIN32 - //////////////////////////////////////////////////////////////////// -// Function: PandaFileStreamBuf::encode_newlines +// Function: PandaFileStreamBuf::encode_newlines_unix // Access: Private -// Description: Posix case: Converts a buffer from \n to \n. +// Description: Unix case: Converts a buffer from \n to \n. // // This is, of course, no conversion at all; but we do // strip out \r characters if they appear in the buffer; @@ -828,8 +854,8 @@ encode_newlines(char *dest, size_t dest_length, // Returns the number of characters placed in dest. //////////////////////////////////////////////////////////////////// size_t PandaFileStreamBuf:: -encode_newlines(char *dest, size_t dest_length, - const char *source, size_t source_length) { +encode_newlines_unix(char *dest, size_t dest_length, + const char *source, size_t source_length) { const char *p = source; // Read from p char *q = dest; // Write to q @@ -850,4 +876,86 @@ encode_newlines(char *dest, size_t dest_length, return q - dest; } -#endif // _WIN32 +//////////////////////////////////////////////////////////////////// +// Function: PandaFileStreamBuf::encode_newlines_mac +// Access: Private +// Description: Classic Mac case: Converts a buffer from \n to \r. +// +// Returns the number of characters placed in dest. +//////////////////////////////////////////////////////////////////// +size_t PandaFileStreamBuf:: +encode_newlines_mac(char *dest, size_t dest_length, + const char *source, size_t source_length) { + const char *p = source; // Read from p + char *q = dest; // Write to q + + while (p < source + source_length) { + assert(q < dest + dest_length); + switch (*p) { + case '\n': + *q++ = '\r'; + break; + + case '\r': + break; + + default: + *q++ = *p; + break; + } + + ++p; + } + + return q - dest; +} + +ostream & +operator << (ostream &out, PandaFileStreamBuf::NewlineMode newline_mode) { + switch (newline_mode) { + case PandaFileStreamBuf::NM_native: + return out << "native"; + + case PandaFileStreamBuf::NM_binary: + return out << "binary"; + + case PandaFileStreamBuf::NM_msdos: + return out << "msdos"; + + case PandaFileStreamBuf::NM_unix: + return out << "unix"; + + case PandaFileStreamBuf::NM_mac: + return out << "mac"; + } + + cerr + << "Invalid NewlineMode value: " << (int)newline_mode << "\n"; + return out; +} + +istream & +operator >> (istream &in, PandaFileStreamBuf::NewlineMode &newline_mode) { + string word; + in >> word; + + if (word == "native") { + newline_mode = PandaFileStreamBuf::NM_native; + } else if (word == "binary") { + newline_mode = PandaFileStreamBuf::NM_binary; + } else if (word == "msdos") { + newline_mode = PandaFileStreamBuf::NM_msdos; + } else if (word == "unix") { + newline_mode = PandaFileStreamBuf::NM_unix; + } else if (word == "mac") { + newline_mode = PandaFileStreamBuf::NM_mac; + } else { + cerr + << "Invalid NewlineMode value: " << word << "\n"; + newline_mode = PandaFileStreamBuf::NM_native; + } + + return in; +} + +#endif // USE_PANDAFILESTREAM diff --git a/dtool/src/dtoolutil/pandaFileStreamBuf.h b/dtool/src/dtoolutil/pandaFileStreamBuf.h index 7e8046dcb4..4de6d8eb5a 100644 --- a/dtool/src/dtoolutil/pandaFileStreamBuf.h +++ b/dtool/src/dtoolutil/pandaFileStreamBuf.h @@ -17,6 +17,8 @@ #include "dtoolbase.h" +#ifdef USE_PANDAFILESTREAM + #if defined(_WIN32) #define WIN32_LEAN_AND_MEAN #include @@ -36,6 +38,15 @@ public: bool is_open() const; void close(); + enum NewlineMode { + NM_native, + NM_binary, + NM_msdos, + NM_unix, + NM_mac, + }; + static NewlineMode _newline_mode; + protected: virtual streampos seekoff(streamoff off, ios_seekdir dir, ios_openmode which); virtual streampos seekpos(streampos pos, ios_openmode which); @@ -53,8 +64,13 @@ private: size_t decode_newlines(char *dest, size_t dest_length, const char *source, size_t source_length); - size_t encode_newlines(char *dest, size_t dest_length, - const char *source, size_t source_length); + + size_t encode_newlines_msdos(char *dest, size_t dest_length, + const char *source, size_t source_length); + size_t encode_newlines_unix(char *dest, size_t dest_length, + const char *source, size_t source_length); + size_t encode_newlines_mac(char *dest, size_t dest_length, + const char *source, size_t source_length); private: string _filename; @@ -74,4 +90,12 @@ private: streampos _gpos; }; +EXPCL_DTOOL ostream & +operator << (ostream &out, PandaFileStreamBuf::NewlineMode newline_mode); + +EXPCL_DTOOL istream & +operator >> (istream &in, PandaFileStreamBuf::NewlineMode &newline_mode); + +#endif // USE_PANDAFILESTREAM + #endif diff --git a/dtool/src/prc/configPageManager.cxx b/dtool/src/prc/configPageManager.cxx index 59967b1ca6..163a2953f7 100644 --- a/dtool/src/prc/configPageManager.cxx +++ b/dtool/src/prc/configPageManager.cxx @@ -348,6 +348,12 @@ reload_implicit_pages() { _currently_loading = false; invalidate_cache(); + +#ifdef USE_PANDAFILESTREAM + // Update this very low-level config variable here, for lack of any + // better place. + PandaFileStreamBuf::_newline_mode = newline_mode; +#endif // USE_PANDAFILESTREAM } //////////////////////////////////////////////////////////////////// diff --git a/dtool/src/prc/config_prc.cxx b/dtool/src/prc/config_prc.cxx index 4690d2d721..644639e77b 100644 --- a/dtool/src/prc/config_prc.cxx +++ b/dtool/src/prc/config_prc.cxx @@ -14,10 +14,22 @@ #include "config_prc.h" #include "configVariableBool.h" +#include "pandaFileStreamBuf.h" NotifyCategoryDef(prc, ""); ConfigVariableBool assert_abort ("assert-abort", false, - "Set this true to trigger a core dump and/or stack trace when the first assertion fails"); + PRC_DESC("Set this true to trigger a core dump and/or stack trace when the first assertion fails")); +#ifdef USE_PANDAFILESTREAM + +ConfigVariableEnum newline_mode +("newline-mode", PandaFileStreamBuf::NM_native, + PRC_DESC("Controls how newlines are written by Panda applications writing " + "to a text file. The default, \"native\", means to write newlines " + "appropriate to the current platform. You may also specify \"binary\", " + "to avoid molesting the file data, or one of \"msdos\", \"unix\", " + "or \"mac\".")); + +#endif // USE_PANDAFILESTREAM diff --git a/dtool/src/prc/config_prc.h b/dtool/src/prc/config_prc.h index 321a6db00d..f472be3ddc 100644 --- a/dtool/src/prc/config_prc.h +++ b/dtool/src/prc/config_prc.h @@ -24,5 +24,9 @@ NotifyCategoryDecl(prc, EXPCL_DTOOLCONFIG, EXPTP_DTOOLCONFIG); extern ConfigVariableBool assert_abort; +#ifdef USE_PANDAFILESTREAM +extern ConfigVariableEnum newline_mode; +#endif + #endif