CXX-236 Backport server r2.6.1..r2.6.2 changes

Author: acmorrow

SConstruct

@@ -299,6 +299,8 @@ add_option("mongod-concurrency-level", "Concurrency level, \"global\" or \"db\""
 
 add_option('build-fast-and-loose', "NEVER for production builds", 0, False)
 
+add_option('disable-warnings-as-errors', "Don't add -Werror to compiler command line", 0, False)
+
 add_option('propagate-shell-environment',
            "Pass shell environment to sub-processes (NEVER for production builds)",
            0, False)
@@ -840,7 +842,9 @@ if nix:
                          "-Winvalid-pch"] )
     # env.Append( " -Wconversion" ) TODO: this doesn't really work yet
     if linux or darwin:
-        env.Append( CCFLAGS=["-Werror", "-pipe"] )
+        env.Append( CCFLAGS=["-pipe"] )
+        if not has_option("disable-warnings-as-errors"):
+            env.Append( CCFLAGS=["-Werror"] )
 
     env.Append( CPPDEFINES=["_FILE_OFFSET_BITS=64"] )
     env.Append( CXXFLAGS=["-Wnon-virtual-dtor", "-Woverloaded-virtual"] )

src/mongo/base/error_codes.err

@@ -87,6 +87,7 @@ error_code("IndexOptionsConflict", 85 )
 error_code("IndexKeySpecsConflict", 86 )
 
 # Non-sequential error codes (for compatibility only)
+error_code("NetworkTimeout", 89)
 error_code("NotMaster", 10107) #this comes from assert_util.h
 error_code("DuplicateKey", 11000)
 error_code("InterruptedAtShutdown", 11600)
@@ -95,7 +96,7 @@ error_code("OutOfDiskSpace", 14031 )
 error_code("BackgroundOperationInProgressForDatabase", 12586);
 error_code("BackgroundOperationInProgressForNamespace", 12587);
 
-error_class("NetworkError", ["HostUnreachable", "HostNotFound"])
+error_class("NetworkError", ["HostUnreachable", "HostNotFound", "NetworkTimeout"])
 error_class("Interruption", ["Interrupted", "InterruptedAtShutdown", "ExceededTimeLimit"])
 error_class("IndexCreationError", ["CannotCreateIndex", "IndexOptionsConflict", 
                                    "IndexKeySpecsConflict", "IndexAlreadyExists"])

src/mongo/bson/bsonobjbuilder.h

@@ -793,7 +793,7 @@ namespace mongo {
             return _b.subarrayStart( num() );
         }
 
-        // These should only be used where you really need interface compatability with BSONObjBuilder
+        // These should only be used where you really need interface compatibility with BSONObjBuilder
         // Currently they are only used by update.cpp and it should probably stay that way
         BufBuilder &subobjStart( const StringData& name ) {
             fill( name );

src/mongo/client/dbclientinterface.h

@@ -191,6 +191,7 @@ namespace mongo {
      */
     class MONGO_CLIENT_API ConnectionString {
     public:
+
         enum ConnectionType { INVALID , MASTER , PAIR , SET , SYNC, CUSTOM };
 
         ConnectionString() {
@@ -295,6 +296,11 @@ namespace mongo {
             return _connectHook;
         }
 
+        // Allows ConnectionStrings to be stored more easily in sets/maps
+        bool operator<(const ConnectionString& other) const {
+            return _string < other._string;
+        }
+
         //
         // FOR TESTING ONLY - useful to be able to directly mock a connection string without
         // including the entire client library.

src/mongo/client/syncclusterconnection.cpp

@@ -291,22 +291,47 @@ namespace mongo {
         return isOk( info );
     }
 
+    void SyncClusterConnection::attachQueryHandler( QueryHandler* handler ) {
+        _customQueryHandler.reset( handler );
+    }
+
     auto_ptr<DBClientCursor> SyncClusterConnection::_queryOnActive(const string &ns, Query query, int nToReturn, int nToSkip,
             const BSONObj *fieldsToReturn, int queryOptions, int batchSize ) {
 
+        if ( _customQueryHandler && _customQueryHandler->canHandleQuery( ns, query ) ) {
+
+            LOG( 2 ) << "custom query handler used for query on " << ns << ": "
+                     << query.toString() << endl;
+
+            return _customQueryHandler->handleQuery( _connAddresses,
+                                                     ns,
+                                                     query,
+                                                     nToReturn,
+                                                     nToSkip,
+                                                     fieldsToReturn,
+                                                     queryOptions,
+                                                     batchSize );
+        }
+
         for ( size_t i=0; i<_conns.size(); i++ ) {
             try {
                 auto_ptr<DBClientCursor> cursor =
                     _conns[i]->query( ns , query , nToReturn , nToSkip , fieldsToReturn , queryOptions , batchSize );
                 if ( cursor.get() )
                     return cursor;
-                log() << "query failed to: " << _conns[i]->toString() << " no data" << endl;
+
+                log() << "query on " << ns << ": " << query.toString() << " failed to: "
+                      << _conns[i]->toString() << " no data" << endl;
             }
             catch ( std::exception& e ) {
-                log() << "query failed to: " << _conns[i]->toString() << " exception: " << e.what() << endl;
+
+                log() << "query on " << ns << ": " << query.toString() << " failed to: "
+                      << _conns[i]->toString() << " exception: " << e.what() << endl;
             }
             catch ( ... ) {
-                log() << "query failed to: " << _conns[i]->toString() << " exception" << endl;
+
+                log() << "query on " << ns << ": " << query.toString() << " failed to: "
+                      << _conns[i]->toString() << " exception" << endl;
             }
         }
         throw UserException( 8002 , str::stream() << "all servers down/unreachable when querying: " << _address );

src/mongo/client/syncclusterconnection.h

@@ -47,6 +47,8 @@ namespace mongo {
         using DBClientBase::update;
         using DBClientBase::remove;
 
+        class QueryHandler;
+
         /**
          * @param commaSeparated should be 3 hosts comma separated
          */
@@ -116,6 +118,13 @@ namespace mongo {
         virtual void setRunCommandHook(DBClientWithCommands::RunCommandHookFunc func);
         virtual void setPostRunCommandHook(DBClientWithCommands::PostRunCommandHookFunc func);
 
+        /**
+         * Allow custom query processing through an external (e.g. mongos-only) service.
+         *
+         * Takes ownership of attached handler.
+         */
+        void attachQueryHandler( QueryHandler* handler );
+
     protected:
         virtual void _auth(const BSONObj& params);
 
@@ -132,14 +141,47 @@ namespace mongo {
         string _address;
         vector<string> _connAddresses;
         vector<DBClientConnection*> _conns;
-        map<string,int> _lockTypes;
-        mongo::mutex _mutex;
 
         vector<BSONObj> _lastErrors;
 
+        // Optionally attached by user
+        scoped_ptr<QueryHandler> _customQueryHandler;
+
+        mongo::mutex _mutex;
+        map<string,int> _lockTypes;
+        // End mutex
+
         double _socketTimeout;
     };
 
+    /**
+     * Interface for custom query processing for the SCC.
+     * Allows plugging different host query behaviors for different types of queries.
+     */
+    class SyncClusterConnection::QueryHandler {
+    public:
+
+        virtual ~QueryHandler() {};
+
+        /**
+         * Returns true if the query can be processed using this handler.
+         */
+        virtual bool canHandleQuery( const string& ns, Query query ) = 0;
+
+        /**
+         * Returns a cursor on one of the hosts with the desired results for the query.
+         * May throw or return an empty auto_ptr on failure.
+         */
+        virtual auto_ptr<DBClientCursor> handleQuery( const vector<string>& hosts,
+                                                      const string &ns,
+                                                      Query query,
+                                                      int nToReturn,
+                                                      int nToSkip,
+                                                      const BSONObj *fieldsToReturn,
+                                                      int queryOptions,
+                                                      int batchSize ) = 0;
+    };
+
     class MONGO_CLIENT_API UpdateNotTheSame : public UserException {
     public:
         UpdateNotTheSame( int code , const string& msg , const vector<string>& addrs , const vector<BSONObj>& lastErrors )

src/mongo/db/jsobj.cpp

@@ -211,26 +211,38 @@ namespace mongo {
             if (format == Strict) {
                 Date_t d = date();
                 s << "{ \"$date\" : ";
-                if (static_cast<long long>(d.millis) < 0) {
-                    s << "{ \"$numberLong\" : \"" << static_cast<long long>(d.millis) << "\" }";
+                // The two cases in which we cannot convert Date_t::millis to an ISO Date string are
+                // when the date is too large to format (SERVER-13760), and when the date is before
+                // the epoch (SERVER-11273).  Since Date_t internally stores millis as an unsigned
+                // long long, despite the fact that it is logically signed (SERVER-8573), this check
+                // handles both the case where Date_t::millis is too large, and the case where
+                // Date_t::millis is negative (before the epoch).
+                if (d.isFormatable()) {
+                    s << "\"" << dateToISOStringLocal(date()) << "\"";
                 }
                 else {
-                    s << "\"" << dateToISOStringLocal(date()) << "\"";
+                    s << "{ \"$numberLong\" : \"" << static_cast<long long>(d.millis) << "\" }";
                 }
                 s << " }";
             }
             else {
                 s << "Date( ";
                 if (pretty) {
                     Date_t d = date();
-                    if (static_cast<long long>(d.millis) < 0) {
+                    // The two cases in which we cannot convert Date_t::millis to an ISO Date string
+                    // are when the date is too large to format (SERVER-13760), and when the date is
+                    // before the epoch (SERVER-11273).  Since Date_t internally stores millis as an
+                    // unsigned long long, despite the fact that it is logically signed
+                    // (SERVER-8573), this check handles both the case where Date_t::millis is too
+                    // large, and the case where Date_t::millis is negative (before the epoch).
+                    if (d.isFormatable()) {
+                        s << "\"" << dateToISOStringLocal(date()) << "\"";
+                    }
+                    else {
                         // FIXME: This is not parseable by the shell, since it may not fit in a
                         // float
                         s << d.millis;
                     }
-                    else {
-                        s << "\"" << dateToISOStringLocal(date()) << "\"";
-                    }
                 }
                 else {
                     s << date().asInt64();

src/mongo/dbtests/jsontests.cpp

@@ -405,6 +405,14 @@ namespace JsonTests {
                                built.jsonString( Strict ) );
                 ASSERT_EQUALS( "{ \"a\" : Date( 0 ) }", built.jsonString( TenGen ) );
                 ASSERT_EQUALS( "{ \"a\" : Date( 0 ) }", built.jsonString( JS ) );
+
+                // Test dates above our maximum formattable date.  See SERVER-13760.
+                BSONObjBuilder b2;
+                b2.appendDate("a", 32535262800000ULL);
+                BSONObj built2 = b2.done();
+                ASSERT_EQUALS(
+                            "{ \"a\" : { \"$date\" : { \"$numberLong\" : \"32535262800000\" } } }",
+                            built2.jsonString( Strict ) );
             }
 
         private:

src/mongo/util/net/ssl_manager.cpp

@@ -634,6 +634,15 @@ namespace mongo {
     }
 
     bool SSLManager::_setupCA(SSL_CTX* context, const std::string& caFile) {
+        // Set the list of CAs sent to clients
+        STACK_OF (X509_NAME) * certNames = SSL_load_client_CA_file(caFile.c_str());
+        if (certNames == NULL) {
+            error() << "cannot read certificate authority file: " << caFile << " " <<
+                getSSLErrorMessage(ERR_get_error()) << endl;
+            return false;
+        }
+        SSL_CTX_set_client_CA_list(context, certNames);
+
         // Load trusted CA
         if (SSL_CTX_load_verify_locations(context, caFile.c_str(), NULL) != 1) {
             error() << "cannot read certificate authority file: " << caFile << " " <<

src/mongo/util/time_support.cpp

@@ -49,6 +49,15 @@ extern "C" time_t timegm(struct tm *const tmp);
 
 namespace mongo {
 
+    bool Date_t::isFormatable() const {
+        if (sizeof(time_t) == sizeof(int32_t)) {
+            return millis < 2147483647000ULL; // "2038-01-19T03:14:07Z"
+        }
+        else {
+            return millis < 32535215999000ULL; // "3000-12-31T23:59:59Z"
+        }
+    }
+
     // jsTime_virtual_skew is just for testing. a test command manipulates it.
     long long jsTime_virtual_skew = 0;
     boost::thread_specific_ptr<long long> jsTime_virtual_thread_skew;
@@ -118,6 +127,7 @@ namespace mongo {
     }
 
     static inline std::string _dateToISOString(Date_t date, bool local) {
+        invariant(date.isFormatable());
         const int bufSize = 32;
         char buf[bufSize];
         struct tm t;

src/mongo/util/time_support.h

@@ -44,6 +44,7 @@ namespace mongo {
         int64_t asInt64() const {
             return static_cast<int64_t>(millis);
         }
+        bool isFormatable() const;
     };
 
     // uses ISO 8601 dates without trailing Z