diff options
author | Adam Midvidy <amidvidy@gmail.com> | 2014-09-19 16:43:33 -0400 |
---|---|---|
committer | Benety Goh <benety@mongodb.com> | 2014-09-19 16:50:23 -0400 |
commit | 78c5f9dd5ac65eae251df3e87175a0af076d90d9 (patch) | |
tree | e305dcd559f6b8f8a1ed987e6938e5b8406fea19 /src/mongo | |
parent | 63dd64abac9002e71348b7550234131cd01a87f5 (diff) | |
download | mongo-78c5f9dd5ac65eae251df3e87175a0af076d90d9.tar.gz |
SERVER-15210 SERVER-15211 remove OID undefined behavior and make it endian aware
Closes #784
Signed-off-by: Benety Goh <benety@mongodb.com>
Diffstat (limited to 'src/mongo')
26 files changed, 432 insertions, 283 deletions
diff --git a/src/mongo/SConscript b/src/mongo/SConscript index bd342aa40ba..fdf73eef66f 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -151,6 +151,9 @@ env.CppUnitTest('namespacestring_test', ['db/namespace_string_test.cpp'], env.CppUnitTest('update_index_data_test', ['db/update_index_data_test.cpp'], LIBDEPS=['bson','update_index_data','db/common']) +env.CppUnitTest('oid_test', ['bson/oid_test.cpp'], + LIBDEPS=['bson']) + env.Library('path', ['db/matcher/path.cpp', 'db/matcher/path_internal.cpp'], diff --git a/src/mongo/bson/bson_startuptest.cpp b/src/mongo/bson/bson_startuptest.cpp index 5bd779c4600..1ccb31f7f2d 100644 --- a/src/mongo/bson/bson_startuptest.cpp +++ b/src/mongo/bson/bson_startuptest.cpp @@ -47,16 +47,14 @@ namespace mongo { } void testoid() { - OID id; - id.init(); + // hardcoded so that we don't need to generate OIDs before initializers run + OID id("541c5fa6ababec1be47e21b5"); // sleepsecs(3); - OID b; // goes with sleep above... // b.init(); - // verify( memcmp(id.getData(), b.getData(), 12) < 0 ); - b.init( id.str() ); + b.init( id.toString() ); verify( b == id ); } diff --git a/src/mongo/bson/bson_validate.cpp b/src/mongo/bson/bson_validate.cpp index 454109d5886..968cac7cbc6 100644 --- a/src/mongo/bson/bson_validate.cpp +++ b/src/mongo/bson/bson_validate.cpp @@ -197,7 +197,7 @@ namespace mongo { return Status::OK(); case jstOID: - if ( !buffer->skip( sizeof(OID) ) ) + if ( !buffer->skip( OID::kOIDSize ) ) return makeError("invalid bson", idElem); return Status::OK(); @@ -224,7 +224,7 @@ namespace mongo { status = buffer->readUTF8String( NULL ); if ( !status.isOK() ) return status; - buffer->skip( sizeof(OID) ); + buffer->skip( OID::kOIDSize ); return Status::OK(); case RegEx: diff --git a/src/mongo/bson/bsonelement.cpp b/src/mongo/bson/bsonelement.cpp index b6a87cf4925..16db11a9019 100644 --- a/src/mongo/bson/bsonelement.cpp +++ b/src/mongo/bson/bsonelement.cpp @@ -31,6 +31,7 @@ #include "mongo/db/jsobj.h" #include "mongo/util/base64.h" +#include "mongo/util/hex.h" #include "mongo/util/log.h" #include "mongo/util/mongoutils/str.h" @@ -134,7 +135,6 @@ namespace mongo { break; } case DBRef: { - mongo::OID *x = (mongo::OID *) (valuestr() + valuestrsize()); if ( format == TenGen ) s << "Dbref( "; else @@ -142,7 +142,7 @@ namespace mongo { s << '"' << valuestr() << "\", "; if ( format != TenGen ) s << "\"$id\" : "; - s << '"' << *x << "\" "; + s << '"' << mongo::OID::from(valuestr() + valuestrsize()) << "\" "; if ( format == TenGen ) s << ')'; else @@ -459,7 +459,7 @@ namespace mongo { x = 8; break; case jstOID: - x = 12; + x = OID::kOIDSize; break; case Symbol: case Code: @@ -539,7 +539,7 @@ namespace mongo { x = 8; break; case jstOID: - x = 12; + x = OID::kOIDSize; break; case Symbol: case Code: @@ -672,11 +672,7 @@ namespace mongo { break; case DBRef: s << "DBRef('" << valuestr() << "',"; - { - mongo::OID x; - std::memcpy(&x, valuestr() + valuestrsize(), sizeof(x)); - s << x << ')'; - } + s << mongo::OID::from(valuestr() + valuestrsize()) << ')'; break; case jstOID: s << "ObjectId('"; @@ -875,7 +871,7 @@ dodouble: return 1; } case jstOID: - return memcmp(l.value(), r.value(), 12); + return memcmp(l.value(), r.value(), OID::kOIDSize); case Code: case Symbol: case String: diff --git a/src/mongo/bson/bsonelement.h b/src/mongo/bson/bsonelement.h index e2dad9f071d..5cbfadc434f 100644 --- a/src/mongo/bson/bsonelement.h +++ b/src/mongo/bson/bsonelement.h @@ -252,9 +252,7 @@ namespace mongo { /** Retrieve the object ID stored in the object. You must ensure the element is of type jstOID first. */ mongo::OID __oid() const { - class OID oid; - std::memcpy(&oid, value(), sizeof(oid)); - return oid; + return OID::from(value()); } /** True if element is null. */ @@ -454,9 +452,7 @@ namespace mongo { uassert( 10064 , "not a dbref" , type() == DBRef ); const char * start = value(); start += 4 + ConstDataView(start).readLE<int>(); - class OID result; - std::memcpy(&result, start, sizeof(result)); - return result; + return mongo::OID::from(start); } /** this does not use fieldName in the comparison, just the value */ diff --git a/src/mongo/bson/bsonobj.cpp b/src/mongo/bson/bsonobj.cpp index e921c059d34..dce8cfe5064 100644 --- a/src/mongo/bson/bsonobj.cpp +++ b/src/mongo/bson/bsonobj.cpp @@ -32,6 +32,7 @@ #include "mongo/bson/bson_validate.h" #include "mongo/db/json.h" #include "mongo/util/allocator.h" +#include "mongo/util/hex.h" #include "mongo/util/log.h" #include "mongo/util/md5.hpp" #include "mongo/util/mongoutils/str.h" diff --git a/src/mongo/bson/bsonobjbuilder.cpp b/src/mongo/bson/bsonobjbuilder.cpp index 1e0e2564d10..6406ff10991 100644 --- a/src/mongo/bson/bsonobjbuilder.cpp +++ b/src/mongo/bson/bsonobjbuilder.cpp @@ -63,7 +63,6 @@ namespace mongo { appendMaxKey( fieldName ); return; case jstOID: { OID o; - memset(&o, 0, sizeof(o)); appendOID( fieldName , &o); return; } @@ -81,7 +80,6 @@ namespace mongo { appendRegex( fieldName , "" ); return; case DBRef: { OID o; - memset(&o, 0, sizeof(o)); appendDBRef( fieldName , "" , o ); return; } @@ -118,8 +116,7 @@ namespace mongo { case MaxKey: appendMaxKey( fieldName ); return; case jstOID: { - OID o; - memset(&o, 0xFF, sizeof(o)); + OID o = OID::max(); appendOID( fieldName , &o); return; } diff --git a/src/mongo/bson/bsonobjbuilder.h b/src/mongo/bson/bsonobjbuilder.h index 52a9ecc0e2e..cf84f004221 100644 --- a/src/mongo/bson/bsonobjbuilder.h +++ b/src/mongo/bson/bsonobjbuilder.h @@ -306,14 +306,14 @@ namespace mongo { _b.appendNum((char) jstOID); _b.appendStr(fieldName); if ( oid ) - _b.appendBuf( (void *) oid, 12 ); + _b.appendBuf( oid->view().view(), OID::kOIDSize ); else { OID tmp; if ( generateIfBlank ) tmp.init(); else tmp.clear(); - _b.appendBuf( (void *) &tmp, 12 ); + _b.appendBuf( tmp.view().view(), OID::kOIDSize ); } return *this; } @@ -326,7 +326,7 @@ namespace mongo { BSONObjBuilder& append( const StringData& fieldName, OID oid ) { _b.appendNum((char) jstOID); _b.appendStr(fieldName); - _b.appendBuf( (void *) &oid, 12 ); + _b.appendBuf( oid.view().view(), OID::kOIDSize ); return *this; } @@ -492,7 +492,7 @@ namespace mongo { _b.appendStr( fieldName ); _b.appendNum( (int) ns.size() + 1 ); _b.appendStr( ns ); - _b.appendBuf( (void *) &oid, 12 ); + _b.appendBuf( oid.view().view(), OID::kOIDSize ); return *this; } diff --git a/src/mongo/bson/oid.cpp b/src/mongo/bson/oid.cpp index 5ba50f231df..2c726cd96d7 100644 --- a/src/mongo/bson/oid.cpp +++ b/src/mongo/bson/oid.cpp @@ -27,169 +27,146 @@ * then also delete it in the license file. */ -#include "mongo/pch.h" +#include "mongo/platform/basic.h" + +#include "mongo/bson/oid.h" #include <boost/functional/hash.hpp> +#include <boost/scoped_ptr.hpp> +#include "mongo/base/init.h" +#include "mongo/bson/bsonobjbuilder.h" #include "mongo/platform/atomic_word.h" -#include "mongo/platform/process_id.h" #include "mongo/platform/random.h" -#include "mongo/bson/bsonobjbuilder.h" -#include "mongo/bson/oid.h" - -#define verify MONGO_verify - -BOOST_STATIC_ASSERT( sizeof(mongo::OID) == mongo::OID::kOIDSize ); -BOOST_STATIC_ASSERT( sizeof(mongo::OID) == 12 ); +#include "mongo/util/hex.h" namespace mongo { - void OID::hash_combine(size_t &seed) const { - boost::hash_combine(seed, x); - boost::hash_combine(seed, y); - boost::hash_combine(seed, z); +namespace { + boost::scoped_ptr<AtomicUInt32> counter; + + const std::size_t kTimestampOffset = 0; + const std::size_t kInstanceUniqueOffset = kTimestampOffset + + OID::kTimestampSize; + const std::size_t kIncrementOffset = kInstanceUniqueOffset + + OID::kInstanceUniqueSize; + OID::InstanceUnique _instanceUnique; +} // namespace + + MONGO_INITIALIZER_GENERAL(OIDGeneration, MONGO_NO_PREREQUISITES, ("default")) + (InitializerContext* context) { + boost::scoped_ptr<SecureRandom> entropy(SecureRandom::create()); + counter.reset(new AtomicUInt32(uint32_t(entropy->nextInt64()))); + _instanceUnique = OID::InstanceUnique::generate(*entropy); + return Status::OK(); } - size_t OID::Hasher::operator() (const OID& oid) const { - size_t seed = 0; - oid.hash_combine(seed); - return seed; - } + OID::Increment OID::Increment::next() { + uint64_t nextCtr = counter->fetchAndAdd(1); + OID::Increment incr; - // machine # before folding in the process id - OID::MachineAndPid OID::ourMachine; + incr.bytes[0] = uint8_t(nextCtr >> 16); + incr.bytes[1] = uint8_t(nextCtr >> 8); + incr.bytes[2] = uint8_t(nextCtr); - ostream& operator<<( ostream &s, const OID &o ) { - s << o.str(); - return s; + return incr; } - void OID::foldInPid(OID::MachineAndPid& x) { - unsigned p = ProcessId::getCurrent().asUInt32(); - x._pid ^= static_cast<unsigned short>(p); - // when the pid is greater than 16 bits, let the high bits modulate the machine id field. - unsigned short& rest = (unsigned short &) x._machineNumber[1]; - rest ^= p >> 16; + OID::InstanceUnique OID::InstanceUnique::generate(SecureRandom& entropy) { + int64_t rand = entropy.nextInt64(); + OID::InstanceUnique u; + std::memcpy(u.bytes, &rand, kInstanceUniqueSize); + return u; } - OID::MachineAndPid OID::genMachineAndPid() { - BOOST_STATIC_ASSERT( sizeof(mongo::OID::MachineAndPid) == 5 ); - - // we only call this once per process - scoped_ptr<SecureRandom> sr( SecureRandom::create() ); - int64_t n = sr->nextInt64(); - OID::MachineAndPid x = ourMachine = reinterpret_cast<OID::MachineAndPid&>(n); - foldInPid(x); - return x; + void OID::setTimestamp(const OID::Timestamp timestamp) { + _view().writeBE<Timestamp>(timestamp, kTimestampOffset); } - // after folding in the process id - OID::MachineAndPid OID::ourMachineAndPid = OID::genMachineAndPid(); + void OID::setInstanceUnique(const OID::InstanceUnique unique) { + // Byte order doesn't matter here + _view().writeNative<InstanceUnique>(unique, kInstanceUniqueOffset); + } - void OID::regenMachineId() { - ourMachineAndPid = genMachineAndPid(); + void OID::setIncrement(const OID::Increment inc) { + _view().writeNative<Increment>(inc, kIncrementOffset); } - inline bool OID::MachineAndPid::operator!=(const OID::MachineAndPid& rhs) const { - return _pid != rhs._pid || _machineNumber != rhs._machineNumber; + OID::Timestamp OID::getTimestamp() const { + return view().readBE<Timestamp>(kTimestampOffset); } - unsigned OID::getMachineId() { - unsigned char x[4]; - x[0] = ourMachineAndPid._machineNumber[0]; - x[1] = ourMachineAndPid._machineNumber[1]; - x[2] = ourMachineAndPid._machineNumber[2]; - x[3] = 0; - return (unsigned&) x[0]; + OID::InstanceUnique OID::getInstanceUnique() const { + // Byte order doesn't matter here + return view().readNative<InstanceUnique>(kInstanceUniqueOffset); } - void OID::justForked() { - MachineAndPid x = ourMachine; - // we let the random # for machine go into all 5 bytes of MachineAndPid, and then - // xor in the pid into _pid. this reduces the probability of collisions. - foldInPid(x); - ourMachineAndPid = genMachineAndPid(); - verify( x != ourMachineAndPid ); - ourMachineAndPid = x; + OID::Increment OID::getIncrement() const { + return view().readNative<Increment>(kIncrementOffset); } - void OID::init() { - static AtomicUInt32 inc( - static_cast<unsigned>( - scoped_ptr<SecureRandom>(SecureRandom::create())->nextInt64())); - - { - unsigned t = (unsigned) time(0); - unsigned char *T = (unsigned char *) &t; - _time[0] = T[3]; // big endian order because we use memcmp() to compare OID's - _time[1] = T[2]; - _time[2] = T[1]; - _time[3] = T[0]; + void OID::hash_combine(size_t &seed) const { + uint32_t v; + for (int i = 0; i != kOIDSize; i += sizeof(uint32_t)) { + memcpy(&v, _data + i, sizeof(uint32_t)); + boost::hash_combine(seed, v); } + } - _machineAndPid = ourMachineAndPid; + size_t OID::Hasher::operator() (const OID& oid) const { + size_t seed = 0; + oid.hash_combine(seed); + return seed; + } - { - int new_inc = inc.fetchAndAdd(1); - unsigned char *T = (unsigned char *) &new_inc; - _inc[0] = T[2]; - _inc[1] = T[1]; - _inc[2] = T[0]; - } + void OID::regenMachineId() { + boost::scoped_ptr<SecureRandom> entropy(SecureRandom::create()); + _instanceUnique = InstanceUnique::generate(*entropy); } - static AtomicUInt64 _initSequential_sequence; - void OID::initSequential() { + unsigned OID::getMachineId() { + uint32_t ret = 0; + std::memcpy(&ret, _instanceUnique.bytes, sizeof(uint32_t)); + return ret; + } - { - unsigned t = (unsigned) time(0); - unsigned char *T = (unsigned char *) &t; - _time[0] = T[3]; // big endian order because we use memcmp() to compare OID's - _time[1] = T[2]; - _time[2] = T[1]; - _time[3] = T[0]; - } - - { - unsigned long long nextNumber = _initSequential_sequence.fetchAndAdd(1); - unsigned char* numberData = reinterpret_cast<unsigned char*>(&nextNumber); - for ( int i=0; i<8; i++ ) { - data[4+i] = numberData[7-i]; - } - } + void OID::justForked() { + regenMachineId(); + } + + void OID::init() { + // each set* method handles endianness + setTimestamp(time(0)); + setInstanceUnique(_instanceUnique); + setIncrement(Increment::next()); } void OID::init( const std::string& s ) { verify( s.size() == 24 ); const char *p = s.c_str(); - for( size_t i = 0; i < kOIDSize; i++ ) { - data[i] = fromHex(p); + for (std::size_t i = 0; i < kOIDSize; i++) { + _data[i] = fromHex(p); p += 2; } } void OID::init(Date_t date, bool max) { - int time = (int) (date / 1000); - char* T = (char *) &time; - data[0] = T[3]; - data[1] = T[2]; - data[2] = T[1]; - data[3] = T[0]; - - if (max) - *(long long*)(data + 4) = 0xFFFFFFFFFFFFFFFFll; - else - *(long long*)(data + 4) = 0x0000000000000000ll; + setTimestamp(uint32_t(date / 1000)); + uint64_t rest = max ? std::numeric_limits<uint64_t>::max() : 0u; + std::memcpy(_view().view(kInstanceUniqueOffset), &rest, + kInstanceUniqueSize + kIncrementSize); } time_t OID::asTimeT() { - int time; - char* T = (char *) &time; - T[0] = data[3]; - T[1] = data[2]; - T[2] = data[1]; - T[3] = data[0]; - return time; + return getTimestamp(); + } + + std::string OID::toString() const { + return toHexLower(_data, kOIDSize); + } + + std::string OID::toIncString() const { + return toHexLower(getIncrement().bytes, kIncrementSize); } -} +} // namespace mongo diff --git a/src/mongo/bson/oid.h b/src/mongo/bson/oid.h index d5063dc5f1b..a8a3e494124 100644 --- a/src/mongo/bson/oid.h +++ b/src/mongo/bson/oid.h @@ -31,26 +31,43 @@ #include <string> -#include "mongo/bson/util/misc.h" -#include "mongo/util/hex.h" +#include "mongo/base/data_view.h" +#include "mongo/bson/util/builder.h" +#include "mongo/util/time_support.h" namespace mongo { - -#pragma pack(1) - /** Object ID type. - BSON objects typically have an _id field for the object id. This field should be the first - member of the object when present. class OID is a special type that is a 12 byte id which - is likely to be unique to the system. You may also use other types for _id's. - When _id field is missing from a BSON object, on an insert the database may insert one - automatically in certain circumstances. - - Warning: You must call OID::newState() after a fork(). - - Typical contents of the BSON ObjectID is a 12-byte value consisting of a 4-byte timestamp (seconds since epoch), - a 3-byte machine id, a 2-byte process id, and a 3-byte counter. Note that the timestamp and counter fields must - be stored big endian unlike the rest of BSON. This is because they are compared byte-by-byte and we want to ensure - a mostly increasing order. - */ + class SecureRandom; + + /** + * Object ID type. + * BSON objects typically have an _id field for the object id. This field should be the first + * member of the object when present. The OID class is a special type that is a 12 byte id which + * is likely to be unique to the system. You may also use other types for _id's. + * When _id field is missing from a BSON object, on an insert the database may insert one + * automatically in certain circumstances. + * + * The BSON ObjectID is a 12-byte value consisting of a 4-byte timestamp (seconds since epoch), + * in the highest order 4 bytes followed by a 5 byte value unique to this machine AND process, + * followed by a 3 byte counter. + * + * 4 byte timestamp 5 byte process unique 3 byte counter + * |<----------------->|<---------------------->|<-------------> + * OID layout: [----|----|----|----|----|----|----|----|----|----|----|----] + * 0 4 8 12 + * + * The timestamp is a big endian 4 byte signed-integer. + * + * The process unique is an arbitrary sequence of 5 bytes. There are no endianness concerns + * since it is never interpreted as a multi-byte value. + * + * The counter is a big endian 3 byte unsigned integer. + * + * Note: The timestamp and counter are big endian (in contrast to the rest of BSON) because + * we use memcmp to order OIDs, and we want to ensure an increasing order. + * + * Warning: You MUST call OID::justForked() after a fork(). This ensures that each process will + * generate unique OIDs. + */ class OID { public: @@ -63,50 +80,59 @@ namespace mongo { size_t operator() (const OID& oid) const; }; - OID() : a(0), b(0) { } + OID() : _data() {} enum { kOIDSize = 12, - kIncSize = 3 + kTimestampSize = 4, + kInstanceUniqueSize = 5, + kIncrementSize = 3 }; /** init from a 24 char hex std::string */ - explicit OID(const std::string &s) { init(s); } + explicit OID(const std::string &s) { + init(s); + } /** init from a reference to a 12-byte array */ explicit OID(const unsigned char (&arr)[kOIDSize]) { - memcpy(data, arr, sizeof(arr)); + std::memcpy(_data, arr, sizeof(arr)); } /** initialize to 'null' */ - void clear() { a = 0; b = 0; } - - const unsigned char *getData() const { return data; } + void clear() { std::memset(_data, 0, kOIDSize); } - bool operator==(const OID& r) const { return a==r.a && b==r.b; } - bool operator!=(const OID& r) const { return a!=r.a || b!=r.b; } - int compare( const OID& other ) const { return memcmp( data , other.data , kOIDSize ); } - bool operator<( const OID& other ) const { return compare( other ) < 0; } - bool operator<=( const OID& other ) const { return compare( other ) <= 0; } + int compare( const OID& other ) const { return memcmp( _data , other._data , kOIDSize ); } /** @return the object ID output as 24 hex digits */ - std::string str() const { return toHexLower(data, kOIDSize); } - std::string toString() const { return str(); } + std::string toString() const; /** @return the random/sequential part of the object ID as 6 hex digits */ - std::string toIncString() const { return toHexLower(_inc, kIncSize); } + std::string toIncString() const; - static OID gen() { OID o; o.init(); return o; } + static OID gen() { + OID o((no_initialize_tag())); + o.init(); + return o; + } + + // Caller must ensure that the buffer is valid for kOIDSize bytes. + // this is templated because some places use unsigned char vs signed char + template<typename T> + static OID from(T* buf) { + OID o((no_initialize_tag())); + std::memcpy(o._data, buf, OID::kOIDSize); + return o; + } + + static OID max() { + OID o((no_initialize_tag())); + std::memset(o._data, 0xFF, kOIDSize); + return o; + } /** sets the contents to a new oid / randomized value */ void init(); - /** sets the contents to a new oid - * guaranteed to be sequential - * NOT guaranteed to be globally unique - * only unique for this process - * */ - void initSequential(); - /** init from a 24 char hex std::string */ void init( const std::string& s ); @@ -114,9 +140,12 @@ namespace mongo { void init( Date_t date, bool max=false ); time_t asTimeT(); - Date_t asDateT() { return asTimeT() * (long long)1000; } + Date_t asDateT() { return asTimeT() * 1000LL; } - bool isSet() const { return a || b; } + // True iff the OID is not empty + bool isSet() const { + return compare(OID()) != 0; + } /** * this is not consistent @@ -127,45 +156,57 @@ namespace mongo { /** call this after a fork to update the process id */ static void justForked(); - static unsigned getMachineId(); // features command uses - static void regenMachineId(); // used by unit tests + static unsigned getMachineId(); // used by the 'features' command + static void regenMachineId(); - private: - struct MachineAndPid { - unsigned char _machineNumber[3]; - unsigned short _pid; - bool operator!=(const OID::MachineAndPid& rhs) const; + // Timestamp is 4 bytes so we just use int32_t + typedef int32_t Timestamp; + + // Wrappers so we can return stuff by value. + struct InstanceUnique { + static InstanceUnique generate(SecureRandom& entropy); + uint8_t bytes[kInstanceUniqueSize]; }; - static MachineAndPid ourMachine, ourMachineAndPid; - union { - struct { - // 12 bytes total - unsigned char _time[4]; - MachineAndPid _machineAndPid; - unsigned char _inc[3]; - }; - struct { - long long a; - unsigned b; - }; - struct { - // TODO: get rid of this eventually - // this is a hack because of hash_combine with older versions of boost - // on 32-bit platforms - int x; - int y; - int z; - }; - unsigned char data[kOIDSize]; + + struct Increment { + public: + static Increment next(); + uint8_t bytes[kIncrementSize]; }; - static void foldInPid(MachineAndPid& x); - static MachineAndPid genMachineAndPid(); + void setTimestamp(Timestamp timestamp); + void setInstanceUnique(InstanceUnique unique); + void setIncrement(Increment inc); + + Timestamp getTimestamp() const; + InstanceUnique getInstanceUnique() const; + Increment getIncrement() const; + + ConstDataView view() const { + return ConstDataView(_data); + } + + private: + // Internal mutable view + DataView _view() { + return DataView(_data); + } + + // When we are going to immediately overwrite the bytes, there is no point in zero + // initializing the data first. + struct no_initialize_tag {}; + explicit OID(no_initialize_tag) {} + + char _data[kOIDSize]; }; -#pragma pack() - std::ostream& operator<<( std::ostream &s, const OID &o ); - inline StringBuilder& operator<< (StringBuilder& s, const OID& o) { return (s << o.str()); } + inline std::ostream& operator<<(std::ostream &s, const OID &o) { + return (s << o.toString()); + } + + inline StringBuilder& operator<<(StringBuilder& s, const OID& o) { + return (s << o.toString()); + } /** Formatting mode for generating JSON from BSON. See <http://dochub.mongodb.org/core/mongodbextendedjson> @@ -181,4 +222,9 @@ namespace mongo { JS }; -} + inline bool operator==(const OID& lhs, const OID& rhs) { return lhs.compare(rhs) == 0; } + inline bool operator!=(const OID& lhs, const OID& rhs) { return lhs.compare(rhs) != 0; } + inline bool operator<(const OID& lhs, const OID& rhs) { return lhs.compare(rhs) < 0; } + inline bool operator<=(const OID& lhs, const OID& rhs) { return lhs.compare(rhs) <= 0; } + +} // namespace mongo diff --git a/src/mongo/bson/oid_test.cpp b/src/mongo/bson/oid_test.cpp new file mode 100644 index 00000000000..437acad54d2 --- /dev/null +++ b/src/mongo/bson/oid_test.cpp @@ -0,0 +1,147 @@ +/* Copyright 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/bson/oid.h" + +#include "mongo/platform/endian.h" +#include "mongo/unittest/unittest.h" + +namespace { + + using mongo::OID; + + TEST(Equals, Simple) { + OID o1 = OID::gen(); + + ASSERT_EQUALS(o1, o1); + ASSERT_TRUE(o1 == o1); + ASSERT_EQUALS(o1.compare(o1), 0); + } + + TEST(NotEquals, Simple) { + OID o1 = OID::gen(); + OID o2 = OID::gen(); + + ASSERT_FALSE(o1 == o2); + ASSERT_TRUE(o1 != o2); + ASSERT_NOT_EQUALS(o1.compare(o2), 0); + } + + TEST(Increasing, Simple) { + OID o1 = OID::gen(); + OID o2 = OID::gen(); + ASSERT_TRUE(o1 < o2); + } + + TEST(IsSet, Simple) { + OID o; + ASSERT_FALSE(o.isSet()); + o.init(); + ASSERT_TRUE(o.isSet()); + } + + TEST(JustForked, Simple) { + OID o1 = OID::gen(); + OID::justForked(); + OID o2 = OID::gen(); + + ASSERT_TRUE(std::memcmp(o1.getInstanceUnique().bytes, o2.getInstanceUnique().bytes, + OID::kInstanceUniqueSize) != 0); + } + + TEST(TimestampIsBigEndian, Endianness) { + OID o1; // zeroed + OID::Timestamp ts = 123; + o1.setTimestamp(ts); + + int32_t ts_big = mongo::endian::nativeToBig<int32_t>(123); + + const char* oidBytes = o1.view().view(); + ASSERT(std::memcmp(&ts_big, oidBytes, sizeof(int32_t)) == 0); + } + + TEST(IncrementIsBigEndian, Endianness) { + OID o1; // zeroed + OID::Increment incr; + // Increment is a 3 byte counter big endian + incr.bytes[0] = 0xBEu; + incr.bytes[1] = 0xADu; + incr.bytes[2] = 0xDEu; + + o1.setIncrement(incr); + + const char* oidBytes = o1.view().view(); + oidBytes += OID::kTimestampSize + OID::kInstanceUniqueSize; + + // now at start of increment + ASSERT_EQUALS(uint8_t(oidBytes[0]), 0xBEu); + ASSERT_EQUALS(uint8_t(oidBytes[1]), 0xADu); + ASSERT_EQUALS(uint8_t(oidBytes[2]), 0xDEu); + } + + TEST(Basic, Deserialize) { + + uint8_t OIDbytes[] = { + 0xDEu, 0xADu, 0xBEu, 0xEFu, // timestamp is -559038737 (signed) + 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, // unique is 0 + 0x11u, 0x22u, 0x33u // increment is 1122867 + }; + + OID o1 = OID::from(OIDbytes); + + ASSERT_EQUALS(o1.getTimestamp(), -559038737); + OID::InstanceUnique u = o1.getInstanceUnique(); + for (std::size_t i = 0; i < OID::kInstanceUniqueSize; ++i) { + ASSERT_EQUALS(u.bytes[i], 0x00u); + } + OID::Increment i = o1.getIncrement(); + + // construct a uint32_t from increment + // recall that i is a big-endian 3 byte unsigned integer + uint32_t incr = + ((uint32_t(i.bytes[0]) << 16)) | + ((uint32_t(i.bytes[1]) << 8)) | + uint32_t(i.bytes[2]); + + ASSERT_EQUALS(1122867u, incr); + } + + TEST(Basic, FromString) { + + std::string oidStr("541b1a00e8a23afa832b218e"); + uint8_t oidBytes[] = {0x54u, 0x1Bu, 0x1Au, 0x00u, + 0xE8u, 0xA2u, 0x3Au, 0xFAu, + 0x83u, 0x2Bu, 0x21u, 0x8Eu}; + + ASSERT_EQUALS(OID(oidStr), OID::from(oidBytes)); + } + + TEST(Basic, FromStringToString) { + std::string fromStr("541b1a00e8a23afa832b218e"); + ASSERT_EQUALS(OID(fromStr).toString(), fromStr); + } +} diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 85a1e828e3f..2917c8087bf 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -128,12 +128,6 @@ namespace mongo { Timer startupSrandTimer; - struct MyStartupTests { - MyStartupTests() { - verify( sizeof(OID) == 12 ); - } - } mystartupdbcpp; - QueryResult::View emptyMoreResult(long long); @@ -829,7 +823,7 @@ static void startupConfigActions(const std::vector<std::string>& args) { } MONGO_INITIALIZER_GENERAL(CreateAuthorizationManager, - ("SetupInternalSecurityUser"), + ("SetupInternalSecurityUser", "OIDGeneration"), MONGO_NO_DEPENDENTS) (InitializerContext* context) { AuthorizationManager* authzManager = diff --git a/src/mongo/db/driverHelpers.cpp b/src/mongo/db/driverHelpers.cpp index 3b37a39cafa..8ff1a38b3bd 100644 --- a/src/mongo/db/driverHelpers.cpp +++ b/src/mongo/db/driverHelpers.cpp @@ -73,7 +73,7 @@ namespace mongo { const OID& oid = cmdObj.firstElement().__oid(); result.append( "oid" , oid ); - result.append( "str" , oid.str() ); + result.append( "str" , oid.toString() ); return true; } diff --git a/src/mongo/db/index/external_key_generator.cpp b/src/mongo/db/index/external_key_generator.cpp index ffa6675f20f..9f2e053e848 100644 --- a/src/mongo/db/index/external_key_generator.cpp +++ b/src/mongo/db/index/external_key_generator.cpp @@ -117,7 +117,7 @@ namespace { break; case jstOID: size += 1; - size += sizeof(OID); + size += OID::kOIDSize; break; case BinData: { diff --git a/src/mongo/db/pipeline/value.cpp b/src/mongo/db/pipeline/value.cpp index a76c9ea4325..06b98118306 100644 --- a/src/mongo/db/pipeline/value.cpp +++ b/src/mongo/db/pipeline/value.cpp @@ -34,6 +34,7 @@ #include "mongo/db/jsobj.h" #include "mongo/db/pipeline/document.h" +#include "mongo/util/hex.h" #include "mongo/util/mongoutils/str.h" namespace mongo { @@ -166,8 +167,8 @@ namespace mongo { } case jstOID: - BOOST_STATIC_ASSERT(sizeof(_storage.oid) == sizeof(OID)); - memcpy(_storage.oid, elem.OID().getData(), sizeof(OID)); + BOOST_STATIC_ASSERT(sizeof(_storage.oid) == OID::kOIDSize); + memcpy(_storage.oid, elem.OID().view().view(), OID::kOIDSize); break; case Bool: @@ -600,7 +601,7 @@ namespace mongo { } case jstOID: - return memcmp(rL._storage.oid, rR._storage.oid, sizeof(OID)); + return memcmp(rL._storage.oid, rR._storage.oid, OID::kOIDSize); case Code: case Symbol: @@ -1011,7 +1012,7 @@ namespace mongo { return Value(ValueStorage(type)); // simple types - case jstOID: return Value(buf.read<OID>()); + case jstOID: return Value(OID::from(buf.skip(OID::kOIDSize))); case NumberInt: return Value(buf.read<int>()); case NumberLong: return Value(buf.read<long long>()); case NumberDouble: return Value(buf.read<double>()); @@ -1046,7 +1047,7 @@ namespace mongo { Document::SorterDeserializeSettings())); case DBRef: { - OID oid = buf.read<OID>(); + OID oid = OID::from(buf.skip(OID::kOIDSize)); StringData ns = buf.readCStr(); return Value(BSONDBRef(ns, oid)); } diff --git a/src/mongo/db/pipeline/value_internal.h b/src/mongo/db/pipeline/value_internal.h index 818c8c8454a..e5f0b080c12 100644 --- a/src/mongo/db/pipeline/value_internal.h +++ b/src/mongo/db/pipeline/value_internal.h @@ -91,8 +91,7 @@ namespace mongo { ValueStorage(BSONType t, const OID& o) { zero(); type = t; - memcpy(&oid, &o, sizeof(OID)); - BOOST_STATIC_ASSERT(sizeof(OID) == sizeof(oid)); + memcpy(&oid, o.view().view(), OID::kOIDSize); } ValueStorage(const ValueStorage& rhs) { diff --git a/src/mongo/db/storage/mmap_v1/btree/key.cpp b/src/mongo/db/storage/mmap_v1/btree/key.cpp index 79c89dd7cb3..16f95527f45 100644 --- a/src/mongo/db/storage/mmap_v1/btree/key.cpp +++ b/src/mongo/db/storage/mmap_v1/btree/key.cpp @@ -97,7 +97,7 @@ namespace mongo { return x == 0 ? 0 : 1; } case jstOID: - return memcmp(l.value(), r.value(), 12); + return memcmp(l.value(), r.value(), OID::kOIDSize); case Code: case Symbol: case String: @@ -276,7 +276,7 @@ namespace mongo { break; case jstOID: b.appendUChar(coid|bits); - b.appendStruct(e.__oid()); + b.appendBuf(e.__oid().view().view(), OID::kOIDSize); break; case BinData: { @@ -396,9 +396,12 @@ namespace mongo { break; } case coid: - b.appendOID("", (OID *) p); - p += sizeof(OID); - break; + { + OID oid = OID::from(p); + b.appendOID("", &oid); + p += OID::kOIDSize; + break; + } case cbindata: { int len = binDataCodeToLength(*p); @@ -509,10 +512,10 @@ namespace mongo { } case coid: { - int res = memcmp(l, r, sizeof(OID)); - if( res ) + int res = memcmp(l, r, OID::kOIDSize); + if( res ) return res; - l += 12; r += 12; + l += OID::kOIDSize; r += OID::kOIDSize; break; } default: diff --git a/src/mongo/db/storage/mmap_v1/dur_journal.cpp b/src/mongo/db/storage/mmap_v1/dur_journal.cpp index dc2d7d93de8..720e8c6c757 100644 --- a/src/mongo/db/storage/mmap_v1/dur_journal.cpp +++ b/src/mongo/db/storage/mmap_v1/dur_journal.cpp @@ -50,6 +50,7 @@ #include "mongo/util/checksum.h" #include "mongo/util/compress.h" #include "mongo/util/file.h" +#include "mongo/util/hex.h" #include "mongo/util/log.h" #include "mongo/util/logfile.h" #include "mongo/util/mmap.h" diff --git a/src/mongo/db/storage/mmap_v1/extent.cpp b/src/mongo/db/storage/mmap_v1/extent.cpp index 43998a36ed2..4c880ee07d1 100644 --- a/src/mongo/db/storage/mmap_v1/extent.cpp +++ b/src/mongo/db/storage/mmap_v1/extent.cpp @@ -31,6 +31,7 @@ #include "mongo/db/storage/mmap_v1/extent.h" #include "mongo/db/storage/mmap_v1/extent_manager.h" +#include "mongo/util/hex.h" #include "mongo/util/mongoutils/str.h" namespace mongo { diff --git a/src/mongo/dbtests/dbtests.cpp b/src/mongo/dbtests/dbtests.cpp index e040301bde3..e9423d0906c 100644 --- a/src/mongo/dbtests/dbtests.cpp +++ b/src/mongo/dbtests/dbtests.cpp @@ -56,13 +56,13 @@ namespace mongo { int dbtestsMain( int argc, char** argv, char** envp ) { static StaticObserver StaticObserver; setWindowsUnhandledExceptionFilter(); - setGlobalAuthorizationManager(new AuthorizationManager(new AuthzManagerExternalStateMock())); setGlobalEnvironment(new GlobalEnvironmentMongoD()); repl::ReplSettings replSettings; replSettings.oplogSize = 10 * 1024 * 1024; repl::setGlobalReplicationCoordinator(new repl::ReplicationCoordinatorMock(replSettings)); Command::testCommandsEnabled = 1; mongo::runGlobalInitializersOrDie(argc, argv, envp); + setGlobalAuthorizationManager(new AuthorizationManager(new AuthzManagerExternalStateMock())); StartupTest::runTests(); return mongo::dbtests::runDbTests(argc, argv); } diff --git a/src/mongo/dbtests/jsobjtests.cpp b/src/mongo/dbtests/jsobjtests.cpp index 2b4e7918017..23bbb7b9707 100644 --- a/src/mongo/dbtests/jsobjtests.cpp +++ b/src/mongo/dbtests/jsobjtests.cpp @@ -1242,7 +1242,7 @@ namespace JsobjTests { OID b; a.init(); - b.init( a.str() ); + b.init( a.toString() ); ASSERT( a == b ); } @@ -1258,9 +1258,9 @@ namespace JsobjTests { BSONObj o = b.obj(); keyTest(o); - ASSERT( o["a"].__oid().str() == "000000000000000000000000" ); - ASSERT( o["b"].__oid().str() == "000000000000000000000000" ); - ASSERT( o["c"].__oid().str() != "000000000000000000000000" ); + ASSERT( o["a"].__oid().toString() == "000000000000000000000000" ); + ASSERT( o["b"].__oid().toString() == "000000000000000000000000" ); + ASSERT( o["c"].__oid().toString() != "000000000000000000000000" ); } }; @@ -1316,20 +1316,6 @@ namespace JsobjTests { } }; - class Seq { - public: - void run() { - for ( int i=0; i<10000; i++ ) { - OID a; - OID b; - - a.initSequential(); - b.initSequential(); - - ASSERT( a < b ); - } - } - }; } // namespace OIDTests @@ -2285,7 +2271,6 @@ namespace JsobjTests { add< OIDTests::increasing >(); add< OIDTests::ToDate >(); add< OIDTests::FromDate >(); - add< OIDTests::Seq >(); add< ValueStreamTests::LabelBasic >(); add< ValueStreamTests::LabelShares >(); add< ValueStreamTests::LabelDouble >(); diff --git a/src/mongo/dbtests/jsontests.cpp b/src/mongo/dbtests/jsontests.cpp index 97477327984..2b198465558 100644 --- a/src/mongo/dbtests/jsontests.cpp +++ b/src/mongo/dbtests/jsontests.cpp @@ -285,8 +285,9 @@ namespace JsonTests { class DBRef { public: void run() { - OID oid; - memset( &oid, 0xff, 12 ); + char OIDbytes[OID::kOIDSize]; + memset( &OIDbytes, 0xff, OID::kOIDSize ); + OID oid = OID::from(OIDbytes); BSONObjBuilder b; b.appendDBRef( "a", "namespace", oid ); BSONObj built = b.done(); @@ -302,8 +303,9 @@ namespace JsonTests { class DBRefZero { public: void run() { - OID oid; - memset( &oid, 0, 12 ); + char OIDbytes[OID::kOIDSize]; + memset( &OIDbytes, 0, OID::kOIDSize ); + OID oid = OID::from(OIDbytes); BSONObjBuilder b; b.appendDBRef( "a", "namespace", oid ); ASSERT_EQUALS( "{ \"a\" : { \"$ref\" : \"namespace\", \"$id\" : \"000000000000000000000000\" } }", @@ -314,8 +316,9 @@ namespace JsonTests { class ObjectId { public: void run() { - OID oid; - memset( &oid, 0xff, 12 ); + char OIDbytes[OID::kOIDSize]; + memset( &OIDbytes, 0xff, OID::kOIDSize ); + OID oid = OID::from(OIDbytes); BSONObjBuilder b; b.appendOID( "a", &oid ); BSONObj built = b.done(); @@ -1174,7 +1177,6 @@ namespace JsonTests { virtual BSONObj bson() const { BSONObjBuilder b; OID o; - memset( &o, 0, 12 ); BSONObjBuilder subBuilder(b.subobjStart("a")); subBuilder.append("$ref", "ns"); subBuilder.append("$id", o); @@ -1190,7 +1192,6 @@ namespace JsonTests { virtual BSONObj bson() const { BSONObjBuilder b; OID o; - memset( &o, 0, 12 ); BSONObjBuilder subBuilder(b.subobjStart("a")); subBuilder.append("$ref", "ns"); subBuilder.append("$id", o); @@ -1232,8 +1233,9 @@ namespace JsonTests { class Oid2 : public Base { virtual BSONObj bson() const { BSONObjBuilder b; - OID o; - memset( &o, 0x0f, 12 ); + char OIDbytes[OID::kOIDSize]; + memset( &OIDbytes, 0x0f, OID::kOIDSize ); + OID o = OID::from(OIDbytes); b.appendOID( "_id", &o ); return b.obj(); } diff --git a/src/mongo/s/server.cpp b/src/mongo/s/server.cpp index 459af3f2bab..2674e62ee50 100644 --- a/src/mongo/s/server.cpp +++ b/src/mongo/s/server.cpp @@ -393,7 +393,7 @@ namespace mongo { #endif MONGO_INITIALIZER_GENERAL(CreateAuthorizationManager, - ("SetupInternalSecurityUser"), + ("SetupInternalSecurityUser", "OIDGeneration"), MONGO_NO_DEPENDENTS) (InitializerContext* context) { AuthorizationManager* authzManager = diff --git a/src/mongo/scripting/engine_v8.cpp b/src/mongo/scripting/engine_v8.cpp index 71c14260441..b4da62fbe21 100644 --- a/src/mongo/scripting/engine_v8.cpp +++ b/src/mongo/scripting/engine_v8.cpp @@ -1353,7 +1353,7 @@ namespace mongo { v8::HandleScope handle_scope; v8::Handle<v8::Function> idCons = ObjectIdFT()->GetFunction(); v8::Handle<v8::Value> argv[1]; - const string& idString = id.str(); + const string& idString = id.toString(); argv[0] = v8::String::New(idString.c_str(), idString.length()); return handle_scope.Close(idCons->NewInstance(1, argv)); } diff --git a/src/mongo/scripting/v8_db.cpp b/src/mongo/scripting/v8_db.cpp index c6b94282bff..d93822627e8 100644 --- a/src/mongo/scripting/v8_db.cpp +++ b/src/mongo/scripting/v8_db.cpp @@ -42,6 +42,7 @@ #include "mongo/scripting/v8_utils.h" #include "mongo/util/assert_util.h" #include "mongo/util/base64.h" +#include "mongo/util/hex.h" #include "mongo/util/text.h" using namespace std; @@ -689,7 +690,7 @@ namespace mongo { oid.init(s); } - it->ForceSet(scope->v8StringData("str"), v8::String::New(oid.str().c_str())); + it->ForceSet(scope->v8StringData("str"), v8::String::New(oid.toString().c_str())); return it; } diff --git a/src/mongo/tools/export.cpp b/src/mongo/tools/export.cpp index 7ea9793ba2a..ce34b5039d5 100644 --- a/src/mongo/tools/export.cpp +++ b/src/mongo/tools/export.cpp @@ -38,6 +38,7 @@ #include "mongo/tools/mongoexport_options.h" #include "mongo/tools/tool.h" #include "mongo/tools/tool_logger.h" +#include "mongo/util/hex.h" #include "mongo/util/options_parser/option_section.h" using namespace mongo; |