bool _compact(const char *ns, NamespaceDetails *d, string& errmsg, bool validate, BSONObjBuilder& result, double pf, int pb) {
//int les = d->lastExtentSize;
// this is a big job, so might as well make things tidy before we start just to be nice.
getDur().commitNow();
list<DiskLoc> extents;
for( DiskLoc L = d->firstExtent; !L.isNull(); L = L.ext()->xnext )
extents.push_back(L);
log() << "compact " << extents.size() << " extents" << endl;
ProgressMeterHolder pm( cc().curop()->setMessage( "compact extent" , extents.size() ) );
// same data, but might perform a little different after compact?
NamespaceDetailsTransient::get(ns).clearQueryCache();
int nidx = d->nIndexes;
scoped_array<IndexSpec> indexSpecs( new IndexSpec[nidx] );
scoped_array<SortPhaseOne> phase1( new SortPhaseOne[nidx] );
{
NamespaceDetails::IndexIterator ii = d->ii();
int x = 0;
while( ii.more() ) {
BSONObjBuilder b;
IndexDetails& idx = ii.next();
BSONObj::iterator i(idx.info.obj());
while( i.more() ) {
BSONElement e = i.next();
if( !str::equals(e.fieldName(), "v") && !str::equals(e.fieldName(), "background") ) {
b.append(e);
}
}
BSONObj o = b.obj().getOwned();
phase1[x].sorter.reset( new BSONObjExternalSorter( idx.idxInterface(), o.getObjectField("key") ) );
phase1[x].sorter->hintNumObjects( d->stats.nrecords );
indexSpecs[x++].reset(o);
}
}
log() << "compact orphan deleted lists" << endl;
for( int i = 0; i < Buckets; i++ ) {
d->deletedList[i].writing().Null();
}
// before dropping indexes, at least make sure we can allocate one extent!
uassert(14025, "compact error no space available to allocate", !allocateSpaceForANewRecord(ns, d, Record::HeaderSize+1, false).isNull());
// note that the drop indexes call also invalidates all clientcursors for the namespace, which is important and wanted here
log() << "compact dropping indexes" << endl;
BSONObjBuilder b;
if( !dropIndexes(d, ns, "*", errmsg, b, true) ) {
errmsg = "compact drop indexes failed";
log() << errmsg << endl;
return false;
}
getDur().commitNow();
long long skipped = 0;
int n = 0;
for( list<DiskLoc>::iterator i = extents.begin(); i != extents.end(); i++ ) {
skipped += compactExtent(ns, d, *i, n++, indexSpecs, phase1, nidx, validate, pf, pb);
pm.hit();
}
if( skipped ) {
result.append("invalidObjects", skipped);
}
assert( d->firstExtent.ext()->xprev.isNull() );
// indexes will do their own progress meter?
pm.finished();
// build indexes
NamespaceString s(ns);
string si = s.db + ".system.indexes";
for( int i = 0; i < nidx; i++ ) {
killCurrentOp.checkForInterrupt(false);
BSONObj info = indexSpecs[i].info;
log() << "compact create index " << info["key"].Obj().toString() << endl;
try {
precalced = &phase1[i];
theDataFileMgr.insert(si.c_str(), info.objdata(), info.objsize());
}
catch(...) {
precalced = 0;
throw;
}
precalced = 0;
}
return true;
}
开发者ID:adi2188，项目名称:mongo，代码行数:94，代码来源:compact.cpp
示例8: validateNS
//.........这里部分代码省略.........
if ( cl < cl_last )
outOfOrder++;
cl_last = cl;
}
Record *r = c->_current();
len += r->lengthWithHeaders;
nlen += r->netLength();
c->advance();
}
if ( d->capped && !d->capLooped() ) {
ss << " capped outOfOrder:" << outOfOrder;
if ( outOfOrder > 1 ) {
valid = false;
ss << " ???";
}
else ss << " (OK)";
ss << '\n';
}
ss << " " << n << " objects found, nobj:" << d->stats.nrecords << '\n';
ss << " " << len << " bytes data w/headers\n";
ss << " " << nlen << " bytes data wout/headers\n";
}
ss << " deletedList: ";
for ( int i = 0; i < Buckets; i++ ) {
ss << (d->deletedList[i].isNull() ? '0' : '1');
}
ss << endl;
int ndel = 0;
long long delSize = 0;
int incorrect = 0;
for ( int i = 0; i < Buckets; i++ ) {
DiskLoc loc = d->deletedList[i];
try {
int k = 0;
while ( !loc.isNull() ) {
if ( recs.count(loc) )
incorrect++;
ndel++;
if ( loc.questionable() ) {
if( d->capped && !loc.isValid() && i == 1 ) {
/* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid
see comments in namespace.h
*/
break;
}
if ( loc.a() <= 0 || strstr(ns, "hudsonSmall") == 0 ) {
ss << " ?bad deleted loc: " << loc.toString() << " bucket:" << i << " k:" << k << endl;
valid = false;
break;
}
}
DeletedRecord *d = loc.drec();
delSize += d->lengthWithHeaders;
loc = d->nextDeleted;
k++;
killCurrentOp.checkForInterrupt();
}
}
catch (...) {
ss <<" ?exception in deleted chain for bucket " << i << endl;
valid = false;
}
}
ss << " deleted: n: " << ndel << " size: " << delSize << endl;
if ( incorrect ) {
ss << " ?corrupt: " << incorrect << " records from datafile are in deleted list\n";
valid = false;
}
int idxn = 0;
try {
ss << " nIndexes:" << d->nIndexes << endl;
NamespaceDetails::IndexIterator i = d->ii();
while( i.more() ) {
IndexDetails& id = i.next();
ss << " " << id.indexNamespace() << " keys:" <<
id.head.btree()->fullValidate(id.head, id.keyPattern()) << endl;
}
}
catch (...) {
ss << "\n exception during index validate idxn:" << idxn << endl;
valid=false;
}
}
catch (AssertionException) {
ss << "\n exception during validate\n" << endl;
valid = false;
}
if ( !valid )
ss << " ns corrupt, requires dbchk\n";
return ss.str();
}
bool _compact(const char *ns, NamespaceDetails *d, string& errmsg, bool validate,
BSONObjBuilder& result, double pf, int pb, bool useDefaultPadding,
bool preservePadding) {
// this is a big job, so might as well make things tidy before we start just to be nice.
getDur().commitIfNeeded();
list<DiskLoc> extents;
for( DiskLoc L = d->firstExtent(); !L.isNull(); L = L.ext()->xnext )
extents.push_back(L);
log() << "compact " << extents.size() << " extents" << endl;
ProgressMeterHolder pm(cc().curop()->setMessage("compact extent",
"Extent Compacting Progress",
extents.size()));
// same data, but might perform a little different after compact?
Collection* collection = cc().database()->getCollection( ns );
verify( collection );
collection->infoCache()->addedIndex();
verify( d->getCompletedIndexCount() == d->getTotalIndexCount() );
int nidx = d->getCompletedIndexCount();
scoped_array<BSONObj> indexSpecs( new BSONObj[nidx] );
{
NamespaceDetails::IndexIterator ii = d->ii();
// For each existing index...
for( int idxNo = 0; ii.more(); ++idxNo ) {
// Build a new index spec based on the old index spec.
BSONObjBuilder b;
BSONObj::iterator i(ii.next().info.obj());
while( i.more() ) {
BSONElement e = i.next();
if ( str::equals( e.fieldName(), "v" ) ) {
// Drop any preexisting index version spec. The default index version will
// be used instead for the new index.
continue;
}
if ( str::equals( e.fieldName(), "background" ) ) {
// Create the new index in the foreground.
continue;
}
// Pass the element through to the new index spec.
b.append(e);
}
indexSpecs[idxNo] = b.obj().getOwned();
}
}
log() << "compact orphan deleted lists" << endl;
d->orphanDeletedList();
// Start over from scratch with our extent sizing and growth
d->setLastExtentSize( 0 );
// before dropping indexes, at least make sure we can allocate one extent!
uassert(14025, "compact error no space available to allocate", !allocateSpaceForANewRecord(ns, d, Record::HeaderSize+1, false).isNull());
// note that the drop indexes call also invalidates all clientcursors for the namespace, which is important and wanted here
log() << "compact dropping indexes" << endl;
Status status = collection->getIndexCatalog()->dropAllIndexes( true );
if ( !status.isOK() ) {
errmsg = str::stream() << "compact drop indexes failed: " << status.toString();
log() << status.toString() << endl;
return false;
}
getDur().commitIfNeeded();
long long skipped = 0;
int n = 0;
// reset data size and record counts to 0 for this namespace
// as we're about to tally them up again for each new extent
d->setStats( 0, 0 );
for( list<DiskLoc>::iterator i = extents.begin(); i != extents.end(); i++ ) {
skipped += compactExtent(ns, d, *i, n++, nidx, validate, pf, pb,
useDefaultPadding, preservePadding);
pm.hit();
}
if( skipped ) {
result.append("invalidObjects", skipped);
}
verify( d->firstExtent().ext()->xprev.isNull() );
// indexes will do their own progress meter?
pm.finished();
// build indexes
for( int i = 0; i < nidx; i++ ) {
killCurrentOp.checkForInterrupt(false);
BSONObj info = indexSpecs[i];
log() << "compact create index " << info["key"].Obj().toString() << endl;
Status status = collection->getIndexCatalog()->createIndex( info, false );
if ( !status.isOK() ) {
log() << "failed to create index: " << status.toString();
uassertStatusOK( status );
}
//.........这里部分代码省略.........
bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) {
Timer t;
string ns = dbname + '.' + cmdObj.firstElement().valuestr();
string key = cmdObj["key"].valuestrsafe();
BSONObj keyPattern = BSON( key << 1 );
BSONObj query = getQuery( cmdObj );
int bufSize = BSONObjMaxUserSize - 4096;
BufBuilder bb( bufSize );
char * start = bb.buf();
BSONArrayBuilder arr( bb );
BSONElementSet values;
long long nscanned = 0; // locations looked at
long long nscannedObjects = 0; // full objects looked at
long long n = 0; // matches
MatchDetails md;
NamespaceDetails * d = nsdetails( ns );
if ( ! d ) {
result.appendArray( "values" , BSONObj() );
result.append( "stats" , BSON( "n" << 0 << "nscanned" << 0 << "nscannedObjects" << 0 ) );
return true;
}
shared_ptr<Cursor> cursor;
if ( ! query.isEmpty() ) {
cursor = getOptimizedCursor( ns.c_str(), query, BSONObj() );
}
else {
// query is empty, so lets see if we can find an index
// with the key so we don't have to hit the raw data
NamespaceDetails::IndexIterator ii = d->ii();
while ( ii.more() ) {
IndexDetails& idx = ii.next();
if ( d->isMultikey( ii.pos() - 1 ) )
continue;
if ( idx.inKeyPattern( key ) ) {
cursor = getBestGuessCursor( ns.c_str(), BSONObj(), idx.keyPattern() );
if( cursor.get() ) break;
}
}
if ( ! cursor.get() )
cursor = getOptimizedCursor(ns.c_str() , query , BSONObj() );
}
verify( cursor );
string cursorName = cursor->toString();
auto_ptr<ClientCursor> cc (new ClientCursor(QueryOption_NoCursorTimeout, cursor, ns));
// map from indexed field to offset in key object
map<string, int> indexedFields;
if (!cursor->modifiedKeys()) {
// store index information so we can decide if we can
// get something out of the index key rather than full object
int x = 0;
BSONObjIterator i( cursor->indexKeyPattern() );
while ( i.more() ) {
BSONElement e = i.next();
if ( e.isNumber() ) {
// only want basic index fields, not "2d" etc
indexedFields[e.fieldName()] = x;
}
x++;
}
}
while ( cursor->ok() ) {
nscanned++;
bool loadedRecord = false;
if ( cursor->currentMatches( &md ) && !cursor->getsetdup( cursor->currLoc() ) ) {
n++;
BSONObj holder;
BSONElementSet temp;
// Try to get the record from the key fields.
loadedRecord = !getFieldsDotted(indexedFields, cursor, key, temp, holder);
for ( BSONElementSet::iterator i=temp.begin(); i!=temp.end(); ++i ) {
BSONElement e = *i;
if ( values.count( e ) )
continue;
int now = bb.len();
uassert(10044, "distinct too big, 16mb cap", ( now + e.size() + 1024 ) < bufSize );
//.........这里部分代码省略.........
开发者ID:Axv2，项目名称:mongo，代码行数:101，代码来源:distinct.cpp
示例13: validateNS
//.........这里部分代码省略.........
if (full) {
result.append("invalidObjects", nInvalid);
}
result.appendNumber("bytesWithHeaders", len);
result.appendNumber("bytesWithoutHeaders", nlen);
}
BSONArrayBuilder deletedListArray;
for ( int i = 0; i < Buckets; i++ ) {
deletedListArray << d->deletedList[i].isNull();
}
int ndel = 0;
long long delSize = 0;
int incorrect = 0;
for ( int i = 0; i < Buckets; i++ ) {
DiskLoc loc = d->deletedList[i];
try {
int k = 0;
while ( !loc.isNull() ) {
if ( recs.count(loc) )
incorrect++;
ndel++;
if ( loc.questionable() ) {
if( d->isCapped() && !loc.isValid() && i == 1 ) {
/* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid
see comments in namespace.h
*/
break;
}
if ( loc.a() <= 0 || strstr(ns, "hudsonSmall") == 0 ) {
string err (str::stream() << "bad deleted loc: " << loc.toString() << " bucket:" << i << " k:" << k);
errors << err;
valid = false;
break;
}
}
DeletedRecord *d = loc.drec();
delSize += d->lengthWithHeaders();
loc = d->nextDeleted();
k++;
killCurrentOp.checkForInterrupt();
}
}
catch (...) {
errors << ("exception in deleted chain for bucket " + BSONObjBuilder::numStr(i));
valid = false;
}
}
result.appendNumber("deletedCount", ndel);
result.appendNumber("deletedSize", delSize);
if ( incorrect ) {
errors << (BSONObjBuilder::numStr(incorrect) + " records from datafile are in deleted list");
valid = false;
}
int idxn = 0;
try {
result.append("nIndexes", d->nIndexes);
BSONObjBuilder indexes; // not using subObjStart to be exception safe
NamespaceDetails::IndexIterator i = d->ii();
while( i.more() ) {
IndexDetails& id = i.next();
log() << "validating index " << idxn << ": " << id.indexNamespace() << endl;
long long keys = id.idxInterface().fullValidate(id.head, id.keyPattern());
indexes.appendNumber(id.indexNamespace(), keys);
idxn++;
}
result.append("keysPerIndex", indexes.done());
}
catch (...) {
errors << ("exception during index validate idxn " + BSONObjBuilder::numStr(idxn));
valid=false;
}
}
catch (AssertionException) {
errors << "exception during validate";
valid = false;
}
result.appendBool("valid", valid);
result.append("errors", errors.arr());
if ( !full ){
result.append("warning", "Some checks omitted for speed. use {full:true} option to do more thorough scan.");
}
if ( !valid ) {
result.append("advice", "ns corrupt, requires repair");
}
}
bool _compact(const char *ns, NamespaceDetails *d, string& errmsg, bool validate, BSONObjBuilder& result, double pf, int pb) {
// this is a big job, so might as well make things tidy before we start just to be nice.
getDur().commitIfNeeded();
list<DiskLoc> extents;
for( DiskLoc L = d->firstExtent; !L.isNull(); L = L.ext()->xnext )
extents.push_back(L);
log() << "compact " << extents.size() << " extents" << endl;
ProgressMeterHolder pm( cc().curop()->setMessage( "compact extent" , extents.size() ) );
// same data, but might perform a little different after compact?
NamespaceDetailsTransient::get(ns).clearQueryCache();
int nidx = d->nIndexes;
scoped_array<IndexSpec> indexSpecs( new IndexSpec[nidx] );
scoped_array<SortPhaseOne> phase1( new SortPhaseOne[nidx] );
{
NamespaceDetails::IndexIterator ii = d->ii();
// For each existing index...
for( int idxNo = 0; ii.more(); ++idxNo ) {
// Build a new index spec based on the old index spec.
BSONObjBuilder b;
BSONObj::iterator i(ii.next().info.obj());
while( i.more() ) {
BSONElement e = i.next();
if ( str::equals( e.fieldName(), "v" ) ) {
// Drop any preexisting index version spec. The default index version will
// be used instead for the new index.
continue;
}
if ( str::equals( e.fieldName(), "background" ) ) {
// Create the new index in the foreground.
continue;
}
// Pass the element through to the new index spec.
b.append(e);
}
// Add the new index spec to 'indexSpecs'.
BSONObj o = b.obj().getOwned();
indexSpecs[idxNo].reset(o);
// Create an external sorter.
phase1[idxNo].sorter.reset
( new BSONObjExternalSorter
// Use the default index interface, since the new index will be created
// with the default index version.
( IndexInterface::defaultVersion(),
o.getObjectField("key") ) );
phase1[idxNo].sorter->hintNumObjects( d->stats.nrecords );
}
}
log() << "compact orphan deleted lists" << endl;
for( int i = 0; i < Buckets; i++ ) {
d->deletedList[i].writing().Null();
}
// Start over from scratch with our extent sizing and growth
d->lastExtentSize=0;
// before dropping indexes, at least make sure we can allocate one extent!
uassert(14025, "compact error no space available to allocate", !allocateSpaceForANewRecord(ns, d, Record::HeaderSize+1, false).isNull());
// note that the drop indexes call also invalidates all clientcursors for the namespace, which is important and wanted here
log() << "compact dropping indexes" << endl;
BSONObjBuilder b;
if( !dropIndexes(d, ns, "*", errmsg, b, true) ) {
errmsg = "compact drop indexes failed";
log() << errmsg << endl;
return false;
}
getDur().commitIfNeeded();
long long skipped = 0;
int n = 0;
// reset data size and record counts to 0 for this namespace
// as we're about to tally them up again for each new extent
{
NamespaceDetails::Stats *s = getDur().writing(&d->stats);
s->datasize = 0;
s->nrecords = 0;
}
for( list<DiskLoc>::iterator i = extents.begin(); i != extents.end(); i++ ) {
skipped += compactExtent(ns, d, *i, n++, indexSpecs, phase1, nidx, validate, pf, pb);
pm.hit();
}
if( skipped ) {
result.append("invalidObjects", skipped);
}
verify( d->firstExtent.ext()->xprev.isNull() );
// indexes will do their own progress meter?
pm.finished();
//.........这里部分代码省略.........
客服电话
APP下载
官方微信
请发表评论