From 88d7dfacda8ea601a2dee9076073a5e1518cbd46 Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Tue, 7 Jan 2025 21:08:15 +0000 Subject: [PATCH 01/22] First pass of the rewrite embedded bucket id doesn't match control min timestamp --- embedded-timestamp-mismatch/README.md | 109 ++++++++++++ ...embedded_bucket_id_control_min_mismatch.js | 166 ++++++++++++++++++ 2 files changed, 275 insertions(+) create mode 100644 embedded-timestamp-mismatch/README.md create mode 100644 embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js diff --git a/embedded-timestamp-mismatch/README.md b/embedded-timestamp-mismatch/README.md new file mode 100644 index 0000000..4d90160 --- /dev/null +++ b/embedded-timestamp-mismatch/README.md @@ -0,0 +1,109 @@ +# Troubleshoot timestamp embedded in the bucket's ID does not match the control.min timestamp +For more context on this issue, see [SERVER-94471](https://jira.mongodb.org/browse/SERVER-94471). +## Warning +The scripts provided here are best run by those experienced with MongoDB, ideally with guidance from MongoDB Technical Support. They are provided without warranty or guarantee of any kind (see disclaimer below). If you do not have access to MongoDB Technical Support, consider reaching out to our community at [community.mongodb.com](community.mongodb.com) to ask questions. +Some adaptation of these scripts may be required for your use-case. +If you are using these scripts on your own, we strongly recommend: +* reading all instructions before attempting any action, and ensuring you are prepared for all steps. +* taking regular backups of the [dbpaths](https://docs.mongodb.com/manual/core/backups/#back-up-by-copying-underlying-data-files) of each node, and working off of backups as much as possible. +* reviewing the scripts themselves to understand their behavior. +* testing this process and all scripts on a copy of the environment it is to be run. +* for sharded clusters, disabling the balancer. +# Prerequisites +- This script should be run with a user that has [dbAdmin](https://www.mongodb.com/docs/v6.0/reference/built-in-roles/#mongodb-authrole-dbAdmin) permissions on the database(s) for the affected time-series collection(s). +- If running on Atlas: + - We have already reached out to impacted customers. If we have reached out, you can skip the [Determine if You're Impacted section](#Determine-if-You're-Impacted) and start taking the steps under the [Remediation section](#remediation). + - Additionally, we recommend using the [Atlas Admin](https://www.mongodb.com/docs/atlas/security-add-mongodb-users/#built-in-roles) role. +# Determine if You're Impacted +Please see [SERVER-94471](https://jira.mongodb.org/browse/SERVER-94471) for the affected versions. Users can determine if they have been impacted by running [`validate`](https://www.mongodb.com/docs/v7.0/reference/command/validate/) on their Time Series collections and checking the `validate.errors` field to determine if there are buckets with mismatched versions. +The validation command [can be very impactful](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance). To minimize the performance impact of running validate, issue validate to a secondary and follow [these steps](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance:~:text=Validation%20has%20exclusive,the%20hidden%20node). +Example `validate` run on a standalone/replica set: +``` +// Call validate on a mongod process for replica sets. +coll.validate(); +// The warnings field detects mixed-schema buckets. +{ +"ns" : "db.system.buckets.coll", +... +"errors" : [ + Detected one or more documents in this collection incompatible with time-series + specifications. For more info, see logs with log id 6698300., + ... +], +... +} +``` +with the logs: +``` +..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Time-series bucket [...] field is not in ascending order"}}... +``` +or +``` +..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Time-series bucket is v3 but has its measurements in-order on time"}}... +``` +Example `validate` run on a sharded cluster: +``` +// Call validate on mongos for sharded clusters. +coll.validate(); +// The warnings field detects mixed-schema buckets. +// For sharded clusters, this output is an object with a result for every shard in +// the "raw" field. +{ + "ns" : "db.system.buckets.coll", + ... +"errors" : [ + Detected one or more documents in this collection incompatible with time-series + specifications. For more info, see logs with log id 6698300., + ... +], +... +"raw" : { + "shard-0-name" : { + "ns" : "db.system.buckets.coll" + ... +"errors" : [ + Detected one or more documents in this collection incompatible with time-series + specifications. For more info, see logs with log id 6698300., + ... +], +... +}, +"shard-1-name" : { ... }, ... +} +} +``` +with the logs: +``` +..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Time-series bucket [...] field is not in ascending order"}}... +``` +or +``` +..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Time-series bucket is v3 but has its measurements in-order on time"}}... +``` +# Remediation +## Rewrite Bucket Version in a Time Series Collections +While the script is running, the performance of operations on the time-series collection may be impacted. The script does a scan of the whole collection and performs updates on impacted buckets, which may result in a large load if many buckets are affected. +At a high level, the script remediates errors during collection validation by updating the bucket's version from v2 to v3 and vice versa if a v2 bucket has unsorted data, or if a v3 bucket has sorted data. +The full steps are as follows. For each bucket in the time series collection: +- Detect if the bucket has bucket version mismatch. +- Change the buckets with bucket version mismatch to the correct version. +- Validate that there are no bucket version mismatches. +**Warning**: This script directly modifies `.system.buckets` collection —the underlying buckets of the Time Series collection—in order to remediate performance issues. Under normal circumstances, users should not modify this collection. +Please contact [MongoDB Support](https://support.mongodb.com/welcome) with any questions or concerns regarding running this script. +### Running the remediation script +#### 1. Modify the script by populating `collName` with the name of your collection +``` +// ------------------------------------------------------------------------------------ +// Populate collName with the time-series collection with mixed-schema buckets. +// ------------------------------------------------------------------------------------ +const collName = "your_collection_name"; +``` +#### 2. Connect to your sharded cluster using [mongosh](https://www.mongodb.com/docs/mongodb-shell/): +``` +mongosh --uri +``` +#### 3. Load the script `rewrite_timeseries_bucket_version_mismatch.js` +``` +load("rewrite_timeseries_bucket_version_mismatch.js") +``` +#### 4. Repeat steps 1-3 for each time-series collection that was affected. \ No newline at end of file diff --git a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js new file mode 100644 index 0000000..87a14ed --- /dev/null +++ b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js @@ -0,0 +1,166 @@ + +// --------------------------------------------------------------------------------------- +// The script will, for each bucket in the affected time-series collection: +// 1) Detect if the bucket has a mismatch between the embedded bucket id +// timestamp and the control min timestamp. +// 2) Re-insert the measurements of the timestamp-mismatched bucket +// transactionally. +// a) Unpack the measurements +// b) Insert the measurements back into the collection. These will go into +// new buckets. +// c) Delete the mixed-schema bucket from the collection. +// 3) Validate that there are no buckets with a mismatch between the embedded +// bucket id timestamp and the control min timestamp. +// 4) Tell the server there aren't buckets with a mismatch between the embedded +// bucket id timestamp and the control min timestamp. +// ---------------------------------------------------------------------------------------- +let bucketColl; +let tsOptions; +let tempTimeseriesColl; +let tempTimeseriesBucketsColl; + +function setUp() { + bucketColl = db.getCollection('system.buckets.' + collName); + + // Create a temp collection to store measurements from the mixed-schema + // buckets. + tsOptions = + db.runCommand({listCollections: 1.0, filter: {name: coll.getName()}}) + .cursor.firstBatch[0] + .options.timeseries; + + db.createCollection('temp', {timeseries: tsOptions}); + tempTimeseriesColl = db.getCollection('temp'); + tempTimeseriesBucketsColl = db.getCollection('system.buckets.temp'); +} + +// Main function. +function runFixEmbeddedBucketIdControlMinMismatchProcedure() { + setUp(); + print( + 'Finding when embedded bucket ID timestamps don\'t match the control min timestamps in ' + + collName + ' ...\n'); + let cursor = bucketsColl.find({}, {_id: true, control: true}); + + // Mismatched timestamp buckets will have different types for their + // control.min.parameter and control.max.parameter due to type ordering. + // Iterate through all buckets, checking if the control.min and control.max + // types match. If they do not match, re-insert the bucket. + while (cursor.hasNext()) { + const bucket = cursor.next(); + const bucketId = bucket._id; + const controlMinTimestamp = bucket.control.min + + if (bucketHasMismatchedEmbeddedBucketIdAndControlMin( + bucketId, controlMinTimestamp)) { + reinsertMeasurementsFromBucket(bucket._id); + } + } +} + +function getDateFromObjectId(objectId) { + return new Date(parseInt(objectId.substring(0, 8), 16) * 1000); +} + +// +// Helpers to detect whether a given bucket contains a mismatch between the +// embedded bucket id timestamp and the control min timestamp. +// +// We parse each timestamp to a date and use .getTime() to compare them. +// +function bucketHasMismatchedEmbeddedBucketIdAndControlMin( + bucketId, controlMinTime) { + const oidTimestamp = getDateFromObjectId(bucketId) + const controlMinTimestamp = new Date(Date.parse(controlMinTime)); + return oidTimestamp.getTime() == controlMinTimestamp.getTime() +} + +// +// Helpers to perform the re-insertion procedure. +// +function reinsertMeasurementsFromBucket(bucketId) { + print('Re-inserting measurements from bucket ' + bucketId + '...\n'); + + // Prevent concurrent changes on this bucket by setting control.closed. + bucketColl.updateOne({_id: bucketId}, {$set: {'control.closed': true}}); + + // Get the measurements from the mixed-schema bucket. + print('Getting the measurements from the mixed-schema bucket...\n'); + let measurements; + if (tsOptions.metaField) { + measurements = bucketColl + .aggregate([ + {$match: {_id: bucketId}}, { + $_unpackBucket: { + timeField: tsOptions.timeField, + metaField: tsOptions.metaField, + } + } + ]) + .toArray(); + } else { + measurements = bucketColl + .aggregate([ + {$match: {_id: bucketId}}, { + $_unpackBucket: { + timeField: tsOptions.timeField, + } + } + ]) + .toArray(); + } + + // To avoid network roundtrips, insert measurements in the + // temporary time-series collection in one batch and retry if any errors + // are encountered. + let retryTempInsert; + do { + retryTempInsert = false; + try { + tempTimeseriesBucketsColl.deleteMany({}); + tempTimeseriesColl.insertMany(measurements); + } catch (e) { + print('An error occurred ' + e); + retryTempInsert = true; + } + } while (retryTempInsert); + + // Run the bucket re-insertion in a transaction. It is necessary to + // interact with the buckets collection because transactions are not + // supported on the time-series view. + // Additionally, we want to retry this transaction on transient errors + // since we are touching potentially lots of data, which would cause + // excessive cache dirtying. + let hasTransientError; + do { + hasTransientError = false; + try { + const session = db.getMongo().startSession({retryWrites: true}); + session.startTransaction(); + + const sessionBucketColl = + session.getDatabase(db.getName()) + .getCollection('system.buckets.' + collName); + sessionBucketColl.deleteOne({_id: bucketId}); + const bucketDocs = tempTimeseriesBucketsColl.find().toArray(); + sessionBucketColl.insertMany(bucketDocs); + + session.commitTransaction(); + } catch (e) { + if (!shouldRetryTxnOnTransientError(e)) { + throw e; + } + hasTransientError = true; + print('Encountered a transient error. Retrying transaction.'); + continue; + } + } while (hasTransientError); +} + +function shouldRetryTxnOnTransientError(e) { + if ((e.hasOwnProperty('errorLabels') && + e.errorLabels.includes('TransientTransactionError'))) { + return true; + } + return false; +} From 6b8bec35e399b0b7b4b86711a8f4ebbba14f494b Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Wed, 8 Jan 2025 22:00:57 +0000 Subject: [PATCH 02/22] Fixed the script --- ...rite_timeseries_bucket_version_mismatch.js | 12 +- ...embedded_bucket_id_control_min_mismatch.js | 112 +++++++++++++----- 2 files changed, 93 insertions(+), 31 deletions(-) diff --git a/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js b/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js index c0ea76c..c43a92e 100644 --- a/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js +++ b/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js @@ -101,9 +101,17 @@ runFixBucketVersionMismatchProcedure(collName); print('Validating that there are no mismatched bucket versions ...\n'); db.getMongo().setReadPref('secondaryPreferred'); const validateRes = collName.validate({full: true}); -if (validateRes.errors.length != 0) { + +// +// For v8.1.0+, buckets that have a mismatched embedded bucked id timestamp and +// control.min timestamp will lead to an error during validation. +// +// Prior to v8.1.0, buckets that have a mismatched embedded bucked id timestamp +// and control.min timestamp will lead to an warning during validation. +// +if (validateRes.errors.length != 0 || validateRes.warnings.length != 0) { print( - '\nThere is still a time-series bucket with a bucket version mismatch, or there is another error during validation.'); + '\nThere is still a time-series bucket with a bucket version mismatch, or there is another error or warning during validation.'); exit(1); } diff --git a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js index 87a14ed..a774907 100644 --- a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js +++ b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js @@ -1,3 +1,38 @@ +// ------------------------------------------------------------------------------------ +// Populate collName with the time-series collection with a bucket(s) that has +// mismatched embedded bucked id timestamp and control.min timestamp. +// ------------------------------------------------------------------------------------ +const collName = 'your_collection_name'; + +let listCollectionsRes = db.runCommand({ + listCollections: 1.0, + filter: {name: collName} + }).cursor.firstBatch; +if (listCollectionsRes.length == 0) { + print( + 'Collection not found. Populate collName with the time-series collection with a bucket(s) that has mismatched embedded bucked id timestamp and control.min timestamp.'); + exit(1); +} +const coll = db.getCollection(collName); +const bucketsColl = db.getCollection('system.buckets.' + collName); + +// +// NON-MODIFIABLE CODE BELOW +// +// ------------------------------------------------------------------------------------ +// The "temp" collection should not exist prior to running the script. This will +// be used for storing the measurements of the buckets with mismatched embedded +// bucked id timestamp and control.min timestamp. +// ------------------------------------------------------------------------------------ +listCollectionsRes = db.runCommand({ + listCollections: 1.0, + filter: {name: 'temp'} + }).cursor.firstBatch; +if (listCollectionsRes.length != 0) { + print( + 'Collection `temp` should not exist prior to running the script. Rename or drop the collection before running this script'); + exit(1); +} // --------------------------------------------------------------------------------------- // The script will, for each bucket in the affected time-series collection: @@ -8,11 +43,10 @@ // a) Unpack the measurements // b) Insert the measurements back into the collection. These will go into // new buckets. -// c) Delete the mixed-schema bucket from the collection. +// c) Delete the mismatched embedded bucked id timestamp and +// control.min timestamp bucket from the collection. // 3) Validate that there are no buckets with a mismatch between the embedded // bucket id timestamp and the control min timestamp. -// 4) Tell the server there aren't buckets with a mismatch between the embedded -// bucket id timestamp and the control min timestamp. // ---------------------------------------------------------------------------------------- let bucketColl; let tsOptions; @@ -22,8 +56,8 @@ let tempTimeseriesBucketsColl; function setUp() { bucketColl = db.getCollection('system.buckets.' + collName); - // Create a temp collection to store measurements from the mixed-schema - // buckets. + // Create a temp collection to store measurements from the buckets with + // mismatched embedded bucked id timestamp and control.min timestamp. tsOptions = db.runCommand({listCollections: 1.0, filter: {name: coll.getName()}}) .cursor.firstBatch[0] @@ -38,7 +72,7 @@ function setUp() { function runFixEmbeddedBucketIdControlMinMismatchProcedure() { setUp(); print( - 'Finding when embedded bucket ID timestamps don\'t match the control min timestamps in ' + + 'Finding when embedded bucket ID timestamps don\'t match the control min timestamps in ' + collName + ' ...\n'); let cursor = bucketsColl.find({}, {_id: true, control: true}); @@ -48,33 +82,15 @@ function runFixEmbeddedBucketIdControlMinMismatchProcedure() { // types match. If they do not match, re-insert the bucket. while (cursor.hasNext()) { const bucket = cursor.next(); - const bucketId = bucket._id; - const controlMinTimestamp = bucket.control.min + const oidTimestamp = bucket._id.getTimestamp(); + const controlMinTimestamp = bucket.control.min.t - if (bucketHasMismatchedEmbeddedBucketIdAndControlMin( - bucketId, controlMinTimestamp)) { + if (oidTimestamp != controlMinTimestamp) { reinsertMeasurementsFromBucket(bucket._id); } } } -function getDateFromObjectId(objectId) { - return new Date(parseInt(objectId.substring(0, 8), 16) * 1000); -} - -// -// Helpers to detect whether a given bucket contains a mismatch between the -// embedded bucket id timestamp and the control min timestamp. -// -// We parse each timestamp to a date and use .getTime() to compare them. -// -function bucketHasMismatchedEmbeddedBucketIdAndControlMin( - bucketId, controlMinTime) { - const oidTimestamp = getDateFromObjectId(bucketId) - const controlMinTimestamp = new Date(Date.parse(controlMinTime)); - return oidTimestamp.getTime() == controlMinTimestamp.getTime() -} - // // Helpers to perform the re-insertion procedure. // @@ -84,8 +100,10 @@ function reinsertMeasurementsFromBucket(bucketId) { // Prevent concurrent changes on this bucket by setting control.closed. bucketColl.updateOne({_id: bucketId}, {$set: {'control.closed': true}}); - // Get the measurements from the mixed-schema bucket. - print('Getting the measurements from the mixed-schema bucket...\n'); + // Get the measurements from the bucket that has a mismatched embedded bucket + // id timestamp and control.min timestamp. + print( + 'Getting the measurements from the bucket that has a mismatched embedded bucket id timestamp and control.min timestamp...\n'); let measurements; if (tsOptions.metaField) { measurements = bucketColl @@ -164,3 +182,39 @@ function shouldRetryTxnOnTransientError(e) { } return false; } + +// +// Steps 1 & 2: Detect if a bucket has mismatched embedded bucked id timestamps +// and control.min timestamps in the collection and re-inserts buckets with +// these mismatches. +// +print( + 'Re-inserting buckets that have a mismatched embedded bucked id timestamps and control.min timestamps in the collection ...\n'); +runFixEmbeddedBucketIdControlMinMismatchProcedure(); +tempTimeseriesBucketsColl.drop(); + +// +// Step 3: Validate that there are no buckets with mismatched embedded bucked id +// timestamps and control.min timestamps in the collection. +// +print( + 'Validating that there are no buckets that have a mismatched embedded bucked id timestamp and control.min timestamp ...\n'); +db.getMongo().setReadPref('secondaryPreferred'); +const validateRes = coll.validate(); + +// +// For v8.1.0+, buckets that have a mismatched embedded bucked id timestamp and +// control.min timestamp will lead to an error during validation. +// +// Prior to v8.1.0, buckets that have a mismatched embedded bucked id timestamp +// and control.min timestamp will lead to an warning during validation. +// +if (validateRes.errors.length != 0 || validateRes.warnings.length != 0) { + print( + '\nThere is still a bucket that has a mismatched embedded bucked id timestamps and control.min timestamps, or there is another error during validation.'); + exit(1); +} + +print( + '\nScript successfully fixed have a mismatched embedded bucked id timestamp and control.min timestamp!'); +exit(0); From be75f5e8cb67990d1316e3697796b1586e4ed37b Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Wed, 8 Jan 2025 22:21:56 +0000 Subject: [PATCH 03/22] Fixed README --- bucket-version-mismatch/README.md | 20 ++++- embedded-timestamp-mismatch/README.md | 109 ++++++++++++++++++++++---- 2 files changed, 113 insertions(+), 16 deletions(-) diff --git a/bucket-version-mismatch/README.md b/bucket-version-mismatch/README.md index 83a8591..22f559b 100644 --- a/bucket-version-mismatch/README.md +++ b/bucket-version-mismatch/README.md @@ -1,6 +1,9 @@ # Troubleshoot Bucket Version Mismatch in Time Series Collections + For more context on this issue, see [SERVER-94471](https://jira.mongodb.org/browse/SERVER-94471). + ## Warning + The scripts provided here are best run by those experienced with MongoDB, ideally with guidance from MongoDB Technical Support. They are provided without warranty or guarantee of any kind (see disclaimer below). If you do not have access to MongoDB Technical Support, consider reaching out to our community at [community.mongodb.com](community.mongodb.com) to ask questions. Some adaptation of these scripts may be required for your use-case. If you are using these scripts on your own, we strongly recommend: @@ -9,12 +12,16 @@ If you are using these scripts on your own, we strongly recommend: * reviewing the scripts themselves to understand their behavior. * testing this process and all scripts on a copy of the environment it is to be run. * for sharded clusters, disabling the balancer. + # Prerequisites + - This script should be run with a user that has [dbAdmin](https://www.mongodb.com/docs/v6.0/reference/built-in-roles/#mongodb-authrole-dbAdmin) permissions on the database(s) for the affected time-series collection(s). - If running on Atlas: - We have already reached out to impacted customers. If we have reached out, you can skip the [Determine if You're Impacted section](#Determine-if-You're-Impacted) and start taking the steps under the [Remediation section](#remediation). - Additionally, we recommend using the [Atlas Admin](https://www.mongodb.com/docs/atlas/security-add-mongodb-users/#built-in-roles) role. + # Determine if You're Impacted + Please see [SERVER-94471](https://jira.mongodb.org/browse/SERVER-94471) for the affected versions. Users can determine if they have been impacted by running [`validate`](https://www.mongodb.com/docs/v7.0/reference/command/validate/) on their Time Series collections and checking the `validate.errors` field to determine if there are buckets with mismatched versions. The validation command [can be very impactful](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance). To minimize the performance impact of running validate, issue validate to a secondary and follow [these steps](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance:~:text=Validation%20has%20exclusive,the%20hidden%20node). Example `validate` run on a standalone/replica set: @@ -80,8 +87,11 @@ or ``` ..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Time-series bucket is v3 but has its measurements in-order on time"}}... ``` + # Remediation + ## Rewrite Bucket Version in a Time Series Collections + While the script is running, the performance of operations on the time-series collection may be impacted. The script does a scan of the whole collection and performs updates on impacted buckets, which may result in a large load if many buckets are affected. At a high level, the script remediates errors during collection validation by updating the bucket's version from v2 to v3 and vice versa if a v2 bucket has unsorted data, or if a v3 bucket has sorted data. The full steps are as follows. For each bucket in the time series collection: @@ -90,20 +100,28 @@ The full steps are as follows. For each bucket in the time series collection: - Validate that there are no bucket version mismatches. **Warning**: This script directly modifies `.system.buckets` collection —the underlying buckets of the Time Series collection—in order to remediate performance issues. Under normal circumstances, users should not modify this collection. Please contact [MongoDB Support](https://support.mongodb.com/welcome) with any questions or concerns regarding running this script. + ### Running the remediation script + #### 1. Modify the script by populating `collName` with the name of your collection + ``` // ------------------------------------------------------------------------------------ // Populate collName with the time-series collection with mixed-schema buckets. // ------------------------------------------------------------------------------------ const collName = "your_collection_name"; ``` + #### 2. Connect to your sharded cluster using [mongosh](https://www.mongodb.com/docs/mongodb-shell/): + ``` mongosh --uri ``` + #### 3. Load the script `rewrite_timeseries_bucket_version_mismatch.js` + ``` load("rewrite_timeseries_bucket_version_mismatch.js") ``` -#### 4. Repeat steps 1-3 for each time-series collection that was affected. \ No newline at end of file + +#### 4. Repeat steps 1-3 for each time-series collection that was affected. diff --git a/embedded-timestamp-mismatch/README.md b/embedded-timestamp-mismatch/README.md index 4d90160..bf1a1c9 100644 --- a/embedded-timestamp-mismatch/README.md +++ b/embedded-timestamp-mismatch/README.md @@ -1,6 +1,8 @@ # Troubleshoot timestamp embedded in the bucket's ID does not match the control.min timestamp -For more context on this issue, see [SERVER-94471](https://jira.mongodb.org/browse/SERVER-94471). +For more context on this issue, see [SERVER-94559](https://jira.mongodb.org/browse/SERVER-94559). + ## Warning + The scripts provided here are best run by those experienced with MongoDB, ideally with guidance from MongoDB Technical Support. They are provided without warranty or guarantee of any kind (see disclaimer below). If you do not have access to MongoDB Technical Support, consider reaching out to our community at [community.mongodb.com](community.mongodb.com) to ask questions. Some adaptation of these scripts may be required for your use-case. If you are using these scripts on your own, we strongly recommend: @@ -9,19 +11,27 @@ If you are using these scripts on your own, we strongly recommend: * reviewing the scripts themselves to understand their behavior. * testing this process and all scripts on a copy of the environment it is to be run. * for sharded clusters, disabling the balancer. + # Prerequisites + - This script should be run with a user that has [dbAdmin](https://www.mongodb.com/docs/v6.0/reference/built-in-roles/#mongodb-authrole-dbAdmin) permissions on the database(s) for the affected time-series collection(s). - If running on Atlas: - We have already reached out to impacted customers. If we have reached out, you can skip the [Determine if You're Impacted section](#Determine-if-You're-Impacted) and start taking the steps under the [Remediation section](#remediation). - Additionally, we recommend using the [Atlas Admin](https://www.mongodb.com/docs/atlas/security-add-mongodb-users/#built-in-roles) role. + # Determine if You're Impacted -Please see [SERVER-94471](https://jira.mongodb.org/browse/SERVER-94471) for the affected versions. Users can determine if they have been impacted by running [`validate`](https://www.mongodb.com/docs/v7.0/reference/command/validate/) on their Time Series collections and checking the `validate.errors` field to determine if there are buckets with mismatched versions. + +Please see [SERVER-94559](https://jira.mongodb.org/browse/SERVER-94559) for the affected versions. Users can determine if they have been impacted by running [`validate`](https://www.mongodb.com/docs/v7.0/reference/command/validate/) on their Time Series collections and checking the `validate.errors` field (for v8.1+) or the `validate.warnings` field otherwise to determine if there are buckets with mismatched versions. The validation command [can be very impactful](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance). To minimize the performance impact of running validate, issue validate to a secondary and follow [these steps](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance:~:text=Validation%20has%20exclusive,the%20hidden%20node). + +## Validation Result for v8.1+ + Example `validate` run on a standalone/replica set: ``` // Call validate on a mongod process for replica sets. coll.validate(); -// The warnings field detects mixed-schema buckets. +// For v8.1+, errors field detects a bucket(s) that has mismatched embedded bucked id +// timestamp and control.min timestamp. { "ns" : "db.system.buckets.coll", ... @@ -35,17 +45,16 @@ coll.validate(); ``` with the logs: ``` -..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Time-series bucket [...] field is not in ascending order"}}... -``` -or -``` -..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Time-series bucket is v3 but has its measurements in-order on time"}}... +..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Mismatch between the embedded timestamp [...] in the time-series " + "bucket '_id' field and the timestamp [...] in 'control.min' field."}}... ``` + Example `validate` run on a sharded cluster: ``` // Call validate on mongos for sharded clusters. coll.validate(); -// The warnings field detects mixed-schema buckets. +// For v8.1+, errors field detects a bucket(s) that has mismatched embedded bucked id +// timestamp and control.min timestamp. // For sharded clusters, this output is an object with a result for every shard in // the "raw" field. { @@ -74,14 +83,76 @@ coll.validate(); ``` with the logs: ``` -..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Time-series bucket [...] field is not in ascending order"}}... +..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Mismatch between the embedded timestamp [...] in the time-series " + "bucket '_id' field and the timestamp [...] in 'control.min' field."}}... +``` + +## Validation Result Otherwise + +Example `validate` run on a standalone/replica set: +``` +// Call validate on a mongod process for replica sets. +// For versions below v8.1, the warnings field detects a bucket(s) that has mismatched +// embedded bucked id timestamp and control.min timestamp. +coll.validate(); +{ +"ns" : "db.system.buckets.coll", +... +"errors" : [ + Detected one or more documents in this collection incompatible with time-series + specifications. For more info, see logs with log id 6698300., + ... +], +... +} +``` +with the logs: +``` +..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Mismatch between the embedded timestamp [...] in the time-series " + "bucket '_id' field and the timestamp [...] in 'control.min' field."}}... +``` + +Example `validate` run on a sharded cluster: ``` -or +// Call validate on mongos for sharded clusters. +coll.validate(); +// The warnings field detects mixed-schema buckets. +// For sharded clusters, this output is an object with a result for every shard in +// the "raw" field. +{ + "ns" : "db.system.buckets.coll", + ... +"warnings" : [ + Detected one or more documents in this collection incompatible with time-series + specifications. For more info, see logs with log id 6698300., + ... +], +... +"raw" : { + "shard-0-name" : { + "ns" : "db.system.buckets.coll" + ... +"warnings" : [ + Detected one or more documents in this collection incompatible with time-series + specifications. For more info, see logs with log id 6698300., + ... +], +... +}, +"shard-1-name" : { ... }, ... +} +} +``` +with the logs: ``` -..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Time-series bucket is v3 but has its measurements in-order on time"}}... +..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Mismatch between the embedded timestamp [...] in the time-series " + "bucket '_id' field and the timestamp [...] in 'control.min' field."}}... ``` + # Remediation + ## Rewrite Bucket Version in a Time Series Collections + While the script is running, the performance of operations on the time-series collection may be impacted. The script does a scan of the whole collection and performs updates on impacted buckets, which may result in a large load if many buckets are affected. At a high level, the script remediates errors during collection validation by updating the bucket's version from v2 to v3 and vice versa if a v2 bucket has unsorted data, or if a v3 bucket has sorted data. The full steps are as follows. For each bucket in the time series collection: @@ -90,20 +161,28 @@ The full steps are as follows. For each bucket in the time series collection: - Validate that there are no bucket version mismatches. **Warning**: This script directly modifies `.system.buckets` collection —the underlying buckets of the Time Series collection—in order to remediate performance issues. Under normal circumstances, users should not modify this collection. Please contact [MongoDB Support](https://support.mongodb.com/welcome) with any questions or concerns regarding running this script. + ### Running the remediation script + #### 1. Modify the script by populating `collName` with the name of your collection + ``` // ------------------------------------------------------------------------------------ // Populate collName with the time-series collection with mixed-schema buckets. // ------------------------------------------------------------------------------------ const collName = "your_collection_name"; ``` + #### 2. Connect to your sharded cluster using [mongosh](https://www.mongodb.com/docs/mongodb-shell/): + ``` mongosh --uri ``` -#### 3. Load the script `rewrite_timeseries_bucket_version_mismatch.js` + +#### 3. Load the script `rewrite_embedded_bucket_id_control_min_mismatch.js` + ``` -load("rewrite_timeseries_bucket_version_mismatch.js") +load("rewrite_embedded_bucket_id_control_min_mismatch.js") ``` -#### 4. Repeat steps 1-3 for each time-series collection that was affected. \ No newline at end of file + +#### 4. Repeat steps 1-3 for each time-series collection that was affected. From e05f2e506162b972c1670405ce4b8b2059d09cc7 Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Wed, 8 Jan 2025 22:25:32 +0000 Subject: [PATCH 04/22] Fixed message in README --- bucket-version-mismatch/README.md | 5 +++-- embedded-timestamp-mismatch/README.md | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bucket-version-mismatch/README.md b/bucket-version-mismatch/README.md index 22f559b..056427a 100644 --- a/bucket-version-mismatch/README.md +++ b/bucket-version-mismatch/README.md @@ -107,9 +107,10 @@ Please contact [MongoDB Support](https://support.mongodb.com/welcome) with any q ``` // ------------------------------------------------------------------------------------ -// Populate collName with the time-series collection with mixed-schema buckets. +// Populate collName with the time-series collection that failed validation due +// to v2/v3 timeseries buckets not in correct sorted/unsorted order +// respectively. // ------------------------------------------------------------------------------------ -const collName = "your_collection_name"; ``` #### 2. Connect to your sharded cluster using [mongosh](https://www.mongodb.com/docs/mongodb-shell/): diff --git a/embedded-timestamp-mismatch/README.md b/embedded-timestamp-mismatch/README.md index bf1a1c9..d214e77 100644 --- a/embedded-timestamp-mismatch/README.md +++ b/embedded-timestamp-mismatch/README.md @@ -1,4 +1,4 @@ -# Troubleshoot timestamp embedded in the bucket's ID does not match the control.min timestamp +# Troubleshoot Timestamp Embedded in the Bucket's ID does not match the control.min Timestamp For more context on this issue, see [SERVER-94559](https://jira.mongodb.org/browse/SERVER-94559). ## Warning @@ -168,7 +168,8 @@ Please contact [MongoDB Support](https://support.mongodb.com/welcome) with any q ``` // ------------------------------------------------------------------------------------ -// Populate collName with the time-series collection with mixed-schema buckets. +// Populate collName with the time-series collection with a bucket(s) that has +// mismatched embedded bucked id timestamp and control.min timestamp. // ------------------------------------------------------------------------------------ const collName = "your_collection_name"; ``` From a6f803fd9f0b2d30fdf0932dd474f0759e0275a4 Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Wed, 8 Jan 2025 22:32:49 +0000 Subject: [PATCH 05/22] more fixing up --- bucket-version-mismatch/README.md | 73 ++++++++++++++++++++++++++- embedded-timestamp-mismatch/README.md | 4 +- 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/bucket-version-mismatch/README.md b/bucket-version-mismatch/README.md index 056427a..c638d37 100644 --- a/bucket-version-mismatch/README.md +++ b/bucket-version-mismatch/README.md @@ -24,11 +24,80 @@ If you are using these scripts on your own, we strongly recommend: Please see [SERVER-94471](https://jira.mongodb.org/browse/SERVER-94471) for the affected versions. Users can determine if they have been impacted by running [`validate`](https://www.mongodb.com/docs/v7.0/reference/command/validate/) on their Time Series collections and checking the `validate.errors` field to determine if there are buckets with mismatched versions. The validation command [can be very impactful](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance). To minimize the performance impact of running validate, issue validate to a secondary and follow [these steps](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance:~:text=Validation%20has%20exclusive,the%20hidden%20node). + +## Validation Results for v8.1+ + +Example `validate` run on a standalone/replica set: +``` +// Call validate on a mongod process for replica sets. +coll.validate(); +// The errors field detects bucket version mismatch. +{ +"ns" : "db.system.buckets.coll", +... +"errors" : [ + Detected one or more documents in this collection incompatible with time-series + specifications. For more info, see logs with log id 6698300., + ... +], +... +} +``` +with the logs: +``` +..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Time-series bucket [...] field is not in ascending order"}}... +``` +or +``` +..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Time-series bucket is v3 but has its measurements in-order on time"}}... +``` +Example `validate` run on a sharded cluster: +``` +// Call validate on mongos for sharded clusters. +coll.validate(); +// The errors field detects bucket version mismatch. +// For sharded clusters, this output is an object with a result for every shard in +// the "raw" field. +{ + "ns" : "db.system.buckets.coll", + ... +"errors" : [ + Detected one or more documents in this collection incompatible with time-series + specifications. For more info, see logs with log id 6698300., + ... +], +... +"raw" : { + "shard-0-name" : { + "ns" : "db.system.buckets.coll" + ... +"errors" : [ + Detected one or more documents in this collection incompatible with time-series + specifications. For more info, see logs with log id 6698300., + ... +], +... +}, +"shard-1-name" : { ... }, ... +} +} +``` +with the logs: +``` +..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Time-series bucket [...] field is not in ascending order"}}... +``` +or +``` +..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Time-series bucket is v3 but has its measurements in-order on time"}}... +``` + +## Validation Results Before v8.1 + Example `validate` run on a standalone/replica set: ``` // Call validate on a mongod process for replica sets. coll.validate(); -// The warnings field detects mixed-schema buckets. +// The warnings field detects bucket version mismatch. { "ns" : "db.system.buckets.coll", ... @@ -52,7 +121,7 @@ Example `validate` run on a sharded cluster: ``` // Call validate on mongos for sharded clusters. coll.validate(); -// The warnings field detects mixed-schema buckets. +// The warnings field detects bucket version mismatch. // For sharded clusters, this output is an object with a result for every shard in // the "raw" field. { diff --git a/embedded-timestamp-mismatch/README.md b/embedded-timestamp-mismatch/README.md index d214e77..828be42 100644 --- a/embedded-timestamp-mismatch/README.md +++ b/embedded-timestamp-mismatch/README.md @@ -24,7 +24,7 @@ If you are using these scripts on your own, we strongly recommend: Please see [SERVER-94559](https://jira.mongodb.org/browse/SERVER-94559) for the affected versions. Users can determine if they have been impacted by running [`validate`](https://www.mongodb.com/docs/v7.0/reference/command/validate/) on their Time Series collections and checking the `validate.errors` field (for v8.1+) or the `validate.warnings` field otherwise to determine if there are buckets with mismatched versions. The validation command [can be very impactful](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance). To minimize the performance impact of running validate, issue validate to a secondary and follow [these steps](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance:~:text=Validation%20has%20exclusive,the%20hidden%20node). -## Validation Result for v8.1+ +## Validation Results for v8.1+ Example `validate` run on a standalone/replica set: ``` @@ -87,7 +87,7 @@ with the logs: "bucket '_id' field and the timestamp [...] in 'control.min' field."}}... ``` -## Validation Result Otherwise +## Validation Results Before v8.1 Example `validate` run on a standalone/replica set: ``` From 8ee4c39d672acbb0aaaed0e81af66763bc06ea0c Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Wed, 8 Jan 2025 22:36:51 +0000 Subject: [PATCH 06/22] Cleaning up comments --- bucket-version-mismatch/README.md | 8 ++++---- embedded-timestamp-mismatch/README.md | 9 +++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/bucket-version-mismatch/README.md b/bucket-version-mismatch/README.md index c638d37..60db884 100644 --- a/bucket-version-mismatch/README.md +++ b/bucket-version-mismatch/README.md @@ -31,7 +31,7 @@ Example `validate` run on a standalone/replica set: ``` // Call validate on a mongod process for replica sets. coll.validate(); -// The errors field detects bucket version mismatch. +// For v8.1+, the errors field detects bucket version mismatch. { "ns" : "db.system.buckets.coll", ... @@ -55,7 +55,7 @@ Example `validate` run on a sharded cluster: ``` // Call validate on mongos for sharded clusters. coll.validate(); -// The errors field detects bucket version mismatch. +// For v8.1+, the errors field detects bucket version mismatch. // For sharded clusters, this output is an object with a result for every shard in // the "raw" field. { @@ -97,7 +97,7 @@ Example `validate` run on a standalone/replica set: ``` // Call validate on a mongod process for replica sets. coll.validate(); -// The warnings field detects bucket version mismatch. +// For versions below v8.1, the warnings field detects bucket version mismatch. { "ns" : "db.system.buckets.coll", ... @@ -121,7 +121,7 @@ Example `validate` run on a sharded cluster: ``` // Call validate on mongos for sharded clusters. coll.validate(); -// The warnings field detects bucket version mismatch. +// For versions below v8.1, the warnings field detects bucket version mismatch. // For sharded clusters, this output is an object with a result for every shard in // the "raw" field. { diff --git a/embedded-timestamp-mismatch/README.md b/embedded-timestamp-mismatch/README.md index 828be42..730d58d 100644 --- a/embedded-timestamp-mismatch/README.md +++ b/embedded-timestamp-mismatch/README.md @@ -1,4 +1,4 @@ -# Troubleshoot Timestamp Embedded in the Bucket's ID does not match the control.min Timestamp +# Troubleshoot Timestamp Embedded in the Bucket's ID Does Not Match the control.min Timestamp For more context on this issue, see [SERVER-94559](https://jira.mongodb.org/browse/SERVER-94559). ## Warning @@ -30,7 +30,7 @@ Example `validate` run on a standalone/replica set: ``` // Call validate on a mongod process for replica sets. coll.validate(); -// For v8.1+, errors field detects a bucket(s) that has mismatched embedded bucked id +// For v8.1+, the errors field detects a bucket(s) that has mismatched embedded bucked id // timestamp and control.min timestamp. { "ns" : "db.system.buckets.coll", @@ -53,7 +53,7 @@ Example `validate` run on a sharded cluster: ``` // Call validate on mongos for sharded clusters. coll.validate(); -// For v8.1+, errors field detects a bucket(s) that has mismatched embedded bucked id +// For v8.1+, the errors field detects a bucket(s) that has mismatched embedded bucked id // timestamp and control.min timestamp. // For sharded clusters, this output is an object with a result for every shard in // the "raw" field. @@ -116,7 +116,8 @@ Example `validate` run on a sharded cluster: ``` // Call validate on mongos for sharded clusters. coll.validate(); -// The warnings field detects mixed-schema buckets. +// For versions below v8.1, the warnings field detects a bucket(s) that has mismatched +// embedded bucked id timestamp and control.min timestamp. // For sharded clusters, this output is an object with a result for every shard in // the "raw" field. { From cdac0dc7fb0b6fe12c2c1394a009f8aed791f8bf Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Wed, 8 Jan 2025 22:48:13 +0000 Subject: [PATCH 07/22] bucked --> bucket and fixed Remediation section --- bucket-version-mismatch/README.md | 17 ++++++---- ...rite_timeseries_bucket_version_mismatch.js | 4 +-- embedded-timestamp-mismatch/README.md | 31 ++++++++++++------- ...embedded_bucket_id_control_min_mismatch.js | 26 ++++++++-------- timeseries-mixed-schema/README.md | 1 - 5 files changed, 46 insertions(+), 33 deletions(-) diff --git a/bucket-version-mismatch/README.md b/bucket-version-mismatch/README.md index 60db884..3d1a7d4 100644 --- a/bucket-version-mismatch/README.md +++ b/bucket-version-mismatch/README.md @@ -161,15 +161,20 @@ or ## Rewrite Bucket Version in a Time Series Collections -While the script is running, the performance of operations on the time-series collection may be impacted. The script does a scan of the whole collection and performs updates on impacted buckets, which may result in a large load if many buckets are affected. -At a high level, the script remediates errors during collection validation by updating the bucket's version from v2 to v3 and vice versa if a v2 bucket has unsorted data, or if a v3 bucket has sorted data. +While the script is running, the performance of operations on the time-series collection may be impacted. The script does a scan of the whole collection and performs multiple reads and writes per mixed-schema bucket, which may result in a large load if many buckets are affected. + +At a high level, the script remediates performance by rewriting buckets from the mixed-schema format to the older schema. The rewrite is done by unpacking the measurements of the problematic mixed-schema buckets and inserting those measurements back into the collection. + The full steps are as follows. For each bucket in the time series collection: -- Detect if the bucket has bucket version mismatch. -- Change the buckets with bucket version mismatch to the correct version. -- Validate that there are no bucket version mismatches. +- Detect if the bucket has mixed-schema data. +- Re-insert the measurements of the mixed-schema bucket transactionally. + - Unpack the measurements. + - Insert the measurements back into the collection. These will go into new buckets. + - Delete the mixed-schema bucket from the collection. + **Warning**: This script directly modifies `.system.buckets` collection —the underlying buckets of the Time Series collection—in order to remediate performance issues. Under normal circumstances, users should not modify this collection. -Please contact [MongoDB Support](https://support.mongodb.com/welcome) with any questions or concerns regarding running this script. +Please contact [MongoDB Support](https://support.mongodb.com/welcome) with any questions or concerns regarding running this script. ### Running the remediation script #### 1. Modify the script by populating `collName` with the name of your collection diff --git a/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js b/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js index c43a92e..3b9a673 100644 --- a/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js +++ b/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js @@ -103,10 +103,10 @@ db.getMongo().setReadPref('secondaryPreferred'); const validateRes = collName.validate({full: true}); // -// For v8.1.0+, buckets that have a mismatched embedded bucked id timestamp and +// For v8.1.0+, buckets that have a mismatched embedded bucket id timestamp and // control.min timestamp will lead to an error during validation. // -// Prior to v8.1.0, buckets that have a mismatched embedded bucked id timestamp +// Prior to v8.1.0, buckets that have a mismatched embedded bucket id timestamp // and control.min timestamp will lead to an warning during validation. // if (validateRes.errors.length != 0 || validateRes.warnings.length != 0) { diff --git a/embedded-timestamp-mismatch/README.md b/embedded-timestamp-mismatch/README.md index 730d58d..a640ea9 100644 --- a/embedded-timestamp-mismatch/README.md +++ b/embedded-timestamp-mismatch/README.md @@ -4,8 +4,11 @@ For more context on this issue, see [SERVER-94559](https://jira.mongodb.org/brow ## Warning The scripts provided here are best run by those experienced with MongoDB, ideally with guidance from MongoDB Technical Support. They are provided without warranty or guarantee of any kind (see disclaimer below). If you do not have access to MongoDB Technical Support, consider reaching out to our community at [community.mongodb.com](community.mongodb.com) to ask questions. + Some adaptation of these scripts may be required for your use-case. + If you are using these scripts on your own, we strongly recommend: + * reading all instructions before attempting any action, and ensuring you are prepared for all steps. * taking regular backups of the [dbpaths](https://docs.mongodb.com/manual/core/backups/#back-up-by-copying-underlying-data-files) of each node, and working off of backups as much as possible. * reviewing the scripts themselves to understand their behavior. @@ -13,7 +16,6 @@ If you are using these scripts on your own, we strongly recommend: * for sharded clusters, disabling the balancer. # Prerequisites - - This script should be run with a user that has [dbAdmin](https://www.mongodb.com/docs/v6.0/reference/built-in-roles/#mongodb-authrole-dbAdmin) permissions on the database(s) for the affected time-series collection(s). - If running on Atlas: - We have already reached out to impacted customers. If we have reached out, you can skip the [Determine if You're Impacted section](#Determine-if-You're-Impacted) and start taking the steps under the [Remediation section](#remediation). @@ -22,6 +24,7 @@ If you are using these scripts on your own, we strongly recommend: # Determine if You're Impacted Please see [SERVER-94559](https://jira.mongodb.org/browse/SERVER-94559) for the affected versions. Users can determine if they have been impacted by running [`validate`](https://www.mongodb.com/docs/v7.0/reference/command/validate/) on their Time Series collections and checking the `validate.errors` field (for v8.1+) or the `validate.warnings` field otherwise to determine if there are buckets with mismatched versions. + The validation command [can be very impactful](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance). To minimize the performance impact of running validate, issue validate to a secondary and follow [these steps](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance:~:text=Validation%20has%20exclusive,the%20hidden%20node). ## Validation Results for v8.1+ @@ -30,7 +33,7 @@ Example `validate` run on a standalone/replica set: ``` // Call validate on a mongod process for replica sets. coll.validate(); -// For v8.1+, the errors field detects a bucket(s) that has mismatched embedded bucked id +// For v8.1+, the errors field detects a bucket(s) that has mismatched embedded bucket id // timestamp and control.min timestamp. { "ns" : "db.system.buckets.coll", @@ -53,7 +56,7 @@ Example `validate` run on a sharded cluster: ``` // Call validate on mongos for sharded clusters. coll.validate(); -// For v8.1+, the errors field detects a bucket(s) that has mismatched embedded bucked id +// For v8.1+, the errors field detects a bucket(s) that has mismatched embedded bucket id // timestamp and control.min timestamp. // For sharded clusters, this output is an object with a result for every shard in // the "raw" field. @@ -93,7 +96,7 @@ Example `validate` run on a standalone/replica set: ``` // Call validate on a mongod process for replica sets. // For versions below v8.1, the warnings field detects a bucket(s) that has mismatched -// embedded bucked id timestamp and control.min timestamp. +// embedded bucket id timestamp and control.min timestamp. coll.validate(); { "ns" : "db.system.buckets.coll", @@ -117,7 +120,7 @@ Example `validate` run on a sharded cluster: // Call validate on mongos for sharded clusters. coll.validate(); // For versions below v8.1, the warnings field detects a bucket(s) that has mismatched -// embedded bucked id timestamp and control.min timestamp. +// embedded bucket id timestamp and control.min timestamp. // For sharded clusters, this output is an object with a result for every shard in // the "raw" field. { @@ -152,15 +155,21 @@ with the logs: # Remediation -## Rewrite Bucket Version in a Time Series Collections +## Rewrite the Buckets with Mismatched Embedded Bucket ID Timestamp and control.min Timestamp While the script is running, the performance of operations on the time-series collection may be impacted. The script does a scan of the whole collection and performs updates on impacted buckets, which may result in a large load if many buckets are affected. -At a high level, the script remediates errors during collection validation by updating the bucket's version from v2 to v3 and vice versa if a v2 bucket has unsorted data, or if a v3 bucket has sorted data. + +At a high level, the script remediates errors during collection validation by rewriting buckets that are detected to have a mismatched embedded bucket ID timestamp and control.min timestamp. The rewrite is done by unpacking the measurements of the problematic buckets and inserting those measurements back into the collection. + The full steps are as follows. For each bucket in the time series collection: -- Detect if the bucket has bucket version mismatch. -- Change the buckets with bucket version mismatch to the correct version. -- Validate that there are no bucket version mismatches. +- Detect if the bucket has mismatched embedded bucket ID timestamp and control.min timestamp +- Re-insert the measurements of the problematic bucket transactionally. + - Unpack the measurements. + - Insert the measurements back into the collection. These will go into a new buckets. + - Delete the problematic bucket from the collection. + **Warning**: This script directly modifies `.system.buckets` collection —the underlying buckets of the Time Series collection—in order to remediate performance issues. Under normal circumstances, users should not modify this collection. + Please contact [MongoDB Support](https://support.mongodb.com/welcome) with any questions or concerns regarding running this script. ### Running the remediation script @@ -170,7 +179,7 @@ Please contact [MongoDB Support](https://support.mongodb.com/welcome) with any q ``` // ------------------------------------------------------------------------------------ // Populate collName with the time-series collection with a bucket(s) that has -// mismatched embedded bucked id timestamp and control.min timestamp. +// mismatched embedded bucket id timestamp and control.min timestamp. // ------------------------------------------------------------------------------------ const collName = "your_collection_name"; ``` diff --git a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js index a774907..cea37c2 100644 --- a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js +++ b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js @@ -1,6 +1,6 @@ // ------------------------------------------------------------------------------------ // Populate collName with the time-series collection with a bucket(s) that has -// mismatched embedded bucked id timestamp and control.min timestamp. +// mismatched embedded bucket id timestamp and control.min timestamp. // ------------------------------------------------------------------------------------ const collName = 'your_collection_name'; @@ -10,7 +10,7 @@ let listCollectionsRes = db.runCommand({ }).cursor.firstBatch; if (listCollectionsRes.length == 0) { print( - 'Collection not found. Populate collName with the time-series collection with a bucket(s) that has mismatched embedded bucked id timestamp and control.min timestamp.'); + 'Collection not found. Populate collName with the time-series collection with a bucket(s) that has mismatched embedded bucket id timestamp and control.min timestamp.'); exit(1); } const coll = db.getCollection(collName); @@ -22,7 +22,7 @@ const bucketsColl = db.getCollection('system.buckets.' + collName); // ------------------------------------------------------------------------------------ // The "temp" collection should not exist prior to running the script. This will // be used for storing the measurements of the buckets with mismatched embedded -// bucked id timestamp and control.min timestamp. +// bucket id timestamp and control.min timestamp. // ------------------------------------------------------------------------------------ listCollectionsRes = db.runCommand({ listCollections: 1.0, @@ -43,7 +43,7 @@ if (listCollectionsRes.length != 0) { // a) Unpack the measurements // b) Insert the measurements back into the collection. These will go into // new buckets. -// c) Delete the mismatched embedded bucked id timestamp and +// c) Delete the mismatched embedded bucket id timestamp and // control.min timestamp bucket from the collection. // 3) Validate that there are no buckets with a mismatch between the embedded // bucket id timestamp and the control min timestamp. @@ -57,7 +57,7 @@ function setUp() { bucketColl = db.getCollection('system.buckets.' + collName); // Create a temp collection to store measurements from the buckets with - // mismatched embedded bucked id timestamp and control.min timestamp. + // mismatched embedded bucket id timestamp and control.min timestamp. tsOptions = db.runCommand({listCollections: 1.0, filter: {name: coll.getName()}}) .cursor.firstBatch[0] @@ -184,37 +184,37 @@ function shouldRetryTxnOnTransientError(e) { } // -// Steps 1 & 2: Detect if a bucket has mismatched embedded bucked id timestamps +// Steps 1 & 2: Detect if a bucket has mismatched embedded bucket id timestamps // and control.min timestamps in the collection and re-inserts buckets with // these mismatches. // print( - 'Re-inserting buckets that have a mismatched embedded bucked id timestamps and control.min timestamps in the collection ...\n'); + 'Re-inserting buckets that have a mismatched embedded bucket id timestamps and control.min timestamps in the collection ...\n'); runFixEmbeddedBucketIdControlMinMismatchProcedure(); tempTimeseriesBucketsColl.drop(); // -// Step 3: Validate that there are no buckets with mismatched embedded bucked id +// Step 3: Validate that there are no buckets with mismatched embedded bucket id // timestamps and control.min timestamps in the collection. // print( - 'Validating that there are no buckets that have a mismatched embedded bucked id timestamp and control.min timestamp ...\n'); + 'Validating that there are no buckets that have a mismatched embedded bucket id timestamp and control.min timestamp ...\n'); db.getMongo().setReadPref('secondaryPreferred'); const validateRes = coll.validate(); // -// For v8.1.0+, buckets that have a mismatched embedded bucked id timestamp and +// For v8.1.0+, buckets that have a mismatched embedded bucket id timestamp and // control.min timestamp will lead to an error during validation. // -// Prior to v8.1.0, buckets that have a mismatched embedded bucked id timestamp +// Prior to v8.1.0, buckets that have a mismatched embedded bucket id timestamp // and control.min timestamp will lead to an warning during validation. // if (validateRes.errors.length != 0 || validateRes.warnings.length != 0) { print( - '\nThere is still a bucket that has a mismatched embedded bucked id timestamps and control.min timestamps, or there is another error during validation.'); + '\nThere is still a bucket that has a mismatched embedded bucket id timestamps and control.min timestamps, or there is another error during validation.'); exit(1); } print( - '\nScript successfully fixed have a mismatched embedded bucked id timestamp and control.min timestamp!'); + '\nScript successfully fixed have a mismatched embedded bucket id timestamp and control.min timestamp!'); exit(0); diff --git a/timeseries-mixed-schema/README.md b/timeseries-mixed-schema/README.md index 615ce21..a11096d 100644 --- a/timeseries-mixed-schema/README.md +++ b/timeseries-mixed-schema/README.md @@ -45,7 +45,6 @@ coll.validate(); ``` Example `validate` run on a sharded cluster: - ``` // Call validate on mongos for sharded clusters. coll.validate(); From 9910c827ed46d7420745058b8c86919b2a3239f0 Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Wed, 8 Jan 2025 22:50:33 +0000 Subject: [PATCH 08/22] More fixing --- bucket-version-mismatch/README.md | 12 +++++------- timeseries-mixed-schema/README.md | 1 + 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/bucket-version-mismatch/README.md b/bucket-version-mismatch/README.md index 3d1a7d4..c71d428 100644 --- a/bucket-version-mismatch/README.md +++ b/bucket-version-mismatch/README.md @@ -161,16 +161,14 @@ or ## Rewrite Bucket Version in a Time Series Collections -While the script is running, the performance of operations on the time-series collection may be impacted. The script does a scan of the whole collection and performs multiple reads and writes per mixed-schema bucket, which may result in a large load if many buckets are affected. +While the script is running, the performance of operations on the time-series collection may be impacted. The script does a scan of the whole collection and performs updates on impacted buckets, which may result in a large load if many buckets are affected. -At a high level, the script remediates performance by rewriting buckets from the mixed-schema format to the older schema. The rewrite is done by unpacking the measurements of the problematic mixed-schema buckets and inserting those measurements back into the collection. +At a high level, the script remediates errors during collection validation by updating the bucket's version from v2 to v3 and vice versa if a v2 bucket has unsorted data, or if a v3 bucket has sorted data. The full steps are as follows. For each bucket in the time series collection: -- Detect if the bucket has mixed-schema data. -- Re-insert the measurements of the mixed-schema bucket transactionally. - - Unpack the measurements. - - Insert the measurements back into the collection. These will go into new buckets. - - Delete the mixed-schema bucket from the collection. +- Detect if the bucket has bucket version mismatch. +- Change the buckets with bucket version mismatch to the correct version. +- Validate that there are no bucket version mismatches. **Warning**: This script directly modifies `.system.buckets` collection —the underlying buckets of the Time Series collection—in order to remediate performance issues. Under normal circumstances, users should not modify this collection. diff --git a/timeseries-mixed-schema/README.md b/timeseries-mixed-schema/README.md index a11096d..615ce21 100644 --- a/timeseries-mixed-schema/README.md +++ b/timeseries-mixed-schema/README.md @@ -45,6 +45,7 @@ coll.validate(); ``` Example `validate` run on a sharded cluster: + ``` // Call validate on mongos for sharded clusters. coll.validate(); From e8f31f10c1b47fea16aa42cd9540492d5f5d839a Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Wed, 8 Jan 2025 22:52:23 +0000 Subject: [PATCH 09/22] added otherwise --- bucket-version-mismatch/README.md | 2 +- embedded-timestamp-mismatch/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bucket-version-mismatch/README.md b/bucket-version-mismatch/README.md index c71d428..cc2c899 100644 --- a/bucket-version-mismatch/README.md +++ b/bucket-version-mismatch/README.md @@ -161,7 +161,7 @@ or ## Rewrite Bucket Version in a Time Series Collections -While the script is running, the performance of operations on the time-series collection may be impacted. The script does a scan of the whole collection and performs updates on impacted buckets, which may result in a large load if many buckets are affected. +While the script is running, the performance of operations on the time-series collection may be impacted. The script does a scan of the whole collection and performs updates on impacted buckets, which may result in a large load if many buckets are affected. At a high level, the script remediates errors during collection validation by updating the bucket's version from v2 to v3 and vice versa if a v2 bucket has unsorted data, or if a v3 bucket has sorted data. diff --git a/embedded-timestamp-mismatch/README.md b/embedded-timestamp-mismatch/README.md index a640ea9..72035b3 100644 --- a/embedded-timestamp-mismatch/README.md +++ b/embedded-timestamp-mismatch/README.md @@ -23,7 +23,7 @@ If you are using these scripts on your own, we strongly recommend: # Determine if You're Impacted -Please see [SERVER-94559](https://jira.mongodb.org/browse/SERVER-94559) for the affected versions. Users can determine if they have been impacted by running [`validate`](https://www.mongodb.com/docs/v7.0/reference/command/validate/) on their Time Series collections and checking the `validate.errors` field (for v8.1+) or the `validate.warnings` field otherwise to determine if there are buckets with mismatched versions. +Please see [SERVER-94559](https://jira.mongodb.org/browse/SERVER-94559) for the affected versions. Users can determine if they have been impacted by running [`validate`](https://www.mongodb.com/docs/v7.0/reference/command/validate/) on their Time Series collections and checking the `validate.errors` field (for v8.1+) or the `validate.warnings` field (for below v8.1) otherwise to determine if there are buckets with mismatched versions. The validation command [can be very impactful](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance). To minimize the performance impact of running validate, issue validate to a secondary and follow [these steps](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance:~:text=Validation%20has%20exclusive,the%20hidden%20node). From 5951392b48fb31b5af3410c174c7854f3bdf99ac Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Wed, 8 Jan 2025 23:08:17 +0000 Subject: [PATCH 10/22] Fixing up comments --- .../rewrite_timeseries_bucket_version_mismatch.js | 8 ++++---- .../rewrite_embedded_bucket_id_control_min_mismatch.js | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js b/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js index 3b9a673..de254a8 100644 --- a/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js +++ b/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js @@ -103,11 +103,11 @@ db.getMongo().setReadPref('secondaryPreferred'); const validateRes = collName.validate({full: true}); // -// For v8.1.0+, buckets that have a mismatched embedded bucket id timestamp and -// control.min timestamp will lead to an error during validation. +// For v8.1+, buckets that have a bucket version mismatch will lead to a error +// during validation. // -// Prior to v8.1.0, buckets that have a mismatched embedded bucket id timestamp -// and control.min timestamp will lead to an warning during validation. +// Prior to v8.1, buckets that have a bucket version mismatch will lead to a +// warning during validation. // if (validateRes.errors.length != 0 || validateRes.warnings.length != 0) { print( diff --git a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js index cea37c2..3cf552b 100644 --- a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js +++ b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js @@ -203,11 +203,11 @@ db.getMongo().setReadPref('secondaryPreferred'); const validateRes = coll.validate(); // -// For v8.1.0+, buckets that have a mismatched embedded bucket id timestamp and -// control.min timestamp will lead to an error during validation. +// For v8.1+, buckets that have a mismatched embedded bucket id timestamp and +// control.min timestamp will lead to a error during validation. // -// Prior to v8.1.0, buckets that have a mismatched embedded bucket id timestamp -// and control.min timestamp will lead to an warning during validation. +// Prior to v8.1, buckets that have a mismatched embedded bucket id timestamp +// and control.min timestamp will lead to a warning during validation. // if (validateRes.errors.length != 0 || validateRes.warnings.length != 0) { print( From 081d95dbe53968d99b30e0316e0400c4fe684528 Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Wed, 8 Jan 2025 23:09:37 +0000 Subject: [PATCH 11/22] More fixes --- .../rewrite_embedded_bucket_id_control_min_mismatch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js index 3cf552b..ecc4f54 100644 --- a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js +++ b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js @@ -211,7 +211,7 @@ const validateRes = coll.validate(); // if (validateRes.errors.length != 0 || validateRes.warnings.length != 0) { print( - '\nThere is still a bucket that has a mismatched embedded bucket id timestamps and control.min timestamps, or there is another error during validation.'); + '\nThere is still a bucket that has a mismatched embedded bucket id timestamps and control.min timestamps, or there is another error or warning during validation.'); exit(1); } From 307c5a69ab0e2ea123071fedd965f1e74e089854 Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Wed, 8 Jan 2025 23:20:51 +0000 Subject: [PATCH 12/22] Fixed something --- bucket-version-mismatch/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bucket-version-mismatch/README.md b/bucket-version-mismatch/README.md index cc2c899..616ff5d 100644 --- a/bucket-version-mismatch/README.md +++ b/bucket-version-mismatch/README.md @@ -161,7 +161,7 @@ or ## Rewrite Bucket Version in a Time Series Collections -While the script is running, the performance of operations on the time-series collection may be impacted. The script does a scan of the whole collection and performs updates on impacted buckets, which may result in a large load if many buckets are affected. +While the script is running, the performance of operations on the time-series collection may be impacted. The script does a scan of the whole collection and performs updates on impacted buckets, which may result in a large load if many buckets are affected. At a high level, the script remediates errors during collection validation by updating the bucket's version from v2 to v3 and vice versa if a v2 bucket has unsorted data, or if a v3 bucket has sorted data. From 78c5cbb3c42e09531fe34b025b89a2e97f976bb8 Mon Sep 17 00:00:00 2001 From: Stephanie <53684987+seristof@users.noreply.github.com> Date: Thu, 9 Jan 2025 11:34:49 -0500 Subject: [PATCH 13/22] Update embedded-timestamp-mismatch/README.md Co-authored-by: Dan Larkin-York <13419935+dhly-etc@users.noreply.github.com> --- embedded-timestamp-mismatch/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/embedded-timestamp-mismatch/README.md b/embedded-timestamp-mismatch/README.md index 72035b3..931c80d 100644 --- a/embedded-timestamp-mismatch/README.md +++ b/embedded-timestamp-mismatch/README.md @@ -165,8 +165,8 @@ The full steps are as follows. For each bucket in the time series collection: - Detect if the bucket has mismatched embedded bucket ID timestamp and control.min timestamp - Re-insert the measurements of the problematic bucket transactionally. - Unpack the measurements. - - Insert the measurements back into the collection. These will go into a new buckets. - - Delete the problematic bucket from the collection. + - Repack the measurements into new buckets. + - Delete the original, problematic bucket from the collection. **Warning**: This script directly modifies `.system.buckets` collection —the underlying buckets of the Time Series collection—in order to remediate performance issues. Under normal circumstances, users should not modify this collection. From 37cf100b5477157ec20f27178297b2e0938715db Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Thu, 9 Jan 2025 16:37:41 +0000 Subject: [PATCH 14/22] fixed up dan's comments --- .../rewrite_embedded_bucket_id_control_min_mismatch.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js index ecc4f54..3d566a5 100644 --- a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js +++ b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js @@ -41,10 +41,8 @@ if (listCollectionsRes.length != 0) { // 2) Re-insert the measurements of the timestamp-mismatched bucket // transactionally. // a) Unpack the measurements -// b) Insert the measurements back into the collection. These will go into -// new buckets. -// c) Delete the mismatched embedded bucket id timestamp and -// control.min timestamp bucket from the collection. +// b) Repack the measurements into new buckets. +// c) Delete the original, problematic bucket from the collection. // 3) Validate that there are no buckets with a mismatch between the embedded // bucket id timestamp and the control min timestamp. // ---------------------------------------------------------------------------------------- @@ -211,10 +209,10 @@ const validateRes = coll.validate(); // if (validateRes.errors.length != 0 || validateRes.warnings.length != 0) { print( - '\nThere is still a bucket that has a mismatched embedded bucket id timestamps and control.min timestamps, or there is another error or warning during validation.'); + '\nThere is still a bucket(s) that has a mismatched embedded bucket id timestamps and control.min timestamps, or there is another error or warning during validation.'); exit(1); } print( - '\nScript successfully fixed have a mismatched embedded bucket id timestamp and control.min timestamp!'); + '\nScript successfully fixed buckets with mismatched embedded bucket id timestamp and control.min timestamp!'); exit(0); From d64fbd45260438152bd53bc37ad966c717050e34 Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Thu, 9 Jan 2025 16:41:17 +0000 Subject: [PATCH 15/22] Removed two logging lines --- .../rewrite_embedded_bucket_id_control_min_mismatch.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js index 3d566a5..9bf9d20 100644 --- a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js +++ b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js @@ -69,9 +69,6 @@ function setUp() { // Main function. function runFixEmbeddedBucketIdControlMinMismatchProcedure() { setUp(); - print( - 'Finding when embedded bucket ID timestamps don\'t match the control min timestamps in ' + - collName + ' ...\n'); let cursor = bucketsColl.find({}, {_id: true, control: true}); // Mismatched timestamp buckets will have different types for their @@ -100,8 +97,6 @@ function reinsertMeasurementsFromBucket(bucketId) { // Get the measurements from the bucket that has a mismatched embedded bucket // id timestamp and control.min timestamp. - print( - 'Getting the measurements from the bucket that has a mismatched embedded bucket id timestamp and control.min timestamp...\n'); let measurements; if (tsOptions.metaField) { measurements = bucketColl From 5d722a6be1a85a7fb18b632f694b6566986c6950 Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Thu, 9 Jan 2025 17:34:59 +0000 Subject: [PATCH 16/22] Made background: true for the validation --- .../rewrite_embedded_bucket_id_control_min_mismatch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js index 9bf9d20..f92da01 100644 --- a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js +++ b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js @@ -193,7 +193,7 @@ tempTimeseriesBucketsColl.drop(); print( 'Validating that there are no buckets that have a mismatched embedded bucket id timestamp and control.min timestamp ...\n'); db.getMongo().setReadPref('secondaryPreferred'); -const validateRes = coll.validate(); +const validateRes = coll.validate({background: true}); // // For v8.1+, buckets that have a mismatched embedded bucket id timestamp and From 9efc6bcb099f9af6710426e6b42f925bdee67c5f Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Thu, 9 Jan 2025 17:58:44 +0000 Subject: [PATCH 17/22] changed background true for bucket version mismatch as well --- .../rewrite_timeseries_bucket_version_mismatch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js b/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js index de254a8..12eaab7 100644 --- a/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js +++ b/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js @@ -100,7 +100,7 @@ runFixBucketVersionMismatchProcedure(collName); // print('Validating that there are no mismatched bucket versions ...\n'); db.getMongo().setReadPref('secondaryPreferred'); -const validateRes = collName.validate({full: true}); +const validateRes = collName.validate({background: true}); // // For v8.1+, buckets that have a bucket version mismatch will lead to a error From f268030fe12284472af8d2772cc438409590498f Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Thu, 9 Jan 2025 18:04:44 +0000 Subject: [PATCH 18/22] Added background true to mixed schema --- .../rewrite_timeseries_mixed_schema_buckets.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/timeseries-mixed-schema/rewrite_timeseries_mixed_schema_buckets.js b/timeseries-mixed-schema/rewrite_timeseries_mixed_schema_buckets.js index 6208485..56b8631 100644 --- a/timeseries-mixed-schema/rewrite_timeseries_mixed_schema_buckets.js +++ b/timeseries-mixed-schema/rewrite_timeseries_mixed_schema_buckets.js @@ -235,7 +235,7 @@ tempTimeseriesBucketsColl.drop(); // print("Validating that there are no mixed-schema buckets left ...\n"); db.getMongo().setReadPref("secondaryPreferred"); -const validateRes = coll.validate(); +const validateRes = coll.validate({background: true}); if (validateRes.warnings.length != 0) { print("\nThere is still a time-series bucket with mixed-schema data. Try re-running the script to re-insert the buckets missed."); exit(1); From 4e7065ca82611c2bf7c5101f85b099e61fc9ca11 Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Thu, 9 Jan 2025 19:34:31 +0000 Subject: [PATCH 19/22] Added extended time range check and additional comments on extended time range check --- bucket-version-mismatch/README.md | 12 ++++++------ ...write_timeseries_bucket_version_mismatch.js | 4 ++-- embedded-timestamp-mismatch/README.md | 18 +++++++++++------- ..._embedded_bucket_id_control_min_mismatch.js | 15 ++++++++++++--- 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/bucket-version-mismatch/README.md b/bucket-version-mismatch/README.md index 616ff5d..ca1d02d 100644 --- a/bucket-version-mismatch/README.md +++ b/bucket-version-mismatch/README.md @@ -25,13 +25,13 @@ If you are using these scripts on your own, we strongly recommend: Please see [SERVER-94471](https://jira.mongodb.org/browse/SERVER-94471) for the affected versions. Users can determine if they have been impacted by running [`validate`](https://www.mongodb.com/docs/v7.0/reference/command/validate/) on their Time Series collections and checking the `validate.errors` field to determine if there are buckets with mismatched versions. The validation command [can be very impactful](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance). To minimize the performance impact of running validate, issue validate to a secondary and follow [these steps](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance:~:text=Validation%20has%20exclusive,the%20hidden%20node). -## Validation Results for v8.1+ +## Validation Results for v8.1.0+ Example `validate` run on a standalone/replica set: ``` // Call validate on a mongod process for replica sets. coll.validate(); -// For v8.1+, the errors field detects bucket version mismatch. +// For v8.1.0+, the errors field detects bucket version mismatch. { "ns" : "db.system.buckets.coll", ... @@ -55,7 +55,7 @@ Example `validate` run on a sharded cluster: ``` // Call validate on mongos for sharded clusters. coll.validate(); -// For v8.1+, the errors field detects bucket version mismatch. +// For v8.1.0+, the errors field detects bucket version mismatch. // For sharded clusters, this output is an object with a result for every shard in // the "raw" field. { @@ -91,13 +91,13 @@ or ..."c":"STORAGE", "id":6698300, "ctx":"conn9","msg":"Document is not compliant with time-series specifications","attr":{..."reason":{"code":2,"codeName":"BadValue","errmsg":"Time-series bucket is v3 but has its measurements in-order on time"}}... ``` -## Validation Results Before v8.1 +## Validation Results Before v8.1.0 Example `validate` run on a standalone/replica set: ``` // Call validate on a mongod process for replica sets. coll.validate(); -// For versions below v8.1, the warnings field detects bucket version mismatch. +// For versions below v8.1.0, the warnings field detects bucket version mismatch. { "ns" : "db.system.buckets.coll", ... @@ -121,7 +121,7 @@ Example `validate` run on a sharded cluster: ``` // Call validate on mongos for sharded clusters. coll.validate(); -// For versions below v8.1, the warnings field detects bucket version mismatch. +// For versions below v8.1.0, the warnings field detects bucket version mismatch. // For sharded clusters, this output is an object with a result for every shard in // the "raw" field. { diff --git a/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js b/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js index 12eaab7..6bfb754 100644 --- a/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js +++ b/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js @@ -103,10 +103,10 @@ db.getMongo().setReadPref('secondaryPreferred'); const validateRes = collName.validate({background: true}); // -// For v8.1+, buckets that have a bucket version mismatch will lead to a error +// For v8.1.0+, buckets that have a bucket version mismatch will lead to a error // during validation. // -// Prior to v8.1, buckets that have a bucket version mismatch will lead to a +// Prior to v8.1.0, buckets that have a bucket version mismatch will lead to a // warning during validation. // if (validateRes.errors.length != 0 || validateRes.warnings.length != 0) { diff --git a/embedded-timestamp-mismatch/README.md b/embedded-timestamp-mismatch/README.md index 931c80d..2a0c1dc 100644 --- a/embedded-timestamp-mismatch/README.md +++ b/embedded-timestamp-mismatch/README.md @@ -21,19 +21,23 @@ If you are using these scripts on your own, we strongly recommend: - We have already reached out to impacted customers. If we have reached out, you can skip the [Determine if You're Impacted section](#Determine-if-You're-Impacted) and start taking the steps under the [Remediation section](#remediation). - Additionally, we recommend using the [Atlas Admin](https://www.mongodb.com/docs/atlas/security-add-mongodb-users/#built-in-roles) role. +## Validation Version Warning + +This script should be run with clusters that have have been validated with validation on the versions listed [here](https://jira.mongodb.org/browse/SERVER-97441) and [here](https://jira.mongodb.org/browse/SERVER-87065). Running this script when validation fails on other versions will not remediate the validation error/warning associated with this issue, as described in [Determine if You're Impacted](#determine-if-youre-impacted) + # Determine if You're Impacted -Please see [SERVER-94559](https://jira.mongodb.org/browse/SERVER-94559) for the affected versions. Users can determine if they have been impacted by running [`validate`](https://www.mongodb.com/docs/v7.0/reference/command/validate/) on their Time Series collections and checking the `validate.errors` field (for v8.1+) or the `validate.warnings` field (for below v8.1) otherwise to determine if there are buckets with mismatched versions. +Please see [SERVER-94559](https://jira.mongodb.org/browse/SERVER-94559) for the affected versions. Users can determine if they have been impacted by running [`validate`](https://www.mongodb.com/docs/v7.0/reference/command/validate/) on their Time Series collections and checking the `validate.errors` field (for v8.1.0+) or the `validate.warnings` field (for below v8.1.0) otherwise to determine if there are buckets with mismatched versions. The validation command [can be very impactful](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance). To minimize the performance impact of running validate, issue validate to a secondary and follow [these steps](https://www.mongodb.com/docs/v7.0/reference/method/db.collection.validate/#performance:~:text=Validation%20has%20exclusive,the%20hidden%20node). -## Validation Results for v8.1+ +## Validation Results for v8.1.0+ Example `validate` run on a standalone/replica set: ``` // Call validate on a mongod process for replica sets. coll.validate(); -// For v8.1+, the errors field detects a bucket(s) that has mismatched embedded bucket id +// For v8.1.0+, the errors field detects a bucket(s) that has mismatched embedded bucket id // timestamp and control.min timestamp. { "ns" : "db.system.buckets.coll", @@ -56,7 +60,7 @@ Example `validate` run on a sharded cluster: ``` // Call validate on mongos for sharded clusters. coll.validate(); -// For v8.1+, the errors field detects a bucket(s) that has mismatched embedded bucket id +// For v8.1.0+, the errors field detects a bucket(s) that has mismatched embedded bucket id // timestamp and control.min timestamp. // For sharded clusters, this output is an object with a result for every shard in // the "raw" field. @@ -90,12 +94,12 @@ with the logs: "bucket '_id' field and the timestamp [...] in 'control.min' field."}}... ``` -## Validation Results Before v8.1 +## Validation Results Before v8.1.0 Example `validate` run on a standalone/replica set: ``` // Call validate on a mongod process for replica sets. -// For versions below v8.1, the warnings field detects a bucket(s) that has mismatched +// For versions below v8.1.0, the warnings field detects a bucket(s) that has mismatched // embedded bucket id timestamp and control.min timestamp. coll.validate(); { @@ -119,7 +123,7 @@ Example `validate` run on a sharded cluster: ``` // Call validate on mongos for sharded clusters. coll.validate(); -// For versions below v8.1, the warnings field detects a bucket(s) that has mismatched +// For versions below v8.1.0, the warnings field detects a bucket(s) that has mismatched // embedded bucket id timestamp and control.min timestamp. // For sharded clusters, this output is an object with a result for every shard in // the "raw" field. diff --git a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js index f92da01..30e2149 100644 --- a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js +++ b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js @@ -66,6 +66,12 @@ function setUp() { tempTimeseriesBucketsColl = db.getCollection('system.buckets.temp'); } +// Helper function to determine if timestamp is in extended range. +function timestampInExtendedRange(timestamp) { + return timestamp < ISODate('1970-01-01T00:00:00.000Z') || + timestamp > ISODate('2038-01-19T03:14:07.000Z') +} + // Main function. function runFixEmbeddedBucketIdControlMinMismatchProcedure() { setUp(); @@ -80,7 +86,10 @@ function runFixEmbeddedBucketIdControlMinMismatchProcedure() { const oidTimestamp = bucket._id.getTimestamp(); const controlMinTimestamp = bucket.control.min.t - if (oidTimestamp != controlMinTimestamp) { + // If this collection has extended-range measurements, we cannot assert that + // the minTimestamp matches the embedded timestamp. + if (!timestampInExtendedRange(controlMinTimestamp) && + oidTimestamp != controlMinTimestamp) { reinsertMeasurementsFromBucket(bucket._id); } } @@ -196,10 +205,10 @@ db.getMongo().setReadPref('secondaryPreferred'); const validateRes = coll.validate({background: true}); // -// For v8.1+, buckets that have a mismatched embedded bucket id timestamp and +// For v8.1.0+, buckets that have a mismatched embedded bucket id timestamp and // control.min timestamp will lead to a error during validation. // -// Prior to v8.1, buckets that have a mismatched embedded bucket id timestamp +// Prior to v8.1.0, buckets that have a mismatched embedded bucket id timestamp // and control.min timestamp will lead to a warning during validation. // if (validateRes.errors.length != 0 || validateRes.warnings.length != 0) { From 807ce89063237769b6d84496c0be8ca40b5f34af Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Thu, 9 Jan 2025 21:31:26 +0000 Subject: [PATCH 20/22] Made the timestamp comparison more robust --- .../rewrite_embedded_bucket_id_control_min_mismatch.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js index 30e2149..802c5fe 100644 --- a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js +++ b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js @@ -68,8 +68,8 @@ function setUp() { // Helper function to determine if timestamp is in extended range. function timestampInExtendedRange(timestamp) { - return timestamp < ISODate('1970-01-01T00:00:00.000Z') || - timestamp > ISODate('2038-01-19T03:14:07.000Z') + return timestamp < new Date(ISODate('1970-01-01T00:00:00.000Z')).getTime() || + timestamp > new Date(ISODate('2038-01-19T03:14:07.000Z')).getTime() } // Main function. @@ -83,11 +83,12 @@ function runFixEmbeddedBucketIdControlMinMismatchProcedure() { // types match. If they do not match, re-insert the bucket. while (cursor.hasNext()) { const bucket = cursor.next(); - const oidTimestamp = bucket._id.getTimestamp(); - const controlMinTimestamp = bucket.control.min.t + const oidTimestamp = new Date(bucket._id.getTimestamp()).getTime(); + const controlMinTimestamp = new Date(bucket.control.min.t).getTime(); // If this collection has extended-range measurements, we cannot assert that // the minTimestamp matches the embedded timestamp. + print(controlMinTimestamp == oidTimestamp) if (!timestampInExtendedRange(controlMinTimestamp) && oidTimestamp != controlMinTimestamp) { reinsertMeasurementsFromBucket(bucket._id); From d004e1937b6706f34586489f0085a8601ed939b8 Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Fri, 10 Jan 2025 12:31:11 +0000 Subject: [PATCH 21/22] Added re-running note, removed unnecessary print, added checking for specific log id in errors/warnings --- .../rewrite_timeseries_bucket_version_mismatch.js | 7 +++++-- .../rewrite_embedded_bucket_id_control_min_mismatch.js | 8 +++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js b/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js index 6bfb754..bed57fc 100644 --- a/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js +++ b/bucket-version-mismatch/rewrite_timeseries_bucket_version_mismatch.js @@ -109,9 +109,12 @@ const validateRes = collName.validate({background: true}); // Prior to v8.1.0, buckets that have a bucket version mismatch will lead to a // warning during validation. // -if (validateRes.errors.length != 0 || validateRes.warnings.length != 0) { +if ((validateRes.errors.length != 0 && + validateRes.errors.some(x => x.includes('6698300'))) || + (validateRes.warnings.length != 0 && + validateRes.warnings.some(x => x.includes('6698300')))) { print( - '\nThere is still a time-series bucket with a bucket version mismatch, or there is another error or warning during validation.'); + '\nThere is still a time-series bucket(s) that has a bucket version mismatch, or there is another error or warning during validation regarding incompatible time-series documents. Check logs with id 6698300.'); exit(1); } diff --git a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js index 802c5fe..70d7f05 100644 --- a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js +++ b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js @@ -88,7 +88,6 @@ function runFixEmbeddedBucketIdControlMinMismatchProcedure() { // If this collection has extended-range measurements, we cannot assert that // the minTimestamp matches the embedded timestamp. - print(controlMinTimestamp == oidTimestamp) if (!timestampInExtendedRange(controlMinTimestamp) && oidTimestamp != controlMinTimestamp) { reinsertMeasurementsFromBucket(bucket._id); @@ -212,9 +211,12 @@ const validateRes = coll.validate({background: true}); // Prior to v8.1.0, buckets that have a mismatched embedded bucket id timestamp // and control.min timestamp will lead to a warning during validation. // -if (validateRes.errors.length != 0 || validateRes.warnings.length != 0) { +if ((validateRes.errors.length != 0 && + validateRes.errors.some(x => x.includes('6698300'))) || + (validateRes.warnings.length != 0 && + validateRes.warnings.some(x => x.includes('6698300')))) { print( - '\nThere is still a bucket(s) that has a mismatched embedded bucket id timestamps and control.min timestamps, or there is another error or warning during validation.'); + '\nThere is still a time-series bucket(s) that has a mismatched embedded bucket id timestamps and control.min timestamps. Try re-running the script to re-insert missed buckets, or check logs with id 6698300 to see if there is another error or warning during validation regarding incompatible time-series documents.'); exit(1); } From bfb6634c44a02e9b70c2cbcacf827fa5cfda64c4 Mon Sep 17 00:00:00 2001 From: Stephanie Eristoff Date: Fri, 10 Jan 2025 12:38:39 +0000 Subject: [PATCH 22/22] Removed longggg print --- .../rewrite_embedded_bucket_id_control_min_mismatch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js index 70d7f05..d7fa5a7 100644 --- a/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js +++ b/embedded-timestamp-mismatch/rewrite_embedded_bucket_id_control_min_mismatch.js @@ -216,7 +216,7 @@ if ((validateRes.errors.length != 0 && (validateRes.warnings.length != 0 && validateRes.warnings.some(x => x.includes('6698300')))) { print( - '\nThere is still a time-series bucket(s) that has a mismatched embedded bucket id timestamps and control.min timestamps. Try re-running the script to re-insert missed buckets, or check logs with id 6698300 to see if there is another error or warning during validation regarding incompatible time-series documents.'); + '\nThere is still a time-series bucket(s) that has a mismatched embedded bucket id timestamps and control.min timestamps. Try re-running the script to re-insert missed buckets.'); exit(1); }