Hugo Di FrancescoA case-study on where recursion can be useful for enterprise Node.js applications and how to avoid its common pitfalls like RangeError: Maximum call stack size exceeded.
The full repository for this post is on GitHub: github.com/HugoDF/mongo-query-clause-modification
We’ll be implementing a solution to 2 real-world problems:
- Add an $or query clause to a MongoDB query
- Remove references to a field in an MongoDB query (potentially) using $or and $and
Add an $or query clause to a MongoDB query
See the final code at ./src/add-or-clause.js.
The parameters are query and orClause.
query is a MongoDB query which might or might not already contain an $or and/or $and clause.
orClause is an object containing and $or clause (it’s a fully-fledged MongoDB query in its own right) eg.
const orClause = {
$or: [
{createdAt: {$exists: false}},
{createdAt: someDate}
]
};
There is initially just 1 thing to look out for:1. the query does not contain an $or clause2. the query contains an $or clause
When there’s no $or clause in the query
If there is no $or clause, we can simply spread our orClause query and the query parameter, ie.
const newQuery = {
...query,
...orClause
};
That is unless there’s and $and in there somewhere, in which case we want to add our orClause to the $and:
const newQuery = {
...query,
$and: [...query.$and, orClause]
};
When there’s an $or clause in the query
If there is an $or clause, we can’t just overwrite it, we need to $and the two $or queries.
We should also keep existing $and clause contents which yields:
const newQuery = {
...queryWithoutOrRemoved,
$and: [
...(query.$and || []),
{ $or: query.$or },
orClause
]
};
Full solution
This is also available at ./src/add-or-clause.js.
function addOrClause(query, orClause) {
const {$or, ...queryRest} = query;
if ($or) {
return {
...queryRest,
$and: [...(queryRest.$and || []), {$or}, orClause]
};
}
if (queryRest.$and) {
return {
...queryRest,
$and: [...queryRest.$and, orClause]
};
}
return {
...query,
...orClause
};
}
module.exports = addOrClause;
Corresponding Test Suite
We can observe how the different cases map pretty directly to test cases.
const addOrClause = require('./add-or-clause');
test('should add the passed or clause if no $or on the current query', () => {
const orClause = {$or: [{myField: 'value'}, {myField: null}]};
const query = {foo: 'bar'};
expect(addOrClause(query, orClause)).toEqual({
$or: [{myField: 'value'}, {myField: null}],
foo: 'bar'
});
});
describe('when the query already has an $or', () => {
test('should add the passed or clause to and $and that also contains the current query', () => {
const orClause = {$or: [{myField: 'value'}, {myField: null}]};
const query = {$or: [{foo: 'bar'}, {foo: {$exists: false}}]};
expect(addOrClause(query, orClause)).toEqual({
$and: [
{$or: [{foo: 'bar'}, {foo: {$exists: false}}]},
{
$or: [{myField: 'value'}, {myField: null}]
}
]
});
});
describe('when the query has an $and', () => {
test('should keep the $and, add the $or and the current query', () => {
const orClause = {$or: [{myField: 'value'}, {myField: null}]};
const query = {
$or: [{hello: 'world'}],
$and: [{foo: 'bar'}, {bar: 'baz'}]
};
expect(addOrClause(query, orClause)).toEqual({
$and: [
{foo: 'bar'},
{bar: 'baz'},
{$or: [{hello: 'world'}]},
{$or: [{myField: 'value'}, {myField: null}]}
]
});
});
});
});
describe('when the query has an $and query', () => {
test('should add the new or clause to the $and', () => {
const orClause = {$or: [{myField: 'value'}, {myField: null}]};
const query = {$and: [{foo: 'bar'}, {bar: 'baz'}]};
expect(addOrClause(query, orClause)).toEqual({
$and: [
{foo: 'bar'},
{bar: 'baz'},
{$or: [{myField: 'value'}, {myField: null}]}
]
});
});
});
Remove references to a field in an MongoDB query (potentially) using $or and $and
See the full solution at ./src/remove-field-references.js.
In this case we’re creating a function that takes 2 parameters: query (MongoDB query as above) and fieldName (name of the field we want to remove references to).
Remove top-level fields
The simplest thing to do is remove references to the field at the top-level of the object.
We can create a simple omit function using destructuring and recursion
const omit = (obj, [field, ...nextFields]) => {
const {[field]: ignore, ...rest} = obj;
return nextFields.length > 0 ? omit(rest, nextFields) : rest;
};
And use it:
const newQuery = omit(query, [fieldName]);
Remove fields in any $or clause
To remove fields in an $or clause (which is a fully-fledged query) is as simple as taking the $or value (which is an array) and running a recursion of the function onto it.
This will remove fields at the top-level of the $or sub-queries and in nest $or fields’ sub-queries.
We want to make sure to remove empty $or sub-queries, since { $or: [{ }, {}]} is an invalid query.
We default the query’s $or to an empty array and check length before spreading it back into the newQuery. This is because { $or: [] } is an invalid query.
We’re also careful to remove the top-level $or when spreading filteredTopLevel so that if the new $or is an empty array, the old $or is ommitted.
function removeFieldReferences (query, fieldName) {
const filteredTopLevel = omit(query, [fieldName]);
const newOr = (filteredTopLevel.$or || [])
.map(q => removeFieldReferences(q, fieldName))
.filter(q => Object.keys(q).length > 0);
return {
...omit(filteredTopLevel, ['$or']),
...(newOr.length > 0 ? {$or: newOr} : {})
};
}
Remove fields in any $and clause
The rationale for the $and solution is the same as for the $or solution.
We recurse and check that we’re not generating an invalid query by omitting empty arrays and objects:
function removeFieldReferences (query, fieldName) {
const filteredTopLevel = omit(query, [fieldName]);
const newAnd = (filteredTopLevel.$and || [])
.map(q => removeFieldReferences(q, fieldName))
.filter(q => Object.keys(q).length > 0);
return {
...omit(filteredTopLevel, ['$and']),
...(newAnd.length > 0 ? {$and: newAnd} : {})
};
}
Check that we’re not likely to bust the stack
The actual implementation has a maxDepth 3rd parameter defaulted to 5.
When maxDepth is equal to 0, we return the query without any treatment (arguably we should run the top-level filter).
On recursive calls to removeFieldReferences we pass (q, fieldName, maxDepth - 1) so that we’re not going any deeper than we need to by accident.
This avoids RangeError: Maximum call stack size exceeded.
Final Implementation
This is also available at ./src/remove-field-references.js.
const omit = (obj, [field, ...nextFields]) => {
const {[field]: ignore, ...rest} = obj;
return nextFields.length > 0 ? omit(rest, nextFields) : rest;
};
function removeFieldReferences(query, fieldName, maxDepth = 5) {
if (maxDepth <= 0) {
return query;
}
const filteredTopLevel = omit(query, [fieldName]);
const newOr = (filteredTopLevel.$or || [])
.map(q => removeFieldReferences(q, fieldName, maxDepth - 1))
.filter(q => Object.keys(q).length > 0);
const newAnd = (filteredTopLevel.$and || [])
.map(q => removeFieldReferences(q, fieldName, maxDepth - 1))
.filter(q => Object.keys(q).length > 0);
return {
...omit(filteredTopLevel, ['$or', '$and']),
...(newOr.length > 0 ? {$or: newOr} : {}),
...(newAnd.length > 0 ? {$and: newAnd} : {})
};
}
module.exports = removeFieldReferences;
Corresponding Test Suite
We can observe how the different cases map pretty directly to test cases.
const removeFieldReferences = require('./remove-field-references');
test('should remove top-level fields', () => {
const query = {
hello: 'value'
};
const fieldName = 'hello';
expect(removeFieldReferences(query, fieldName).hello).toBeUndefined();
});
test('should return passed query when maxDepth is hit (avoids busting the stack by default)', () => {
const query = {
hello: 'value'
};
const fieldName = 'hello';
expect(removeFieldReferences(query, fieldName, 0)).toEqual(query);
});
test('should remove references to the field in top-level $or queries', () => {
const query = {
$or: [
{hello: 'value', otherField: 'not-related'},
{hello: 'othervalue', otherField: 'even-less-related'}
]
};
const fieldName = 'hello';
expect(removeFieldReferences(query, fieldName)).toEqual({
$or: [{otherField: 'not-related'}, {otherField: 'even-less-related'}]
});
});
test('should remove $or clauses where the query becomes empty on omission of a field', () => {
const query = {
$or: [{hello: 'value'}, {otherField: 'not-related'}]
};
const fieldName = 'hello';
expect(removeFieldReferences(query, fieldName)).toEqual({
$or: [{otherField: 'not-related'}]
});
});
test('should remove references to field in top-level queries inside of $and', () => {
const query = {
$and: [
{hello: 'value', otherField: 'value'},
{hello: 'other-value', otherField: 'value'}
]
};
const fieldName = 'hello';
expect(removeFieldReferences(query, fieldName)).toEqual({
$and: [{otherField: 'value'}, {otherField: 'value'}]
});
});
test('should remove $and clause if all queries end up filtered out', () => {
const query = {
foo: 'bar',
$and: [{hello: 'value'}, {hello: 'other-value'}]
};
const fieldName = 'hello';
expect(removeFieldReferences(query, fieldName)).toEqual({foo: 'bar'});
});
test('should remove references to field in nested $or inside of $and', () => {
const query = {
$and: [
{
$or: [{hello: 'value'}, {hello: null}]
},
{otherField: 'not-related'}
]
};
const fieldName = 'hello';
expect(removeFieldReferences(query, fieldName)).toEqual({
$and: [{otherField: 'not-related'}]
});
});
