Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions packages/compass-crud/src/utils/cancellable-queries.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ describe('cancellable-queries', function () {
const cluster = mochaTestServer();
let dataService: DataService;
let preferences: PreferencesAccess;
let abortController;
let signal;
let abortController: AbortController;
let signal: AbortSignal;

before(async function () {
preferences = await createSandboxFromDefaultPreferences();
Expand Down Expand Up @@ -99,7 +99,7 @@ describe('cancellable-queries', function () {
dataService,
preferences,
'cancel.numbers',
null,
{},
{
signal,
}
Expand Down Expand Up @@ -138,7 +138,7 @@ describe('cancellable-queries', function () {
dataService,
preferences,
'cancel.numbers',
'this is not a filter',
{},
{
signal,
hint: { _id_: 1 }, // this collection doesn't have this index so this query should fail
Expand Down
16 changes: 5 additions & 11 deletions packages/compass-crud/src/utils/cancellable-queries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,19 +79,13 @@ export async function fetchShardingKeys(
}
): Promise<BSONObject> {
try {
const docs = await dataService.find(
'config.collections',
{
_id: ns as any,
// unsplittable introduced in SPM-3364 to mark unsharded collections
// that are still being tracked in the catalog
unsplittable: { $ne: true },
},
{ maxTimeMS, projection: { key: 1, _id: 0 } },
const shardKey = await dataService.fetchShardKey(
ns,
{ maxTimeMS },
{ abortSignal: signal }
);
return docs.length ? docs[0].key : {};
} catch (err: any) {
return shardKey ?? {};
} catch (err) {
// rethrow if we aborted along the way
if (dataService.isCancelError(err)) {
throw err;
Expand Down
3 changes: 2 additions & 1 deletion packages/compass-crud/src/utils/data-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ export type RequiredDataServiceProps =
| 'collectionStats'
| 'collectionInfo'
| 'listCollections'
| 'isListSearchIndexesSupported';
| 'isListSearchIndexesSupported'
| 'fetchShardKey';
// TODO: It might make sense to refactor the DataService interface to be closer to
// { ..., getCSFLEMode(): 'unavailable' } | { ..., getCSFLEMode(): 'unavailable' | 'enabled' | 'disabled', isUpdateAllowed(): ..., knownSchemaForCollection(): ... }
// so that either these methods are always present together or always absent
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,19 @@ export const getPropertyTooltip = (
return null;
};

const HIDDEN_INDEX_TEXT = 'HIDDEN';
const SHARD_KEY_INDEX_TEXT = 'SHARD KEY';

export const getPropertyText = (
property: RegularIndex['properties'][number]
): string => {
if (property === 'shardKey') {
return SHARD_KEY_INDEX_TEXT;
}

return property;
};

const PropertyBadgeWithTooltip: React.FunctionComponent<{
text: string;
link: string;
Expand Down Expand Up @@ -65,8 +78,6 @@ type PropertyFieldProps = {
properties: RegularIndex['properties'];
};

const HIDDEN_INDEX_TEXT = 'HIDDEN';

const PropertyField: React.FunctionComponent<PropertyFieldProps> = ({
extra,
properties,
Expand All @@ -79,7 +90,7 @@ const PropertyField: React.FunctionComponent<PropertyFieldProps> = ({
return (
<PropertyBadgeWithTooltip
key={property}
text={property}
text={getPropertyText(property)}
link={getIndexHelpLink(property) ?? '#'}
tooltip={getPropertyTooltip(property, extra)}
/>
Expand Down
2 changes: 2 additions & 0 deletions packages/compass-indexes/src/utils/index-link-helper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@ const HELP_URLS = {
'https://docs.mongodb.com/master/reference/bson-type-comparison-order/#collation',
COLLATION_REF: 'https://docs.mongodb.com/master/reference/collation',
HIDDEN: 'https://www.mongodb.com/docs/manual/core/index-hidden/',
SHARDKEY: 'https://www.mongodb.com/docs/manual/core/sharding-shard-key/',
UNKNOWN: null,
};

export type HELP_URL_KEY =
| 'shardKey' // The only camelCase key at the moment.
| Uppercase<keyof typeof HELP_URLS>
| Lowercase<keyof typeof HELP_URLS>;

Expand Down
136 changes: 135 additions & 1 deletion packages/data-service/src/data-service.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ describe('DataService', function () {
fatal: () => {},
};

let dataServiceLogTest;
let dataServiceLogTest: DataService | undefined;

beforeEach(function () {
logs.length = 0;
Expand Down Expand Up @@ -1557,6 +1557,24 @@ describe('DataService', function () {
});
});

describe('#fetchShardKey', function () {
beforeEach(async function () {
await mongoClient
.db(testDatabaseName)
.collection(testCollectionName)
.createIndex(
{
a: 1,
},
{}
);
});

it('fetches the shard key (there is none in this test)', async function () {
expect(await dataService.fetchShardKey(testNamespace)).to.equal(null);
});
});

describe('CSFLE logging', function () {
it('picks a selected set of CSFLE options for logging', function () {
const fleOptions: ConnectionFleOptions = {
Expand Down Expand Up @@ -2010,6 +2028,122 @@ describe('DataService', function () {
});
});

context('with real sharded cluster', function () {
this.slow(10_000);
this.timeout(20_000);

const cluster = mochaTestServer({
topology: 'sharded',
secondaries: 0,
});

let dataService: DataServiceImpl;
let mongoClient: MongoClient;
let connectionOptions: ConnectionOptions;
let testCollectionName: string;
let testDatabaseName: string;
let testNamespace: string;

before(async function () {
testDatabaseName = `compass-data-service-sharded-tests`;
const connectionString = cluster().connectionString;
connectionOptions = {
connectionString,
};

mongoClient = new MongoClient(connectionOptions.connectionString);
await mongoClient.connect();

dataService = new DataServiceImpl(connectionOptions);
await dataService.connect();
});

after(async function () {
// eslint-disable-next-line no-console
await dataService?.disconnect().catch(console.log);
await mongoClient?.close();
});

beforeEach(async function () {
testCollectionName = `coll-${new UUID().toString()}`;
testNamespace = `${testDatabaseName}.${testCollectionName}`;

await mongoClient
.db(testDatabaseName)
.collection(testCollectionName)
.insertMany(TEST_DOCS);

await mongoClient
.db(testDatabaseName)
.collection(testCollectionName)
.createIndex(
{
a: 1,
},
{}
);
});

afterEach(async function () {
sinon.restore();

await mongoClient
.db(testDatabaseName)
.collection(testCollectionName)
.drop();
});

describe('with a sharded collection', function () {
beforeEach(async function () {
await runCommand(dataService['_database']('admin', 'META'), {
shardCollection: testNamespace,
key: {
a: 1,
},
// We don't run the shardCollection command outside of tests
// so it isn't part of the runCommand type.
} as unknown as Parameters<typeof runCommand>[1]);
});

describe('#fetchShardKey', function () {
it('fetches the shard key', async function () {
expect(await dataService.fetchShardKey(testNamespace)).to.deep.equal({
a: 1,
});

// Can be cancelled.
const abortController = new AbortController();
const abortSignal = abortController.signal;
const promise = dataService
.fetchShardKey(
testNamespace,
{},
{ abortSignal: abortSignal as unknown as AbortSignal }
)
.catch((err) => err);
abortController.abort();
const error = await promise;

expect(dataService.isCancelError(error)).to.be.true;
});
});

describe('#indexes', function () {
it('includes the shard key', async function () {
const indexes = await dataService.indexes(testNamespace);

expect(indexes.length).to.equal(2);
expect(
indexes.find((index) => index.key._id === 1)?.properties
).to.not.include('shardKey');
expect(
indexes.find((index) => index.key.a === 1)?.properties
).to.include('shardKey');
});
});
});
});

context('with mocked client', function () {
function createDataServiceWithMockedClient(
clientConfig: Partial<ClientMockOptions>
Expand Down
68 changes: 61 additions & 7 deletions packages/data-service/src/data-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -733,6 +733,19 @@ export interface DataService {
}
): Promise<Document[]>;

/**
* Fetch shard keys for the collection from the collections config.
*
* @param ns - The namespace to try to find shard key for.
* @param options - The query options.
* @param executionOptions - The execution options.
*/
fetchShardKey(
ns: string,
options?: Omit<FindOptions, 'projection'>,
executionOptions?: ExecutionOptions
): Promise<Record<string, unknown> | null>;

/*** Insert ***/

/**
Expand Down Expand Up @@ -2234,6 +2247,44 @@ class DataServiceImpl extends WithLogContext implements DataService {
return indexToProgress;
}

@op(mongoLogId(1_001_000_380))
async fetchShardKey(
ns: string,
options: Omit<FindOptions, 'projection'> = {},
executionOptions?: ExecutionOptions
): Promise<Record<string, unknown> | null> {
const docs = await this.find(
'config.collections',
{
// eslint-disable-next-line @typescript-eslint/no-explicit-any
_id: ns as any,
// unsplittable introduced in SPM-3364 to mark unsharded collections
// that are still being tracked in the catalog
unsplittable: { $ne: true },
},
{ ...options, projection: { key: 1, _id: 0 } },
{ abortSignal: executionOptions?.abortSignal }
);
return docs.length ? docs[0].key : null;
}

private async _fetchShardKeyWithSilentFail(
...args: Parameters<DataService['fetchShardKey']>
): ReturnType<DataService['fetchShardKey']> {
try {
return await this.fetchShardKey(...args);
} catch (err) {
// Rethrow if we aborted along the way.
if (this.isCancelError(err)) {
throw err;
}

// Return null on error.
// This is oftentimes a lack of permissions to run a find on config.collections.
return null;
}
}

@op(mongoLogId(1_001_000_047))
async indexes(
ns: string,
Expand All @@ -2245,19 +2296,21 @@ class DataServiceImpl extends WithLogContext implements DataService {
);
return indexes.map((compactIndexEntry) => {
const [name, keys] = compactIndexEntry;
return createIndexDefinition(ns, {
return createIndexDefinition(ns, null, {
name,
key: Object.fromEntries(keys),
});
});
}

const [indexes, indexStats, indexSizes, indexProgress] = await Promise.all([
this._collection(ns, 'CRUD').indexes({ ...options, full: true }),
this._indexStats(ns),
this._indexSizes(ns),
this._indexProgress(ns),
]);
const [indexes, indexStats, indexSizes, indexProgress, shardKey] =
await Promise.all([
this._collection(ns, 'CRUD').indexes({ ...options, full: true }),
this._indexStats(ns),
this._indexSizes(ns),
this._indexProgress(ns),
this._fetchShardKeyWithSilentFail(ns),
]);

const maxSize = Math.max(...Object.values(indexSizes));

Expand All @@ -2269,6 +2322,7 @@ class DataServiceImpl extends WithLogContext implements DataService {
const name = index.name;
return createIndexDefinition(
ns,
shardKey ?? null,
index,
indexStats[name],
indexSizes[name],
Expand Down
Loading
Loading