Skip to content

Sort server-side in SavedObject export #55128

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tylersmalley merged 5 commits into from
Jan 28, 2020
Merged

Sort server-side in SavedObject export #55128

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0c735e1
Sort SavedObjects server-side instead of by _id in ES
tylersmalley Jan 16, 2020
c821ba3
Adds comment
tylersmalley Jan 24, 2020
48883f9
Sorts as part of fetchObjectsToExport
tylersmalley Jan 27, 2020
4935025
Merge remote-tracking branch 'upstream/master' into remove-sort-by-id
tylersmalley Jan 27, 2020
6b9becc
Cleanup object assignment
tylersmalley Jan 28, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 73 additions & 6 deletions src/core/server/saved_objects/export/get_sorted_objects_for_export.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,6 @@ describe('getSortedObjectsForExport()', () => {
"namespace": undefined,
"perPage": 500,
"search": undefined,
"sortField": "_id",
"sortOrder": "asc",
"type": Array [
"index-pattern",
"search",
Expand Down Expand Up @@ -256,8 +254,6 @@ describe('getSortedObjectsForExport()', () => {
"namespace": undefined,
"perPage": 500,
"search": "foo",
"sortField": "_id",
"sortOrder": "asc",
"type": Array [
"index-pattern",
"search",
Expand Down Expand Up @@ -345,8 +341,6 @@ describe('getSortedObjectsForExport()', () => {
"namespace": "foo",
"perPage": 500,
"search": undefined,
"sortField": "_id",
"sortOrder": "asc",
"type": Array [
"index-pattern",
"search",
Expand Down Expand Up @@ -399,6 +393,79 @@ describe('getSortedObjectsForExport()', () => {
).rejects.toThrowErrorMatchingInlineSnapshot(`"Can't export more than 1 objects"`);
});

test('sorts objects within type', async () => {
savedObjectsClient.find.mockResolvedValueOnce({
Copy link
Member Author

@tylersmalley tylersmalley Jan 27, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There previously was only coverage when objects were provided, resulting in the bulk request - so this also has the benefit of including that path using find.

total: 3,
per_page: 10000,
page: 1,
saved_objects: [
{
id: '3',
type: 'index-pattern',
attributes: {
name: 'baz',
},
references: [],
},
{
id: '1',
type: 'index-pattern',
attributes: {
name: 'foo',
},
references: [],
},
{
id: '2',
type: 'index-pattern',
attributes: {
name: 'bar',
},
references: [],
},
],
});
const exportStream = await getSortedObjectsForExport({
exportSizeLimit: 10000,
savedObjectsClient,
types: ['index-pattern'],
});
const response = await readStreamToCompletion(exportStream);
expect(response).toMatchInlineSnapshot(`
Array [
Object {
"attributes": Object {
"name": "foo",
},
"id": "1",
"references": Array [],
"type": "index-pattern",
},
Object {
"attributes": Object {
"name": "bar",
},
"id": "2",
"references": Array [],
"type": "index-pattern",
},
Object {
"attributes": Object {
"name": "baz",
},
"id": "3",
"references": Array [],
"type": "index-pattern",
},
Object {
"exportedCount": 3,
"missingRefCount": 0,
"missingReferences": Array [],
},
]
`);
});

test('exports selected objects and sorts them', async () => {
savedObjectsClient.bulkGet.mockResolvedValueOnce({
saved_objects: [
Expand Down
19 changes: 12 additions & 7 deletions src/core/server/saved_objects/export/get_sorted_objects_for_export.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

import Boom from 'boom';
import { createListStream } from '../../../../legacy/utils/streams';
import { SavedObjectsClientContract } from '../types';
import { SavedObjectsClientContract, SavedObject } from '../types';
import { fetchNestedDependencies } from './inject_nested_depdendencies';
import { sortObjects } from './sort_objects';

Expand Down Expand Up @@ -105,15 +105,17 @@ async function fetchObjectsToExport({
const findResponse = await savedObjectsClient.find({
type: types,
search,
sortField: '_id',
sortOrder: 'asc',
perPage: exportSizeLimit,
namespace,
});
if (findResponse.total > exportSizeLimit) {
throw Boom.badRequest(`Can't export more than ${exportSizeLimit} objects`);
}
return findResponse.saved_objects;

// sorts server-side by _id, since it's only available in fielddata
return findResponse.saved_objects.sort((a: SavedObject, b: SavedObject) =>
a.id > b.id ? 1 : -1
);
} else {
throw Boom.badRequest('Either `type` or `objects` are required.');
}
Expand All @@ -137,14 +139,17 @@ export async function getSortedObjectsForExport({
exportSizeLimit,
namespace,
});
let exportedObjects = [...rootObjects];
let exportedObjects = [];
let missingReferences: SavedObjectsExportResultDetails['missingReferences'] = [];

if (includeReferencesDeep) {
const fetchResult = await fetchNestedDependencies(rootObjects, savedObjectsClient, namespace);
exportedObjects = fetchResult.objects;
exportedObjects = sortObjects(fetchResult.objects);
missingReferences = fetchResult.missingRefs;
} else {
exportedObjects = sortObjects(rootObjects);
}
exportedObjects = sortObjects(exportedObjects);

const exportDetails: SavedObjectsExportResultDetails = {
exportedCount: exportedObjects.length,
missingRefCount: missingReferences.length,
Expand Down