From a2081f65c6b6d93f0d275fa6e3b946b55bc04403 Mon Sep 17 00:00:00 2001 From: Kenan Warren Date: Wed, 28 Apr 2021 17:20:39 -0400 Subject: [PATCH 1/7] Add user entity --- docs/jupiterone.md | 6 +- src/provider/SonarqubeClient.test.ts | 97 ++++++++++++- src/provider/SonarqubeClient.ts | 9 ++ .../recording.har | 118 ++++++++++++++++ .../recording.har | 127 ++++++++++++++++++ src/provider/types.ts | 13 ++ src/steps/constants.ts | 6 + .../recording.har | 127 ++++++++++++++++++ src/steps/user/converter.test.ts | 44 ++++++ src/steps/user/converter.ts | 31 +++++ src/steps/user/index.test.ts | 70 ++++++++++ src/steps/user/index.ts | 41 ++++++ 12 files changed, 687 insertions(+), 2 deletions(-) create mode 100644 src/provider/__recordings__/iterateUsersShouldFailWithInvalidToken_3184218964/recording.har create mode 100644 src/provider/__recordings__/iterateUsersShouldFetchUsersWithValidConfig_1014868188/recording.har create mode 100644 src/steps/user/__recordings__/fetchUsersShouldCollectData_357354744/recording.har create mode 100644 src/steps/user/converter.test.ts create mode 100644 src/steps/user/converter.ts create mode 100644 src/steps/user/index.test.ts create mode 100644 src/steps/user/index.ts diff --git a/docs/jupiterone.md b/docs/jupiterone.md index f7691ef..909ed30 100644 --- a/docs/jupiterone.md +++ b/docs/jupiterone.md @@ -18,7 +18,11 @@ ## Requirements - JupiterOne requires an API token. You need permission to create a user in - Sonarqube that will be used to obtain the API token. + Sonarqube that will be used to obtain the API token. The token should have the + `Administer System` permission to allow the ability to pull extra user + metadata. More information on this can be found in the sonarqube api + documentation of your instance + (`/web_api/api/users/search`). - You must have permission in JupiterOne to install new integrations. ## Support diff --git a/src/provider/SonarqubeClient.test.ts b/src/provider/SonarqubeClient.test.ts index 445a70b..cfcb165 100644 --- a/src/provider/SonarqubeClient.test.ts +++ b/src/provider/SonarqubeClient.test.ts @@ -7,7 +7,7 @@ import { } from '@jupiterone/integration-sdk-testing'; import { createSonarqubeClient } from '.'; -import { SonarqubeProject, SonarqubeUserGroup } from './types'; +import { SonarqubeProject, SonarqubeUserGroup, SonarqubeUser } from './types'; describe('#SonarqubeClient', () => { let recording: Recording; @@ -144,3 +144,98 @@ describe('#iterateUserGroups', () => { ); }); }); + +describe('#iterateUsers', () => { + let recording: Recording; + + afterEach(async () => { + await recording.stop(); + }); + + test('should fail with invalid token', async () => { + recording = setupRecording({ + directory: __dirname, + name: 'iterateUsersShouldFailWithInvalidToken', + options: { + matchRequestsBy: { + url: { + hostname: false, + }, + }, + recordFailedRequests: true, + }, + mutateEntry: mutations.unzipGzippedRecordingEntry, + }); + + const context = createMockStepExecutionContext({ + instanceConfig: { + baseUrl: 'http://localhost:9000', + apiToken: 'string-value', + }, + }); + const provider = createSonarqubeClient(context.instance.config); + await expect( + provider.iterateUsers(() => { + // do nothing + }), + ).rejects.toThrowError(IntegrationProviderAuthenticationError); + }); + + test('should fetch users with valid config', async () => { + recording = setupRecording({ + directory: __dirname, + name: 'iterateUsersShouldFetchUsersWithValidConfig', + options: { + matchRequestsBy: { + url: { + hostname: false, + }, + }, + recordFailedRequests: true, + }, + mutateEntry: mutations.unzipGzippedRecordingEntry, + }); + + const context = createMockStepExecutionContext({ + instanceConfig: { + baseUrl: process.env.BASE_URL || 'http://localhost:9000', + apiToken: process.env.API_TOKEN || 'string-value', + }, + }); + const provider = createSonarqubeClient(context.instance.config); + + const results: SonarqubeUser[] = []; + await provider.iterateUsers((user) => { + results.push(user); + }); + + expect(results).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + login: expect.any(String), + name: expect.any(String), + active: expect.any(Boolean), + email: expect.any(String), + groups: expect.any(Array), + tokensCount: expect.any(Number), + local: expect.any(Boolean), + externalIdentity: expect.any(String), + externalProvider: expect.any(String), + avatar: expect.any(String), + }), + expect.objectContaining({ + login: expect.any(String), + name: expect.any(String), + active: expect.any(Boolean), + email: expect.any(String), + groups: expect.any(Array), + tokensCount: expect.any(Number), + local: expect.any(Boolean), + externalIdentity: expect.any(String), + externalProvider: expect.any(String), + avatar: expect.any(String), + }), + ]), + ); + }); +}); diff --git a/src/provider/SonarqubeClient.ts b/src/provider/SonarqubeClient.ts index 3d8a233..d53b898 100644 --- a/src/provider/SonarqubeClient.ts +++ b/src/provider/SonarqubeClient.ts @@ -13,6 +13,7 @@ import { PaginatedResponse, ValidationResponse, SonarqubeUserGroup, + SonarqubeUser, } from './types'; /** @@ -60,6 +61,14 @@ export class SonarqubeClient { ); } + async iterateUsers(iteratee: ResourceIteratee): Promise { + return this.iterateResources<'users', SonarqubeUser>( + '/users/search', + 'users', + iteratee, + ); + } + async fetchAuthenticationValidate(): Promise { return this.makeSingularRequest('/authentication/validate') as Promise< ValidationResponse diff --git a/src/provider/__recordings__/iterateUsersShouldFailWithInvalidToken_3184218964/recording.har b/src/provider/__recordings__/iterateUsersShouldFailWithInvalidToken_3184218964/recording.har new file mode 100644 index 0000000..f31d6b1 --- /dev/null +++ b/src/provider/__recordings__/iterateUsersShouldFailWithInvalidToken_3184218964/recording.har @@ -0,0 +1,118 @@ +{ + "log": { + "_recordingName": "iterateUsersShouldFailWithInvalidToken", + "creator": { + "comment": "persister:JupiterOneIntegationFSPersister", + "name": "Polly.JS", + "version": "4.3.0" + }, + "entries": [ + { + "_id": "0b4558b795c818d05f759e540e92a9ba", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "_fromType": "array", + "name": "connection", + "value": "close" + }, + { + "name": "host", + "value": "localhost:9000" + } + ], + "headersSize": 271, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [ + { + "name": "page", + "value": "1" + }, + { + "name": "per_page", + "value": "100" + } + ], + "url": "http://localhost:9000/api/users/search?page=1&per_page=100" + }, + "response": { + "bodySize": 0, + "content": { + "mimeType": "text/plain", + "size": 0 + }, + "cookies": [], + "headers": [ + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "content-length", + "value": "0" + }, + { + "name": "date", + "value": "Wed, 28 Apr 2021 20:52:49 GMT" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 172, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 401, + "statusText": "Unauthorized" + }, + "startedDateTime": "2021-04-28T20:52:49.778Z", + "time": 13, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 13 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/src/provider/__recordings__/iterateUsersShouldFetchUsersWithValidConfig_1014868188/recording.har b/src/provider/__recordings__/iterateUsersShouldFetchUsersWithValidConfig_1014868188/recording.har new file mode 100644 index 0000000..061c354 --- /dev/null +++ b/src/provider/__recordings__/iterateUsersShouldFetchUsersWithValidConfig_1014868188/recording.har @@ -0,0 +1,127 @@ +{ + "log": { + "_recordingName": "iterateUsersShouldFetchUsersWithValidConfig", + "creator": { + "comment": "persister:JupiterOneIntegationFSPersister", + "name": "Polly.JS", + "version": "4.3.0" + }, + "entries": [ + { + "_id": "0b4558b795c818d05f759e540e92a9ba", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "_fromType": "array", + "name": "connection", + "value": "close" + }, + { + "name": "host", + "value": "localhost:9000" + } + ], + "headersSize": 307, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [ + { + "name": "page", + "value": "1" + }, + { + "name": "per_page", + "value": "100" + } + ], + "url": "http://localhost:9000/api/users/search?page=1&per_page=100" + }, + "response": { + "bodySize": 551, + "content": { + "mimeType": "application/json", + "size": 551, + "text": "{\"paging\":{\"pageIndex\":1,\"pageSize\":50,\"total\":2},\"users\":[{\"login\":\"admin\",\"name\":\"Administrator\",\"active\":true,\"groups\":[\"sonar-administrators\",\"sonar-users\"],\"tokensCount\":3,\"local\":true,\"externalIdentity\":\"admin\",\"externalProvider\":\"sonarqube\",\"lastConnectionDate\":\"2021-04-28T20:41:19+0000\"},{\"login\":\"testUser\",\"name\":\"kenanwarren\",\"active\":true,\"email\":\"kenan.warren@jupiterone.com\",\"groups\":[\"sonar-users\"],\"tokensCount\":0,\"local\":true,\"externalIdentity\":\"testUser\",\"externalProvider\":\"sonarqube\",\"avatar\":\"1894ec27fc39e5557aae3f4e2af24d45\"}]}" + }, + "cookies": [], + "headers": [ + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "cache-control", + "value": "no-cache, no-store, must-revalidate" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "content-length", + "value": "551" + }, + { + "name": "date", + "value": "Wed, 28 Apr 2021 20:52:49 GMT" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 258, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2021-04-28T20:52:49.796Z", + "time": 13, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 13 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/src/provider/types.ts b/src/provider/types.ts index ac6bca9..9305ed6 100644 --- a/src/provider/types.ts +++ b/src/provider/types.ts @@ -15,6 +15,19 @@ export interface SonarqubeUserGroup { default: boolean; } +export interface SonarqubeUser { + login: string; + name: string; + active: boolean; + email: string; + groups: string[]; + tokensCount: number; + local: boolean; + externalIdentity: string; + externalProvider: string; + avatar: string; +} + export interface Pagination { pageIndex: number; pageSize: number; diff --git a/src/steps/constants.ts b/src/steps/constants.ts index 0c75565..c7bdc8b 100644 --- a/src/steps/constants.ts +++ b/src/steps/constants.ts @@ -1,6 +1,7 @@ export const Steps = { PROJECTS: 'fetch-projects', USER_GROUPS: 'fetch-user-groups', + USERS: 'fetch-users', }; export const Entities = { @@ -14,4 +15,9 @@ export const Entities = { _type: 'sonarqube_user_group', _class: ['UserGroup'], }, + USER: { + resourceName: 'User', + _type: 'sonarqube_user', + _class: ['User'], + }, }; diff --git a/src/steps/user/__recordings__/fetchUsersShouldCollectData_357354744/recording.har b/src/steps/user/__recordings__/fetchUsersShouldCollectData_357354744/recording.har new file mode 100644 index 0000000..93364e7 --- /dev/null +++ b/src/steps/user/__recordings__/fetchUsersShouldCollectData_357354744/recording.har @@ -0,0 +1,127 @@ +{ + "log": { + "_recordingName": "fetchUsersShouldCollectData", + "creator": { + "comment": "persister:JupiterOneIntegationFSPersister", + "name": "Polly.JS", + "version": "4.3.0" + }, + "entries": [ + { + "_id": "0b4558b795c818d05f759e540e92a9ba", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "_fromType": "array", + "name": "connection", + "value": "close" + }, + { + "name": "host", + "value": "localhost:9000" + } + ], + "headersSize": 307, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [ + { + "name": "page", + "value": "1" + }, + { + "name": "per_page", + "value": "100" + } + ], + "url": "http://localhost:9000/api/users/search?page=1&per_page=100" + }, + "response": { + "bodySize": 551, + "content": { + "mimeType": "application/json", + "size": 551, + "text": "{\"paging\":{\"pageIndex\":1,\"pageSize\":50,\"total\":2},\"users\":[{\"login\":\"admin\",\"name\":\"Administrator\",\"active\":true,\"groups\":[\"sonar-administrators\",\"sonar-users\"],\"tokensCount\":3,\"local\":true,\"externalIdentity\":\"admin\",\"externalProvider\":\"sonarqube\",\"lastConnectionDate\":\"2021-04-28T20:41:19+0000\"},{\"login\":\"testUser\",\"name\":\"kenanwarren\",\"active\":true,\"email\":\"kenan.warren@jupiterone.com\",\"groups\":[\"sonar-users\"],\"tokensCount\":0,\"local\":true,\"externalIdentity\":\"testUser\",\"externalProvider\":\"sonarqube\",\"avatar\":\"1894ec27fc39e5557aae3f4e2af24d45\"}]}" + }, + "cookies": [], + "headers": [ + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "cache-control", + "value": "no-cache, no-store, must-revalidate" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "content-length", + "value": "551" + }, + { + "name": "date", + "value": "Wed, 28 Apr 2021 20:52:49 GMT" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 258, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2021-04-28T20:52:49.713Z", + "time": 43, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 43 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/src/steps/user/converter.test.ts b/src/steps/user/converter.test.ts new file mode 100644 index 0000000..8bcb4a7 --- /dev/null +++ b/src/steps/user/converter.test.ts @@ -0,0 +1,44 @@ +import { createUserEntity } from './converter'; +import { SonarqubeUser } from '../../provider/types'; + +describe('#createUserEntity', () => { + test('should convert to entity', () => { + const user = { + login: 'userlogin1', + name: 'User Name 1', + active: true, + email: 'user1@email.org', + groups: ['user-group-1'], + tokensCount: 0, + local: true, + externalIdentity: 'userlogin1', + externalProvider: 'sonarqube', + } as SonarqubeUser; + + const entity = createUserEntity(user); + + expect(entity).toEqual( + expect.objectContaining({ + _key: 'userlogin1', + _type: 'sonarqube_user', + _class: ['User'], + username: 'userlogin1', + email: 'user1@email.org', + shortLoginId: 'userlogin1', + name: 'User Name 1', + login: 'userlogin1', + active: true, + tokensCount: 0, + local: true, + externalIdentity: 'userlogin1', + externalProvider: 'sonarqube', + _rawData: [ + { + name: 'default', + rawData: user, + }, + ], + }), + ); + }); +}); diff --git a/src/steps/user/converter.ts b/src/steps/user/converter.ts new file mode 100644 index 0000000..4cf2958 --- /dev/null +++ b/src/steps/user/converter.ts @@ -0,0 +1,31 @@ +import { + createIntegrationEntity, + Entity, +} from '@jupiterone/integration-sdk-core'; + +import { Entities } from '../constants'; +import { SonarqubeUser } from '../../provider/types'; + +export function createUserEntity(user: SonarqubeUser): Entity { + return createIntegrationEntity({ + entityData: { + source: user, + assign: { + _key: user.login, + _type: Entities.USER._type, + _class: Entities.USER._class, + username: user.login, + email: user.email, + shortLoginId: user.login, + name: user.name, + login: user.login, + active: user.active, + tokensCount: user.tokensCount, + local: user.local, + externalIdentity: user.externalIdentity, + externalProvider: user.externalProvider, + avatar: user.avatar, + }, + }, + }); +} diff --git a/src/steps/user/index.test.ts b/src/steps/user/index.test.ts new file mode 100644 index 0000000..8b6d855 --- /dev/null +++ b/src/steps/user/index.test.ts @@ -0,0 +1,70 @@ +import { + createMockStepExecutionContext, + Recording, + setupRecording, +} from '@jupiterone/integration-sdk-testing'; +import { fetchUsers } from '.'; + +describe('#fetchUsers', () => { + let recording: Recording; + + afterEach(async () => { + await recording.stop(); + }); + + test('should collect data', async () => { + recording = setupRecording({ + directory: __dirname, + name: 'fetchUsersShouldCollectData', + options: { + matchRequestsBy: { + url: { + hostname: false, + }, + }, + }, + }); + + const context = createMockStepExecutionContext({ + instanceConfig: { + baseUrl: process.env.BASE_URL || 'http://localhost:9000', + apiToken: process.env.API_TOKEN || 'string-value', + }, + }); + await fetchUsers(context); + + expect(context.jobState.collectedEntities).toHaveLength(2); + expect(context.jobState.collectedRelationships).toHaveLength(0); + expect(context.jobState.collectedEntities).toEqual([ + expect.objectContaining({ + _key: expect.any(String), + _class: ['User'], + _type: 'sonarqube_user', + username: expect.any(String), + shortLoginId: expect.any(String), + name: expect.any(String), + login: expect.any(String), + active: expect.any(Boolean), + tokensCount: expect.any(Number), + local: expect.any(Boolean), + externalIdentity: expect.any(String), + externalProvider: expect.any(String), + }), + expect.objectContaining({ + _key: expect.any(String), + _class: ['User'], + _type: 'sonarqube_user', + username: expect.any(String), + email: expect.any(String), + shortLoginId: expect.any(String), + name: expect.any(String), + login: expect.any(String), + active: expect.any(Boolean), + tokensCount: expect.any(Number), + local: expect.any(Boolean), + externalIdentity: expect.any(String), + externalProvider: expect.any(String), + }), + ]); + }); +}); diff --git a/src/steps/user/index.ts b/src/steps/user/index.ts new file mode 100644 index 0000000..55d239f --- /dev/null +++ b/src/steps/user/index.ts @@ -0,0 +1,41 @@ +import { + Entity, + IntegrationStep, + IntegrationStepExecutionContext, +} from '@jupiterone/integration-sdk-core'; + +import { Entities, Steps } from '../constants'; +import { createUserEntity } from './converter'; +import { createSonarqubeClient } from '../../provider'; +import { SonarqubeUser } from '../../provider/types'; +import { SonarqubeIntegrationConfig } from '../../types'; + +export async function fetchUsers({ + instance, + jobState, +}: IntegrationStepExecutionContext) { + const client = createSonarqubeClient(instance.config); + const userKeys = new Set(); + const addUserEntity = async (user: SonarqubeUser): Promise => { + const userEntity = createUserEntity(user); + if (!userKeys.has(userEntity._key)) { + await jobState.addEntity(userEntity); + userKeys.add(userEntity._key); + } + return userEntity; + }; + + await client.iterateUsers(async (user) => { + await addUserEntity(user); + }); +} + +export const userSteps: IntegrationStep[] = [ + { + id: Steps.USERS, + name: 'Users', + entities: [Entities.USER], + executionHandler: fetchUsers, + relationships: [], + }, +]; From 1c349094c613b4de6d34b6e79577dae6100d11f2 Mon Sep 17 00:00:00 2001 From: Kenan Warren Date: Thu, 29 Apr 2021 11:15:38 -0400 Subject: [PATCH 2/7] Add pagination test for sq client and fix pagination bug --- src/provider/SonarqubeClient.test.ts | 65 ++-- src/provider/SonarqubeClient.ts | 20 +- .../recording.har | 118 ------ .../recording.har | 139 ++++++- .../recording.har | 20 +- .../recording.har | 353 ++++++++++++++++++ .../recording.har | 131 ++++++- .../recording.har | 139 ++++++- src/steps/constants.ts | 11 + .../recording.har | 139 ++++++- src/steps/project/index.test.ts | 11 +- .../recording.har | 131 ++++++- .../recording.har | 139 ++++++- 13 files changed, 1181 insertions(+), 235 deletions(-) delete mode 100644 src/provider/__recordings__/SonarqubeClientShouldFailWithInvalidToken_1469573639/recording.har rename src/provider/__recordings__/{iterateUsersShouldFailWithInvalidToken_3184218964 => iterateResourcesShouldFailWithInvalidToken_1367495729}/recording.har (85%) create mode 100644 src/provider/__recordings__/iterateResourcesShouldPaginateCorrectly_4072412313/recording.har diff --git a/src/provider/SonarqubeClient.test.ts b/src/provider/SonarqubeClient.test.ts index cfcb165..822f4a9 100644 --- a/src/provider/SonarqubeClient.test.ts +++ b/src/provider/SonarqubeClient.test.ts @@ -9,7 +9,7 @@ import { import { createSonarqubeClient } from '.'; import { SonarqubeProject, SonarqubeUserGroup, SonarqubeUser } from './types'; -describe('#SonarqubeClient', () => { +describe('#iterateResources', () => { let recording: Recording; afterEach(async () => { @@ -19,7 +19,7 @@ describe('#SonarqubeClient', () => { test('should fail with invalid token', async () => { recording = setupRecording({ directory: __dirname, - name: 'SonarqubeClientShouldFailWithInvalidToken', + name: 'iterateResourcesShouldFailWithInvalidToken', options: { matchRequestsBy: { url: { @@ -44,6 +44,38 @@ describe('#SonarqubeClient', () => { }), ).rejects.toThrowError(IntegrationProviderAuthenticationError); }); + + test('should paginate correctly', async () => { + recording = setupRecording({ + directory: __dirname, + name: 'iterateResourcesShouldPaginateCorrectly', + options: { + matchRequestsBy: { + url: { + hostname: false, + }, + }, + recordFailedRequests: true, + }, + mutateEntry: mutations.unzipGzippedRecordingEntry, + }); + + const context = createMockStepExecutionContext({ + instanceConfig: { + baseUrl: process.env.BASE_URL || 'http://localhost:9000', + apiToken: process.env.API_TOKEN || 'string-value', + }, + }); + const provider = createSonarqubeClient(context.instance.config); + const results: SonarqubeProject[] = []; + await provider.iterateProjects( + (project) => { + results.push(project); + }, + { ps: '1' }, + ); + expect(results).toHaveLength(2); + }); }); describe('#iterateProjects', () => { @@ -152,35 +184,6 @@ describe('#iterateUsers', () => { await recording.stop(); }); - test('should fail with invalid token', async () => { - recording = setupRecording({ - directory: __dirname, - name: 'iterateUsersShouldFailWithInvalidToken', - options: { - matchRequestsBy: { - url: { - hostname: false, - }, - }, - recordFailedRequests: true, - }, - mutateEntry: mutations.unzipGzippedRecordingEntry, - }); - - const context = createMockStepExecutionContext({ - instanceConfig: { - baseUrl: 'http://localhost:9000', - apiToken: 'string-value', - }, - }); - const provider = createSonarqubeClient(context.instance.config); - await expect( - provider.iterateUsers(() => { - // do nothing - }), - ).rejects.toThrowError(IntegrationProviderAuthenticationError); - }); - test('should fetch users with valid config', async () => { recording = setupRecording({ directory: __dirname, diff --git a/src/provider/SonarqubeClient.ts b/src/provider/SonarqubeClient.ts index d53b898..b659517 100644 --- a/src/provider/SonarqubeClient.ts +++ b/src/provider/SonarqubeClient.ts @@ -43,29 +43,37 @@ export class SonarqubeClient { async iterateProjects( iteratee: ResourceIteratee, + params?: NodeJS.Dict, ): Promise { return this.iterateResources<'components', SonarqubeProject>( '/projects/search', 'components', iteratee, + params, ); } async iterateUserGroups( iteratee: ResourceIteratee, + params?: NodeJS.Dict, ): Promise { return this.iterateResources<'groups', SonarqubeUserGroup>( '/user_groups/search', 'groups', iteratee, + params, ); } - async iterateUsers(iteratee: ResourceIteratee): Promise { + async iterateUsers( + iteratee: ResourceIteratee, + params?: NodeJS.Dict, + ): Promise { return this.iterateResources<'users', SonarqubeUser>( '/users/search', 'users', iteratee, + params, ); } @@ -118,13 +126,15 @@ export class SonarqubeClient { endpoint: string, iterableObjectKey: T, iteratee: ResourceIteratee, + params?: NodeJS.Dict, ): Promise { let page = 1; do { const searchParams = new URLSearchParams({ - page: String(page), - per_page: String(ITEMS_PER_PAGE), + p: String(page), + ps: String(ITEMS_PER_PAGE), + ...params, }); const parametizedEndpoint = `${endpoint}?${searchParams.toString()}`; @@ -143,9 +153,9 @@ export class SonarqubeClient { } if (result[iterableObjectKey].length) { - page = 0; // stop pagination, we've reached the end of the line - } else { page += 1; + } else { + page = 0; // stop pagination, we've reached the end of the line } } while (page); } diff --git a/src/provider/__recordings__/SonarqubeClientShouldFailWithInvalidToken_1469573639/recording.har b/src/provider/__recordings__/SonarqubeClientShouldFailWithInvalidToken_1469573639/recording.har deleted file mode 100644 index 5779699..0000000 --- a/src/provider/__recordings__/SonarqubeClientShouldFailWithInvalidToken_1469573639/recording.har +++ /dev/null @@ -1,118 +0,0 @@ -{ - "log": { - "_recordingName": "SonarqubeClientShouldFailWithInvalidToken", - "creator": { - "comment": "persister:JupiterOneIntegationFSPersister", - "name": "Polly.JS", - "version": "4.3.0" - }, - "entries": [ - { - "_id": "2545f37658d1831edd318d3143667333", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 0, - "cookies": [], - "headers": [ - { - "_fromType": "array", - "name": "authorization", - "value": "[REDACTED]" - }, - { - "_fromType": "array", - "name": "accept", - "value": "*/*" - }, - { - "_fromType": "array", - "name": "user-agent", - "value": "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" - }, - { - "_fromType": "array", - "name": "accept-encoding", - "value": "gzip,deflate" - }, - { - "_fromType": "array", - "name": "connection", - "value": "close" - }, - { - "name": "host", - "value": "localhost:9000" - } - ], - "headersSize": 274, - "httpVersion": "HTTP/1.1", - "method": "GET", - "queryString": [ - { - "name": "page", - "value": "1" - }, - { - "name": "per_page", - "value": "100" - } - ], - "url": "http://localhost:9000/api/projects/search?page=1&per_page=100" - }, - "response": { - "bodySize": 0, - "content": { - "mimeType": "text/plain", - "size": 0 - }, - "cookies": [], - "headers": [ - { - "name": "x-frame-options", - "value": "SAMEORIGIN" - }, - { - "name": "x-xss-protection", - "value": "1; mode=block" - }, - { - "name": "x-content-type-options", - "value": "nosniff" - }, - { - "name": "content-length", - "value": "0" - }, - { - "name": "date", - "value": "Thu, 29 Apr 2021 12:57:16 GMT" - }, - { - "name": "connection", - "value": "close" - } - ], - "headersSize": 172, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 401, - "statusText": "Unauthorized" - }, - "startedDateTime": "2021-04-29T12:57:16.568Z", - "time": 20, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 20 - } - } - ], - "pages": [], - "version": "1.2" - } -} diff --git a/src/provider/__recordings__/iterateProjectsShouldFetchProjectsWithValidConfig_1902452194/recording.har b/src/provider/__recordings__/iterateProjectsShouldFetchProjectsWithValidConfig_1902452194/recording.har index 32d4e7a..8bc7c34 100644 --- a/src/provider/__recordings__/iterateProjectsShouldFetchProjectsWithValidConfig_1902452194/recording.har +++ b/src/provider/__recordings__/iterateProjectsShouldFetchProjectsWithValidConfig_1902452194/recording.har @@ -8,7 +8,7 @@ }, "entries": [ { - "_id": "2545f37658d1831edd318d3143667333", + "_id": "afa2e0ad675a0d044ee717045008a6eb", "_order": 0, "cache": {}, "request": { @@ -45,27 +45,27 @@ "value": "localhost:9000" } ], - "headersSize": 310, + "headersSize": 301, "httpVersion": "HTTP/1.1", "method": "GET", "queryString": [ { - "name": "page", + "name": "p", "value": "1" }, { - "name": "per_page", + "name": "ps", "value": "100" } ], - "url": "http://localhost:9000/api/projects/search?page=1&per_page=100" + "url": "http://localhost:9000/api/projects/search?p=1&ps=100" }, "response": { - "bodySize": 280, + "bodySize": 365, "content": { "mimeType": "application/json", - "size": 280, - "text": "{\"paging\":{\"pageIndex\":1,\"pageSize\":100,\"total\":1},\"components\":[{\"key\":\"escribirio_integration-test-repo\",\"name\":\"integration-test-repo\",\"qualifier\":\"TRK\",\"visibility\":\"public\",\"lastAnalysisDate\":\"2021-04-28T12:19:58+0000\",\"revision\":\"85444305d6f8efa62f3999b09f8a9a85d0fff02d\"}]}" + "size": 365, + "text": "{\"paging\":{\"pageIndex\":1,\"pageSize\":100,\"total\":2},\"components\":[{\"key\":\"escribirio_integration-test-repo\",\"name\":\"integration-test-repo\",\"qualifier\":\"TRK\",\"visibility\":\"public\",\"lastAnalysisDate\":\"2021-04-28T12:19:58+0000\",\"revision\":\"85444305d6f8efa62f3999b09f8a9a85d0fff02d\"},{\"key\":\"test-project\",\"name\":\"test-project\",\"qualifier\":\"TRK\",\"visibility\":\"public\"}]}" }, "cookies": [], "headers": [ @@ -91,11 +91,11 @@ }, { "name": "content-length", - "value": "280" + "value": "365" }, { "name": "date", - "value": "Wed, 28 Apr 2021 13:52:10 GMT" + "value": "Thu, 29 Apr 2021 14:56:24 GMT" }, { "name": "connection", @@ -108,8 +108,8 @@ "status": 200, "statusText": "OK" }, - "startedDateTime": "2021-04-28T13:52:10.283Z", - "time": 13, + "startedDateTime": "2021-04-29T14:56:24.675Z", + "time": 10, "timings": { "blocked": -1, "connect": -1, @@ -117,7 +117,120 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 13 + "wait": 10 + } + }, + { + "_id": "0928a6c3731bd65341e5c488d2b59709", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "_fromType": "array", + "name": "connection", + "value": "close" + }, + { + "name": "host", + "value": "localhost:9000" + } + ], + "headersSize": 301, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [ + { + "name": "p", + "value": "2" + }, + { + "name": "ps", + "value": "100" + } + ], + "url": "http://localhost:9000/api/projects/search?p=2&ps=100" + }, + "response": { + "bodySize": 67, + "content": { + "mimeType": "application/json", + "size": 67, + "text": "{\"paging\":{\"pageIndex\":2,\"pageSize\":100,\"total\":2},\"components\":[]}" + }, + "cookies": [], + "headers": [ + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "cache-control", + "value": "no-cache, no-store, must-revalidate" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "content-length", + "value": "67" + }, + { + "name": "date", + "value": "Thu, 29 Apr 2021 14:59:53 GMT" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 257, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2021-04-29T14:59:53.626Z", + "time": 8, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 8 } } ], diff --git a/src/provider/__recordings__/iterateUsersShouldFailWithInvalidToken_3184218964/recording.har b/src/provider/__recordings__/iterateResourcesShouldFailWithInvalidToken_1367495729/recording.har similarity index 85% rename from src/provider/__recordings__/iterateUsersShouldFailWithInvalidToken_3184218964/recording.har rename to src/provider/__recordings__/iterateResourcesShouldFailWithInvalidToken_1367495729/recording.har index f31d6b1..d67be9a 100644 --- a/src/provider/__recordings__/iterateUsersShouldFailWithInvalidToken_3184218964/recording.har +++ b/src/provider/__recordings__/iterateResourcesShouldFailWithInvalidToken_1367495729/recording.har @@ -1,6 +1,6 @@ { "log": { - "_recordingName": "iterateUsersShouldFailWithInvalidToken", + "_recordingName": "iterateResourcesShouldFailWithInvalidToken", "creator": { "comment": "persister:JupiterOneIntegationFSPersister", "name": "Polly.JS", @@ -8,7 +8,7 @@ }, "entries": [ { - "_id": "0b4558b795c818d05f759e540e92a9ba", + "_id": "afa2e0ad675a0d044ee717045008a6eb", "_order": 0, "cache": {}, "request": { @@ -45,20 +45,20 @@ "value": "localhost:9000" } ], - "headersSize": 271, + "headersSize": 265, "httpVersion": "HTTP/1.1", "method": "GET", "queryString": [ { - "name": "page", + "name": "p", "value": "1" }, { - "name": "per_page", + "name": "ps", "value": "100" } ], - "url": "http://localhost:9000/api/users/search?page=1&per_page=100" + "url": "http://localhost:9000/api/projects/search?p=1&ps=100" }, "response": { "bodySize": 0, @@ -86,7 +86,7 @@ }, { "name": "date", - "value": "Wed, 28 Apr 2021 20:52:49 GMT" + "value": "Thu, 29 Apr 2021 14:56:24 GMT" }, { "name": "connection", @@ -99,8 +99,8 @@ "status": 401, "statusText": "Unauthorized" }, - "startedDateTime": "2021-04-28T20:52:49.778Z", - "time": 13, + "startedDateTime": "2021-04-29T14:56:24.627Z", + "time": 17, "timings": { "blocked": -1, "connect": -1, @@ -108,7 +108,7 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 13 + "wait": 17 } } ], diff --git a/src/provider/__recordings__/iterateResourcesShouldPaginateCorrectly_4072412313/recording.har b/src/provider/__recordings__/iterateResourcesShouldPaginateCorrectly_4072412313/recording.har new file mode 100644 index 0000000..cf1e7dc --- /dev/null +++ b/src/provider/__recordings__/iterateResourcesShouldPaginateCorrectly_4072412313/recording.har @@ -0,0 +1,353 @@ +{ + "log": { + "_recordingName": "iterateResourcesShouldPaginateCorrectly", + "creator": { + "comment": "persister:JupiterOneIntegationFSPersister", + "name": "Polly.JS", + "version": "4.3.0" + }, + "entries": [ + { + "_id": "0c77ec9c721c0cf9c20425105be6df9d", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "_fromType": "array", + "name": "connection", + "value": "close" + }, + { + "name": "host", + "value": "localhost:9000" + } + ], + "headersSize": 299, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [ + { + "name": "p", + "value": "1" + }, + { + "name": "ps", + "value": "1" + } + ], + "url": "http://localhost:9000/api/projects/search?p=1&ps=1" + }, + "response": { + "bodySize": 278, + "content": { + "mimeType": "application/json", + "size": 278, + "text": "{\"paging\":{\"pageIndex\":1,\"pageSize\":1,\"total\":2},\"components\":[{\"key\":\"escribirio_integration-test-repo\",\"name\":\"integration-test-repo\",\"qualifier\":\"TRK\",\"visibility\":\"public\",\"lastAnalysisDate\":\"2021-04-28T12:19:58+0000\",\"revision\":\"85444305d6f8efa62f3999b09f8a9a85d0fff02d\"}]}" + }, + "cookies": [], + "headers": [ + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "cache-control", + "value": "no-cache, no-store, must-revalidate" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "content-length", + "value": "278" + }, + { + "name": "date", + "value": "Thu, 29 Apr 2021 14:56:24 GMT" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 258, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2021-04-29T14:56:24.657Z", + "time": 10, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 10 + } + }, + { + "_id": "47c101193d68fe033414982c610742af", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "_fromType": "array", + "name": "connection", + "value": "close" + }, + { + "name": "host", + "value": "localhost:9000" + } + ], + "headersSize": 299, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [ + { + "name": "p", + "value": "2" + }, + { + "name": "ps", + "value": "1" + } + ], + "url": "http://localhost:9000/api/projects/search?p=2&ps=1" + }, + "response": { + "bodySize": 149, + "content": { + "mimeType": "application/json", + "size": 149, + "text": "{\"paging\":{\"pageIndex\":2,\"pageSize\":1,\"total\":2},\"components\":[{\"key\":\"test-project\",\"name\":\"test-project\",\"qualifier\":\"TRK\",\"visibility\":\"public\"}]}" + }, + "cookies": [], + "headers": [ + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "cache-control", + "value": "no-cache, no-store, must-revalidate" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "content-length", + "value": "149" + }, + { + "name": "date", + "value": "Thu, 29 Apr 2021 14:59:53 GMT" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 258, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2021-04-29T14:59:53.573Z", + "time": 26, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 26 + } + }, + { + "_id": "e837646f26221ea881480e5f37eca45d", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "_fromType": "array", + "name": "connection", + "value": "close" + }, + { + "name": "host", + "value": "localhost:9000" + } + ], + "headersSize": 299, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [ + { + "name": "p", + "value": "3" + }, + { + "name": "ps", + "value": "1" + } + ], + "url": "http://localhost:9000/api/projects/search?p=3&ps=1" + }, + "response": { + "bodySize": 65, + "content": { + "mimeType": "application/json", + "size": 65, + "text": "{\"paging\":{\"pageIndex\":3,\"pageSize\":1,\"total\":2},\"components\":[]}" + }, + "cookies": [], + "headers": [ + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "cache-control", + "value": "no-cache, no-store, must-revalidate" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "content-length", + "value": "65" + }, + { + "name": "date", + "value": "Thu, 29 Apr 2021 14:59:53 GMT" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 257, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2021-04-29T14:59:53.602Z", + "time": 14, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 14 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/src/provider/__recordings__/iterateUserGroupsShouldFetchUserGroupsWithValidConfig_1034298020/recording.har b/src/provider/__recordings__/iterateUserGroupsShouldFetchUserGroupsWithValidConfig_1034298020/recording.har index 6f0367a..97e6c3c 100644 --- a/src/provider/__recordings__/iterateUserGroupsShouldFetchUserGroupsWithValidConfig_1034298020/recording.har +++ b/src/provider/__recordings__/iterateUserGroupsShouldFetchUserGroupsWithValidConfig_1034298020/recording.har @@ -8,7 +8,7 @@ }, "entries": [ { - "_id": "8a49b5d504f234197e1cd616efb83230", + "_id": "4c5fe1f36c23db0a0869cc48e9fb22c0", "_order": 0, "cache": {}, "request": { @@ -45,20 +45,20 @@ "value": "localhost:9000" } ], - "headersSize": 313, + "headersSize": 304, "httpVersion": "HTTP/1.1", "method": "GET", "queryString": [ { - "name": "page", + "name": "p", "value": "1" }, { - "name": "per_page", + "name": "ps", "value": "100" } ], - "url": "http://localhost:9000/api/user_groups/search?page=1&per_page=100" + "url": "http://localhost:9000/api/user_groups/search?p=1&ps=100" }, "response": { "bodySize": 349, @@ -95,7 +95,7 @@ }, { "name": "date", - "value": "Wed, 28 Apr 2021 16:41:24 GMT" + "value": "Thu, 29 Apr 2021 14:56:24 GMT" }, { "name": "connection", @@ -108,8 +108,8 @@ "status": 200, "statusText": "OK" }, - "startedDateTime": "2021-04-28T16:41:24.511Z", - "time": 19, + "startedDateTime": "2021-04-29T14:56:24.690Z", + "time": 9, "timings": { "blocked": -1, "connect": -1, @@ -117,7 +117,120 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 19 + "wait": 9 + } + }, + { + "_id": "275a1f10e4acd99c9d73a5b7ea3729c9", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "_fromType": "array", + "name": "connection", + "value": "close" + }, + { + "name": "host", + "value": "localhost:9000" + } + ], + "headersSize": 304, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [ + { + "name": "p", + "value": "2" + }, + { + "name": "ps", + "value": "100" + } + ], + "url": "http://localhost:9000/api/user_groups/search?p=2&ps=100" + }, + "response": { + "bodySize": 63, + "content": { + "mimeType": "application/json", + "size": 63, + "text": "{\"paging\":{\"pageIndex\":2,\"pageSize\":100,\"total\":2},\"groups\":[]}" + }, + "cookies": [], + "headers": [ + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "cache-control", + "value": "no-cache, no-store, must-revalidate" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "content-length", + "value": "63" + }, + { + "name": "date", + "value": "Thu, 29 Apr 2021 14:59:53 GMT" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 257, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2021-04-29T14:59:53.641Z", + "time": 10, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 10 } } ], diff --git a/src/provider/__recordings__/iterateUsersShouldFetchUsersWithValidConfig_1014868188/recording.har b/src/provider/__recordings__/iterateUsersShouldFetchUsersWithValidConfig_1014868188/recording.har index 061c354..9bea459 100644 --- a/src/provider/__recordings__/iterateUsersShouldFetchUsersWithValidConfig_1014868188/recording.har +++ b/src/provider/__recordings__/iterateUsersShouldFetchUsersWithValidConfig_1014868188/recording.har @@ -8,7 +8,7 @@ }, "entries": [ { - "_id": "0b4558b795c818d05f759e540e92a9ba", + "_id": "defa6e4768bece4bd25a0a746586eedb", "_order": 0, "cache": {}, "request": { @@ -45,27 +45,27 @@ "value": "localhost:9000" } ], - "headersSize": 307, + "headersSize": 298, "httpVersion": "HTTP/1.1", "method": "GET", "queryString": [ { - "name": "page", + "name": "p", "value": "1" }, { - "name": "per_page", + "name": "ps", "value": "100" } ], - "url": "http://localhost:9000/api/users/search?page=1&per_page=100" + "url": "http://localhost:9000/api/users/search?p=1&ps=100" }, "response": { - "bodySize": 551, + "bodySize": 552, "content": { "mimeType": "application/json", - "size": 551, - "text": "{\"paging\":{\"pageIndex\":1,\"pageSize\":50,\"total\":2},\"users\":[{\"login\":\"admin\",\"name\":\"Administrator\",\"active\":true,\"groups\":[\"sonar-administrators\",\"sonar-users\"],\"tokensCount\":3,\"local\":true,\"externalIdentity\":\"admin\",\"externalProvider\":\"sonarqube\",\"lastConnectionDate\":\"2021-04-28T20:41:19+0000\"},{\"login\":\"testUser\",\"name\":\"kenanwarren\",\"active\":true,\"email\":\"kenan.warren@jupiterone.com\",\"groups\":[\"sonar-users\"],\"tokensCount\":0,\"local\":true,\"externalIdentity\":\"testUser\",\"externalProvider\":\"sonarqube\",\"avatar\":\"1894ec27fc39e5557aae3f4e2af24d45\"}]}" + "size": 552, + "text": "{\"paging\":{\"pageIndex\":1,\"pageSize\":100,\"total\":2},\"users\":[{\"login\":\"admin\",\"name\":\"Administrator\",\"active\":true,\"groups\":[\"sonar-administrators\",\"sonar-users\"],\"tokensCount\":3,\"local\":true,\"externalIdentity\":\"admin\",\"externalProvider\":\"sonarqube\",\"lastConnectionDate\":\"2021-04-29T14:36:46+0000\"},{\"login\":\"testUser\",\"name\":\"kenanwarren\",\"active\":true,\"email\":\"kenan.warren@jupiterone.com\",\"groups\":[\"sonar-users\"],\"tokensCount\":0,\"local\":true,\"externalIdentity\":\"testUser\",\"externalProvider\":\"sonarqube\",\"avatar\":\"1894ec27fc39e5557aae3f4e2af24d45\"}]}" }, "cookies": [], "headers": [ @@ -91,11 +91,11 @@ }, { "name": "content-length", - "value": "551" + "value": "552" }, { "name": "date", - "value": "Wed, 28 Apr 2021 20:52:49 GMT" + "value": "Thu, 29 Apr 2021 14:56:24 GMT" }, { "name": "connection", @@ -108,8 +108,8 @@ "status": 200, "statusText": "OK" }, - "startedDateTime": "2021-04-28T20:52:49.796Z", - "time": 13, + "startedDateTime": "2021-04-29T14:56:24.703Z", + "time": 18, "timings": { "blocked": -1, "connect": -1, @@ -117,7 +117,120 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 13 + "wait": 18 + } + }, + { + "_id": "f6e1abd1107fea9c5496446667ca0ff3", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "_fromType": "array", + "name": "connection", + "value": "close" + }, + { + "name": "host", + "value": "localhost:9000" + } + ], + "headersSize": 298, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [ + { + "name": "p", + "value": "2" + }, + { + "name": "ps", + "value": "100" + } + ], + "url": "http://localhost:9000/api/users/search?p=2&ps=100" + }, + "response": { + "bodySize": 62, + "content": { + "mimeType": "application/json", + "size": 62, + "text": "{\"paging\":{\"pageIndex\":2,\"pageSize\":100,\"total\":2},\"users\":[]}" + }, + "cookies": [], + "headers": [ + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "cache-control", + "value": "no-cache, no-store, must-revalidate" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "content-length", + "value": "62" + }, + { + "name": "date", + "value": "Thu, 29 Apr 2021 14:59:53 GMT" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 257, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2021-04-29T14:59:53.657Z", + "time": 11, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 11 } } ], diff --git a/src/steps/constants.ts b/src/steps/constants.ts index c7bdc8b..b0db81d 100644 --- a/src/steps/constants.ts +++ b/src/steps/constants.ts @@ -1,3 +1,5 @@ +import { RelationshipClass } from '@jupiterone/integration-sdk-core'; + export const Steps = { PROJECTS: 'fetch-projects', USER_GROUPS: 'fetch-user-groups', @@ -21,3 +23,12 @@ export const Entities = { _class: ['User'], }, }; + +export const Relationships = { + GROUP_HAS_USER: { + _type: 'sonarqube_user_group_has_user', + sourceType: Entities.USER_GROUP._type, + _class: RelationshipClass.HAS, + targetType: Entities.USER._type, + }, +}; diff --git a/src/steps/project/__recordings__/fetchProjectsShouldCollectData_2137769016/recording.har b/src/steps/project/__recordings__/fetchProjectsShouldCollectData_2137769016/recording.har index 3b76ce7..af67d49 100644 --- a/src/steps/project/__recordings__/fetchProjectsShouldCollectData_2137769016/recording.har +++ b/src/steps/project/__recordings__/fetchProjectsShouldCollectData_2137769016/recording.har @@ -8,7 +8,7 @@ }, "entries": [ { - "_id": "2545f37658d1831edd318d3143667333", + "_id": "afa2e0ad675a0d044ee717045008a6eb", "_order": 0, "cache": {}, "request": { @@ -45,27 +45,27 @@ "value": "localhost:9000" } ], - "headersSize": 310, + "headersSize": 301, "httpVersion": "HTTP/1.1", "method": "GET", "queryString": [ { - "name": "page", + "name": "p", "value": "1" }, { - "name": "per_page", + "name": "ps", "value": "100" } ], - "url": "http://localhost:9000/api/projects/search?page=1&per_page=100" + "url": "http://localhost:9000/api/projects/search?p=1&ps=100" }, "response": { - "bodySize": 280, + "bodySize": 365, "content": { "mimeType": "application/json", - "size": 280, - "text": "{\"paging\":{\"pageIndex\":1,\"pageSize\":100,\"total\":1},\"components\":[{\"key\":\"escribirio_integration-test-repo\",\"name\":\"integration-test-repo\",\"qualifier\":\"TRK\",\"visibility\":\"public\",\"lastAnalysisDate\":\"2021-04-28T12:19:58+0000\",\"revision\":\"85444305d6f8efa62f3999b09f8a9a85d0fff02d\"}]}" + "size": 365, + "text": "{\"paging\":{\"pageIndex\":1,\"pageSize\":100,\"total\":2},\"components\":[{\"key\":\"escribirio_integration-test-repo\",\"name\":\"integration-test-repo\",\"qualifier\":\"TRK\",\"visibility\":\"public\",\"lastAnalysisDate\":\"2021-04-28T12:19:58+0000\",\"revision\":\"85444305d6f8efa62f3999b09f8a9a85d0fff02d\"},{\"key\":\"test-project\",\"name\":\"test-project\",\"qualifier\":\"TRK\",\"visibility\":\"public\"}]}" }, "cookies": [], "headers": [ @@ -91,11 +91,11 @@ }, { "name": "content-length", - "value": "280" + "value": "365" }, { "name": "date", - "value": "Wed, 28 Apr 2021 13:52:10 GMT" + "value": "Thu, 29 Apr 2021 14:56:24 GMT" }, { "name": "connection", @@ -108,8 +108,8 @@ "status": 200, "statusText": "OK" }, - "startedDateTime": "2021-04-28T13:52:10.243Z", - "time": 26, + "startedDateTime": "2021-04-29T14:56:24.600Z", + "time": 28, "timings": { "blocked": -1, "connect": -1, @@ -117,7 +117,120 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 26 + "wait": 28 + } + }, + { + "_id": "0928a6c3731bd65341e5c488d2b59709", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "_fromType": "array", + "name": "connection", + "value": "close" + }, + { + "name": "host", + "value": "localhost:9000" + } + ], + "headersSize": 301, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [ + { + "name": "p", + "value": "2" + }, + { + "name": "ps", + "value": "100" + } + ], + "url": "http://localhost:9000/api/projects/search?p=2&ps=100" + }, + "response": { + "bodySize": 67, + "content": { + "mimeType": "application/json", + "size": 67, + "text": "{\"paging\":{\"pageIndex\":2,\"pageSize\":100,\"total\":2},\"components\":[]}" + }, + "cookies": [], + "headers": [ + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "cache-control", + "value": "no-cache, no-store, must-revalidate" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "content-length", + "value": "67" + }, + { + "name": "date", + "value": "Thu, 29 Apr 2021 15:00:35 GMT" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 257, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2021-04-29T15:00:35.190Z", + "time": 23, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 23 } } ], diff --git a/src/steps/project/index.test.ts b/src/steps/project/index.test.ts index 67f1935..05be20f 100644 --- a/src/steps/project/index.test.ts +++ b/src/steps/project/index.test.ts @@ -33,7 +33,7 @@ describe('#fetchProjects', () => { }); await fetchProjects(context); - expect(context.jobState.collectedEntities).toHaveLength(1); + expect(context.jobState.collectedEntities).toHaveLength(2); expect(context.jobState.collectedRelationships).toHaveLength(0); expect(context.jobState.collectedEntities).toEqual([ expect.objectContaining({ @@ -47,6 +47,15 @@ describe('#fetchProjects', () => { visibility: expect.any(String), lastAnalysisDate: expect.any(String), }), + expect.objectContaining({ + _key: expect.any(String), + _class: ['Project'], + _type: 'sonarqube_project', + key: expect.any(String), + name: expect.any(String), + qualifier: expect.any(String), + visibility: expect.any(String), + }), ]); }); }); diff --git a/src/steps/user-group/__recordings__/fetchUserGroupsShouldCollectData_1142389439/recording.har b/src/steps/user-group/__recordings__/fetchUserGroupsShouldCollectData_1142389439/recording.har index 292b754..99ce240 100644 --- a/src/steps/user-group/__recordings__/fetchUserGroupsShouldCollectData_1142389439/recording.har +++ b/src/steps/user-group/__recordings__/fetchUserGroupsShouldCollectData_1142389439/recording.har @@ -8,7 +8,7 @@ }, "entries": [ { - "_id": "8a49b5d504f234197e1cd616efb83230", + "_id": "4c5fe1f36c23db0a0869cc48e9fb22c0", "_order": 0, "cache": {}, "request": { @@ -45,20 +45,20 @@ "value": "localhost:9000" } ], - "headersSize": 313, + "headersSize": 304, "httpVersion": "HTTP/1.1", "method": "GET", "queryString": [ { - "name": "page", + "name": "p", "value": "1" }, { - "name": "per_page", + "name": "ps", "value": "100" } ], - "url": "http://localhost:9000/api/user_groups/search?page=1&per_page=100" + "url": "http://localhost:9000/api/user_groups/search?p=1&ps=100" }, "response": { "bodySize": 349, @@ -95,7 +95,7 @@ }, { "name": "date", - "value": "Wed, 28 Apr 2021 16:41:24 GMT" + "value": "Thu, 29 Apr 2021 14:56:24 GMT" }, { "name": "connection", @@ -108,8 +108,8 @@ "status": 200, "statusText": "OK" }, - "startedDateTime": "2021-04-28T16:41:24.487Z", - "time": 38, + "startedDateTime": "2021-04-29T14:56:24.600Z", + "time": 34, "timings": { "blocked": -1, "connect": -1, @@ -117,7 +117,120 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 38 + "wait": 34 + } + }, + { + "_id": "275a1f10e4acd99c9d73a5b7ea3729c9", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "_fromType": "array", + "name": "connection", + "value": "close" + }, + { + "name": "host", + "value": "localhost:9000" + } + ], + "headersSize": 304, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [ + { + "name": "p", + "value": "2" + }, + { + "name": "ps", + "value": "100" + } + ], + "url": "http://localhost:9000/api/user_groups/search?p=2&ps=100" + }, + "response": { + "bodySize": 63, + "content": { + "mimeType": "application/json", + "size": 63, + "text": "{\"paging\":{\"pageIndex\":2,\"pageSize\":100,\"total\":2},\"groups\":[]}" + }, + "cookies": [], + "headers": [ + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "cache-control", + "value": "no-cache, no-store, must-revalidate" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "content-length", + "value": "63" + }, + { + "name": "date", + "value": "Thu, 29 Apr 2021 15:00:35 GMT" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 257, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2021-04-29T15:00:35.190Z", + "time": 23, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 23 } } ], diff --git a/src/steps/user/__recordings__/fetchUsersShouldCollectData_357354744/recording.har b/src/steps/user/__recordings__/fetchUsersShouldCollectData_357354744/recording.har index 93364e7..c09f736 100644 --- a/src/steps/user/__recordings__/fetchUsersShouldCollectData_357354744/recording.har +++ b/src/steps/user/__recordings__/fetchUsersShouldCollectData_357354744/recording.har @@ -8,7 +8,7 @@ }, "entries": [ { - "_id": "0b4558b795c818d05f759e540e92a9ba", + "_id": "defa6e4768bece4bd25a0a746586eedb", "_order": 0, "cache": {}, "request": { @@ -45,27 +45,27 @@ "value": "localhost:9000" } ], - "headersSize": 307, + "headersSize": 298, "httpVersion": "HTTP/1.1", "method": "GET", "queryString": [ { - "name": "page", + "name": "p", "value": "1" }, { - "name": "per_page", + "name": "ps", "value": "100" } ], - "url": "http://localhost:9000/api/users/search?page=1&per_page=100" + "url": "http://localhost:9000/api/users/search?p=1&ps=100" }, "response": { - "bodySize": 551, + "bodySize": 552, "content": { "mimeType": "application/json", - "size": 551, - "text": "{\"paging\":{\"pageIndex\":1,\"pageSize\":50,\"total\":2},\"users\":[{\"login\":\"admin\",\"name\":\"Administrator\",\"active\":true,\"groups\":[\"sonar-administrators\",\"sonar-users\"],\"tokensCount\":3,\"local\":true,\"externalIdentity\":\"admin\",\"externalProvider\":\"sonarqube\",\"lastConnectionDate\":\"2021-04-28T20:41:19+0000\"},{\"login\":\"testUser\",\"name\":\"kenanwarren\",\"active\":true,\"email\":\"kenan.warren@jupiterone.com\",\"groups\":[\"sonar-users\"],\"tokensCount\":0,\"local\":true,\"externalIdentity\":\"testUser\",\"externalProvider\":\"sonarqube\",\"avatar\":\"1894ec27fc39e5557aae3f4e2af24d45\"}]}" + "size": 552, + "text": "{\"paging\":{\"pageIndex\":1,\"pageSize\":100,\"total\":2},\"users\":[{\"login\":\"admin\",\"name\":\"Administrator\",\"active\":true,\"groups\":[\"sonar-administrators\",\"sonar-users\"],\"tokensCount\":3,\"local\":true,\"externalIdentity\":\"admin\",\"externalProvider\":\"sonarqube\",\"lastConnectionDate\":\"2021-04-29T14:36:46+0000\"},{\"login\":\"testUser\",\"name\":\"kenanwarren\",\"active\":true,\"email\":\"kenan.warren@jupiterone.com\",\"groups\":[\"sonar-users\"],\"tokensCount\":0,\"local\":true,\"externalIdentity\":\"testUser\",\"externalProvider\":\"sonarqube\",\"avatar\":\"1894ec27fc39e5557aae3f4e2af24d45\"}]}" }, "cookies": [], "headers": [ @@ -91,11 +91,11 @@ }, { "name": "content-length", - "value": "551" + "value": "552" }, { "name": "date", - "value": "Wed, 28 Apr 2021 20:52:49 GMT" + "value": "Thu, 29 Apr 2021 14:56:24 GMT" }, { "name": "connection", @@ -108,8 +108,8 @@ "status": 200, "statusText": "OK" }, - "startedDateTime": "2021-04-28T20:52:49.713Z", - "time": 43, + "startedDateTime": "2021-04-29T14:56:24.601Z", + "time": 32, "timings": { "blocked": -1, "connect": -1, @@ -117,7 +117,120 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 43 + "wait": 32 + } + }, + { + "_id": "f6e1abd1107fea9c5496446667ca0ff3", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "authorization", + "value": "[REDACTED]" + }, + { + "_fromType": "array", + "name": "accept", + "value": "*/*" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "_fromType": "array", + "name": "connection", + "value": "close" + }, + { + "name": "host", + "value": "localhost:9000" + } + ], + "headersSize": 298, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [ + { + "name": "p", + "value": "2" + }, + { + "name": "ps", + "value": "100" + } + ], + "url": "http://localhost:9000/api/users/search?p=2&ps=100" + }, + "response": { + "bodySize": 62, + "content": { + "mimeType": "application/json", + "size": 62, + "text": "{\"paging\":{\"pageIndex\":2,\"pageSize\":100,\"total\":2},\"users\":[]}" + }, + "cookies": [], + "headers": [ + { + "name": "x-frame-options", + "value": "SAMEORIGIN" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "cache-control", + "value": "no-cache, no-store, must-revalidate" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "content-length", + "value": "62" + }, + { + "name": "date", + "value": "Thu, 29 Apr 2021 15:00:35 GMT" + }, + { + "name": "connection", + "value": "close" + } + ], + "headersSize": 257, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2021-04-29T15:00:35.191Z", + "time": 26, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 26 } } ], From 7d4243600f58068862bafb176ff00594ddd8375c Mon Sep 17 00:00:00 2001 From: Kenan Warren Date: Thu, 29 Apr 2021 11:21:13 -0400 Subject: [PATCH 3/7] Remove deduplication given it seems impossible to have dupe keys --- src/steps/project/index.ts | 16 +++------------- src/steps/user-group/index.ts | 16 +++------------- src/steps/user/index.ts | 14 +++----------- 3 files changed, 9 insertions(+), 37 deletions(-) diff --git a/src/steps/project/index.ts b/src/steps/project/index.ts index e405b51..d5b0595 100644 --- a/src/steps/project/index.ts +++ b/src/steps/project/index.ts @@ -7,7 +7,6 @@ import { import { Entities, Steps } from '../constants'; import { createProjectEntity } from './converter'; import { createSonarqubeClient } from '../../provider'; -import { SonarqubeProject } from '../../provider/types'; import { SonarqubeIntegrationConfig } from '../../types'; export async function fetchProjects({ @@ -15,21 +14,12 @@ export async function fetchProjects({ jobState, }: IntegrationStepExecutionContext) { const client = createSonarqubeClient(instance.config); - const projectKeys = new Set(); - const addProjectEntity = async ( - project: SonarqubeProject, - ): Promise => { - const projectEntity = createProjectEntity(project); - if (!projectKeys.has(projectEntity._key)) { - await jobState.addEntity(projectEntity); - projectKeys.add(projectEntity._key); - } - return projectEntity; - }; + const convertedProjects: Entity[] = []; await client.iterateProjects(async (project) => { - await addProjectEntity(project); + convertedProjects.push(createProjectEntity(project)); }); + await jobState.addEntities(convertedProjects); } export const projectSteps: IntegrationStep[] = [ diff --git a/src/steps/user-group/index.ts b/src/steps/user-group/index.ts index 0f4ebbf..1590591 100644 --- a/src/steps/user-group/index.ts +++ b/src/steps/user-group/index.ts @@ -7,7 +7,6 @@ import { import { Entities, Steps } from '../constants'; import { createUserGroupEntity } from './converter'; import { createSonarqubeClient } from '../../provider'; -import { SonarqubeUserGroup } from '../../provider/types'; import { SonarqubeIntegrationConfig } from '../../types'; export async function fetchUserGroups({ @@ -15,21 +14,12 @@ export async function fetchUserGroups({ jobState, }: IntegrationStepExecutionContext) { const client = createSonarqubeClient(instance.config); - const userGroupKeys = new Set(); - const addUserGroupEntity = async ( - userGroup: SonarqubeUserGroup, - ): Promise => { - const userGroupEntity = createUserGroupEntity(userGroup); - if (!userGroupKeys.has(userGroupEntity._key)) { - await jobState.addEntity(userGroupEntity); - userGroupKeys.add(userGroupEntity._key); - } - return userGroupEntity; - }; + const convertedUserGroups: Entity[] = []; await client.iterateUserGroups(async (userGroup) => { - await addUserGroupEntity(userGroup); + convertedUserGroups.push(createUserGroupEntity(userGroup)); }); + await jobState.addEntities(convertedUserGroups); } export const userGroupSteps: IntegrationStep[] = [ diff --git a/src/steps/user/index.ts b/src/steps/user/index.ts index 55d239f..6d8da7b 100644 --- a/src/steps/user/index.ts +++ b/src/steps/user/index.ts @@ -7,7 +7,6 @@ import { import { Entities, Steps } from '../constants'; import { createUserEntity } from './converter'; import { createSonarqubeClient } from '../../provider'; -import { SonarqubeUser } from '../../provider/types'; import { SonarqubeIntegrationConfig } from '../../types'; export async function fetchUsers({ @@ -15,19 +14,12 @@ export async function fetchUsers({ jobState, }: IntegrationStepExecutionContext) { const client = createSonarqubeClient(instance.config); - const userKeys = new Set(); - const addUserEntity = async (user: SonarqubeUser): Promise => { - const userEntity = createUserEntity(user); - if (!userKeys.has(userEntity._key)) { - await jobState.addEntity(userEntity); - userKeys.add(userEntity._key); - } - return userEntity; - }; + const convertedUsers: Entity[] = []; await client.iterateUsers(async (user) => { - await addUserEntity(user); + convertedUsers.push(createUserEntity(user)); }); + await jobState.addEntities(convertedUsers); } export const userSteps: IntegrationStep[] = [ From 8e5d6b28596dc83db8dd752f38c544708ce2208b Mon Sep 17 00:00:00 2001 From: Kenan Warren Date: Thu, 29 Apr 2021 11:47:51 -0400 Subject: [PATCH 4/7] Convert to toMatchGraphObjectSchema test pattern --- src/steps/project/converter.test.ts | 2 +- src/steps/project/converter.ts | 7 +++- src/steps/project/index.test.ts | 43 ++++++++++---------- src/steps/user-group/converter.test.ts | 2 +- src/steps/user-group/converter.ts | 7 +++- src/steps/user-group/index.test.ts | 42 ++++++++++---------- src/steps/user/converter.test.ts | 2 +- src/steps/user/converter.ts | 7 +++- src/steps/user/index.test.ts | 54 +++++++++++--------------- test/schemas.ts | 18 --------- 10 files changed, 85 insertions(+), 99 deletions(-) delete mode 100644 test/schemas.ts diff --git a/src/steps/project/converter.test.ts b/src/steps/project/converter.test.ts index a08abc0..8e6ec06 100644 --- a/src/steps/project/converter.test.ts +++ b/src/steps/project/converter.test.ts @@ -16,7 +16,7 @@ describe('#createProjectEntity', () => { expect(entity).toEqual( expect.objectContaining({ - _key: 'project-key-1', + _key: 'sonarqube-project:project-key-1', _type: 'sonarqube_project', _class: ['Project'], key: 'project-key-1', diff --git a/src/steps/project/converter.ts b/src/steps/project/converter.ts index 32d377c..3831c57 100644 --- a/src/steps/project/converter.ts +++ b/src/steps/project/converter.ts @@ -6,12 +6,17 @@ import { import { Entities } from '../constants'; import { SonarqubeProject } from '../../provider/types'; +const PROJECT_KEY_PREFIX = 'sonarqube-project'; +export function createProjectEntityIdentifier(key: string): string { + return `${PROJECT_KEY_PREFIX}:${key}`; +} + export function createProjectEntity(project: SonarqubeProject): Entity { return createIntegrationEntity({ entityData: { source: project, assign: { - _key: project.key, + _key: createProjectEntityIdentifier(project.key), _type: Entities.PROJECT._type, _class: Entities.PROJECT._class, id: project.key, diff --git a/src/steps/project/index.test.ts b/src/steps/project/index.test.ts index 05be20f..1b25875 100644 --- a/src/steps/project/index.test.ts +++ b/src/steps/project/index.test.ts @@ -35,27 +35,26 @@ describe('#fetchProjects', () => { expect(context.jobState.collectedEntities).toHaveLength(2); expect(context.jobState.collectedRelationships).toHaveLength(0); - expect(context.jobState.collectedEntities).toEqual([ - expect.objectContaining({ - _key: expect.any(String), - _class: ['Project'], - _type: 'sonarqube_project', - key: expect.any(String), - name: expect.any(String), - qualifier: expect.any(String), - revision: expect.any(String), - visibility: expect.any(String), - lastAnalysisDate: expect.any(String), - }), - expect.objectContaining({ - _key: expect.any(String), - _class: ['Project'], - _type: 'sonarqube_project', - key: expect.any(String), - name: expect.any(String), - qualifier: expect.any(String), - visibility: expect.any(String), - }), - ]); + expect(context.jobState.collectedEntities).toMatchGraphObjectSchema({ + _class: ['Project'], + schema: { + additionalProperties: true, + properties: { + _type: { const: 'sonarqube_project' }, + _key: { type: 'string' }, + key: { type: 'string' }, + name: { type: 'string' }, + qualifier: { type: 'string' }, + revision: { type: 'string' }, + visibility: { type: 'string' }, + lastAnalysisDate: { type: 'string' }, + _rawData: { + type: 'array', + items: { type: 'object' }, + }, + }, + required: ['name'], + }, + }); }); }); diff --git a/src/steps/user-group/converter.test.ts b/src/steps/user-group/converter.test.ts index bc846c1..d513d11 100644 --- a/src/steps/user-group/converter.test.ts +++ b/src/steps/user-group/converter.test.ts @@ -15,7 +15,7 @@ describe('#createUserGroupEntity', () => { expect(entity).toEqual( expect.objectContaining({ - _key: 'user-group-id-1', + _key: 'sonarqube-user-group:user-group-id-1', _type: 'sonarqube_user_group', _class: ['UserGroup'], id: 'user-group-id-1', diff --git a/src/steps/user-group/converter.ts b/src/steps/user-group/converter.ts index eae3b0f..86cc92f 100644 --- a/src/steps/user-group/converter.ts +++ b/src/steps/user-group/converter.ts @@ -6,12 +6,17 @@ import { import { Entities } from '../constants'; import { SonarqubeUserGroup } from '../../provider/types'; +const USER_GROUP_ID_PREFIX = 'sonarqube-user-group'; +export function createUserGroupEntityIdentifier(id: string): string { + return `${USER_GROUP_ID_PREFIX}:${id}`; +} + export function createUserGroupEntity(userGroup: SonarqubeUserGroup): Entity { return createIntegrationEntity({ entityData: { source: userGroup, assign: { - _key: userGroup.id, + _key: createUserGroupEntityIdentifier(userGroup.id), _type: Entities.USER_GROUP._type, _class: Entities.USER_GROUP._class, id: userGroup.id, diff --git a/src/steps/user-group/index.test.ts b/src/steps/user-group/index.test.ts index c97c3b5..08e1255 100644 --- a/src/steps/user-group/index.test.ts +++ b/src/steps/user-group/index.test.ts @@ -35,27 +35,25 @@ describe('#fetchUserGroups', () => { expect(context.jobState.collectedEntities).toHaveLength(2); expect(context.jobState.collectedRelationships).toHaveLength(0); - expect(context.jobState.collectedEntities).toEqual([ - expect.objectContaining({ - _key: expect.any(String), - _class: ['UserGroup'], - _type: 'sonarqube_user_group', - id: expect.any(String), - name: expect.any(String), - description: expect.any(String), - membersCount: expect.any(Number), - default: expect.any(Boolean), - }), - expect.objectContaining({ - _key: expect.any(String), - _class: ['UserGroup'], - _type: 'sonarqube_user_group', - id: expect.any(String), - name: expect.any(String), - description: expect.any(String), - membersCount: expect.any(Number), - default: expect.any(Boolean), - }), - ]); + expect(context.jobState.collectedEntities).toMatchGraphObjectSchema({ + _class: ['UserGroup'], + schema: { + additionalProperties: true, + properties: { + _type: { const: 'sonarqube_user_group' }, + _key: { type: 'string' }, + id: { type: 'string' }, + name: { type: 'string' }, + description: { type: 'string' }, + membersCount: { type: 'number' }, + default: { type: 'boolean' }, + _rawData: { + type: 'array', + items: { type: 'object' }, + }, + }, + required: ['name'], + }, + }); }); }); diff --git a/src/steps/user/converter.test.ts b/src/steps/user/converter.test.ts index 8bcb4a7..4d07222 100644 --- a/src/steps/user/converter.test.ts +++ b/src/steps/user/converter.test.ts @@ -19,7 +19,7 @@ describe('#createUserEntity', () => { expect(entity).toEqual( expect.objectContaining({ - _key: 'userlogin1', + _key: 'sonarqube-user:userlogin1', _type: 'sonarqube_user', _class: ['User'], username: 'userlogin1', diff --git a/src/steps/user/converter.ts b/src/steps/user/converter.ts index 4cf2958..4b9744b 100644 --- a/src/steps/user/converter.ts +++ b/src/steps/user/converter.ts @@ -6,12 +6,17 @@ import { import { Entities } from '../constants'; import { SonarqubeUser } from '../../provider/types'; +const USER_LOGIN_PREFIX = 'sonarqube-user'; +export function createUserEntityIdentifier(login: string): string { + return `${USER_LOGIN_PREFIX}:${login}`; +} + export function createUserEntity(user: SonarqubeUser): Entity { return createIntegrationEntity({ entityData: { source: user, assign: { - _key: user.login, + _key: createUserEntityIdentifier(user.login), _type: Entities.USER._type, _class: Entities.USER._class, username: user.login, diff --git a/src/steps/user/index.test.ts b/src/steps/user/index.test.ts index 8b6d855..f5ab2fc 100644 --- a/src/steps/user/index.test.ts +++ b/src/steps/user/index.test.ts @@ -35,36 +35,28 @@ describe('#fetchUsers', () => { expect(context.jobState.collectedEntities).toHaveLength(2); expect(context.jobState.collectedRelationships).toHaveLength(0); - expect(context.jobState.collectedEntities).toEqual([ - expect.objectContaining({ - _key: expect.any(String), - _class: ['User'], - _type: 'sonarqube_user', - username: expect.any(String), - shortLoginId: expect.any(String), - name: expect.any(String), - login: expect.any(String), - active: expect.any(Boolean), - tokensCount: expect.any(Number), - local: expect.any(Boolean), - externalIdentity: expect.any(String), - externalProvider: expect.any(String), - }), - expect.objectContaining({ - _key: expect.any(String), - _class: ['User'], - _type: 'sonarqube_user', - username: expect.any(String), - email: expect.any(String), - shortLoginId: expect.any(String), - name: expect.any(String), - login: expect.any(String), - active: expect.any(Boolean), - tokensCount: expect.any(Number), - local: expect.any(Boolean), - externalIdentity: expect.any(String), - externalProvider: expect.any(String), - }), - ]); + expect(context.jobState.collectedEntities).toMatchGraphObjectSchema({ + _class: ['User'], + schema: { + additionalProperties: true, + properties: { + _type: { const: 'sonarqube_user' }, + _key: { type: 'string' }, + username: { type: 'string' }, + shortLoginId: { type: 'string' }, + name: { type: 'string' }, + active: { type: 'boolean' }, + tokensCount: { type: 'number' }, + local: { type: 'boolean' }, + externalIdentity: { type: 'string' }, + externalProvider: { type: 'string' }, + _rawData: { + type: 'array', + items: { type: 'object' }, + }, + }, + required: ['login'], + }, + }); }); }); diff --git a/test/schemas.ts b/test/schemas.ts deleted file mode 100644 index a11b94b..0000000 --- a/test/schemas.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { GraphObjectSchema } from '@jupiterone/integration-sdk-testing'; - -export const projectClass = ['Project']; -export const projectSchema: GraphObjectSchema = { - additionalProperties: false, - properties: { - _key: { type: 'string' }, - _class: { const: projectClass }, - _type: { const: 'sonarqube_project' }, - _rawData: { type: 'array', items: { type: 'object' } }, - id: { type: 'string' }, - qualifier: { type: 'string' }, - visibility: { type: 'string' }, - lastAnalysisDate: { type: 'string' }, - revision: { type: 'string' }, - displayName: { type: 'string' }, - }, -}; From 3fb8efb5c54d06cca8d439dfc79804b57eb9f72f Mon Sep 17 00:00:00 2001 From: Kenan Warren Date: Thu, 29 Apr 2021 11:56:32 -0400 Subject: [PATCH 5/7] User Group User Relationship entity --- src/steps/project/index.ts | 2 +- src/steps/user-group/index.ts | 2 +- src/steps/user/index.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/steps/project/index.ts b/src/steps/project/index.ts index d5b0595..e2df6a2 100644 --- a/src/steps/project/index.ts +++ b/src/steps/project/index.ts @@ -16,7 +16,7 @@ export async function fetchProjects({ const client = createSonarqubeClient(instance.config); const convertedProjects: Entity[] = []; - await client.iterateProjects(async (project) => { + await client.iterateProjects((project) => { convertedProjects.push(createProjectEntity(project)); }); await jobState.addEntities(convertedProjects); diff --git a/src/steps/user-group/index.ts b/src/steps/user-group/index.ts index 1590591..712c602 100644 --- a/src/steps/user-group/index.ts +++ b/src/steps/user-group/index.ts @@ -16,7 +16,7 @@ export async function fetchUserGroups({ const client = createSonarqubeClient(instance.config); const convertedUserGroups: Entity[] = []; - await client.iterateUserGroups(async (userGroup) => { + await client.iterateUserGroups((userGroup) => { convertedUserGroups.push(createUserGroupEntity(userGroup)); }); await jobState.addEntities(convertedUserGroups); diff --git a/src/steps/user/index.ts b/src/steps/user/index.ts index 6d8da7b..984c927 100644 --- a/src/steps/user/index.ts +++ b/src/steps/user/index.ts @@ -16,7 +16,7 @@ export async function fetchUsers({ const client = createSonarqubeClient(instance.config); const convertedUsers: Entity[] = []; - await client.iterateUsers(async (user) => { + await client.iterateUsers((user) => { convertedUsers.push(createUserEntity(user)); }); await jobState.addEntities(convertedUsers); From 8d76aad56de54537562fa20d86c1894757a1e264 Mon Sep 17 00:00:00 2001 From: Kenan Warren Date: Thu, 29 Apr 2021 13:04:09 -0400 Subject: [PATCH 6/7] Add user group user relationship --- docs/jupiterone.md | 9 + src/provider/SonarqubeClient.test.ts | 49 + src/provider/SonarqubeClient.ts | 13 + .../recording.har | 12 +- .../recording.har | 248 +++++ src/provider/types.ts | 3 +- src/steps/constants.ts | 1 + src/steps/index.ts | 3 +- .../recording.har | 934 ++++++++++++++++++ .../recording.har | 18 +- src/steps/user/index.test.ts | 51 +- src/steps/user/index.ts | 58 +- 12 files changed, 1379 insertions(+), 20 deletions(-) create mode 100644 src/provider/__recordings__/iterateUsersGroupsShouldFetchUserGroupsWithValidConfig_3639583757/recording.har create mode 100644 src/steps/user/__recordings__/buildUserGroupUserRelationshipsShouldCollectData_4207018598/recording.har diff --git a/docs/jupiterone.md b/docs/jupiterone.md index 909ed30..a833c6a 100644 --- a/docs/jupiterone.md +++ b/docs/jupiterone.md @@ -86,8 +86,17 @@ The following entities are created: | Resources | Entity `_type` | Entity `_class` | | --------- | ---------------------- | --------------- | | Project | `sonarqube_project` | `Project` | +| User | `sonarqube_user` | `User` | | UserGroup | `sonarqube_user_group` | `UserGroup` | +### Relationships + +The following relationships are created/mapped: + +| Source Entity `_type` | Relationship `_class` | Target Entity `_type` | +| ---------------------- | --------------------- | --------------------- | +| `sonarqube_user_group` | **HAS** | `sonarqube_user` | +