This repository has been archived by the owner on Aug 16, 2024. It is now read-only.
generated from JupiterOne-Archives/integration-template
-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #34 from Creativice-Oy/feature/findings
INT-5777 - Ingest findings and account
Showing
26 changed files
with
2,750 additions
and
2,104 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
import { createAccountEntity } from './converter'; | ||
import { createMockStepExecutionContext } from '@jupiterone/integration-sdk-testing'; | ||
|
||
describe('#createAccountEntity', () => { | ||
test('should convert to entity', () => { | ||
const context = createMockStepExecutionContext({ | ||
instanceConfig: { | ||
baseUrl: process.env.BASE_URL || 'http://localhost:9000', | ||
apiToken: process.env.API_TOKEN || 'string-value', | ||
}, | ||
}); | ||
|
||
const entity = createAccountEntity(context.instance); | ||
|
||
expect(entity).toEqual( | ||
expect.objectContaining({ | ||
_key: 'sonarqube_account', | ||
_type: 'sonarqube_account', | ||
_class: ['Account'], | ||
}), | ||
); | ||
}); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
import { | ||
createIntegrationEntity, | ||
Entity, | ||
IntegrationInstance, | ||
} from '@jupiterone/integration-sdk-core'; | ||
import { SonarqubeIntegrationConfig } from '../../types'; | ||
|
||
import { Entities } from '../constants'; | ||
|
||
export function createAccountEntity( | ||
instance: IntegrationInstance<SonarqubeIntegrationConfig>, | ||
): Entity { | ||
const { id, name } = instance; | ||
return createIntegrationEntity({ | ||
entityData: { | ||
source: { | ||
name, | ||
}, | ||
assign: { | ||
_key: 'sonarqube_account', | ||
_type: Entities.ACCOUNT._type, | ||
_class: Entities.ACCOUNT._class, | ||
id, | ||
name, | ||
}, | ||
}, | ||
}); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
import { | ||
createMockStepExecutionContext, | ||
Recording, | ||
setupRecording, | ||
} from '@jupiterone/integration-sdk-testing'; | ||
import { fetchAccount } from '.'; | ||
|
||
describe('#fetchAccount', () => { | ||
let recording: Recording; | ||
|
||
afterEach(async () => { | ||
await recording.stop(); | ||
}); | ||
|
||
test('should collect data', async () => { | ||
recording = setupRecording({ | ||
directory: __dirname, | ||
name: 'fetchProjectsShouldCollectData', | ||
options: { | ||
matchRequestsBy: { | ||
url: { | ||
hostname: false, | ||
}, | ||
}, | ||
}, | ||
}); | ||
|
||
const context = createMockStepExecutionContext({ | ||
instanceConfig: { | ||
baseUrl: process.env.BASE_URL || 'http://localhost:9000', | ||
apiToken: process.env.API_TOKEN || 'string-value', | ||
}, | ||
}); | ||
await fetchAccount(context); | ||
|
||
expect(context.jobState.collectedEntities).toHaveLength(1); | ||
expect(context.jobState.collectedRelationships).toHaveLength(0); | ||
expect(context.jobState.collectedEntities).toMatchGraphObjectSchema({ | ||
_class: ['Account'], | ||
schema: { | ||
additionalProperties: true, | ||
properties: { | ||
_type: { const: 'sonarqube_account' }, | ||
_key: { type: 'string' }, | ||
name: { type: 'string' }, | ||
id: { type: 'string' }, | ||
}, | ||
}, | ||
}); | ||
}); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
import { | ||
IntegrationStep, | ||
IntegrationStepExecutionContext, | ||
} from '@jupiterone/integration-sdk-core'; | ||
|
||
import { ACCOUNT_ENTITY_KEY, Entities, Steps } from '../constants'; | ||
import { createAccountEntity } from './converter'; | ||
import { SonarqubeIntegrationConfig } from '../../types'; | ||
|
||
export async function fetchAccount({ | ||
instance, | ||
jobState, | ||
}: IntegrationStepExecutionContext<SonarqubeIntegrationConfig>) { | ||
const accountEntity = await jobState.addEntity(createAccountEntity(instance)); | ||
await jobState.setData(ACCOUNT_ENTITY_KEY, accountEntity); | ||
} | ||
|
||
export const accountSteps: IntegrationStep<SonarqubeIntegrationConfig>[] = [ | ||
{ | ||
id: Steps.ACCOUNT, | ||
name: 'Fetch Account', | ||
entities: [Entities.ACCOUNT], | ||
executionHandler: fetchAccount, | ||
relationships: [], | ||
}, | ||
]; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
590 changes: 590 additions & 0 deletions
590
src/steps/finding/__recordings__/fetchFindingsShouldCollectData_149947136/recording.har
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,101 @@ | ||
import { createFindingEntity } from './converter'; | ||
import { SonarqubeFinding } from '../../provider/types'; | ||
|
||
describe('#createFindingEntity', () => { | ||
test('should convert to entity', () => { | ||
const project = { | ||
key: '12345', | ||
rule: 'javascript:S1234', | ||
severity: 'CRITICAL', | ||
component: '12345', | ||
project: 'Test12345hvhIyiTjXiMI4DI', | ||
line: 235, | ||
hash: '81234598a9eac8', | ||
textRange: { | ||
startLine: 235, | ||
endLine: 235, | ||
startOffset: 2, | ||
endOffset: 110, | ||
}, | ||
flows: [], | ||
status: 'OPEN', | ||
message: 'Unexpected var, use let or const instead.', | ||
effort: '5min', | ||
debt: '5min', | ||
author: 'sns@vuln.in', | ||
tags: ['bad-practice', 'es2015'], | ||
creationDate: '2008-07-31T16:52:52+0000', | ||
updateDate: '2012-12-15T02:41:31+0000', | ||
type: 'CODE_SMELL', | ||
scope: 'MAIN', | ||
quickFixAvailable: true, | ||
} as SonarqubeFinding; | ||
|
||
const entity = createFindingEntity(project); | ||
|
||
expect(entity).toEqual( | ||
expect.objectContaining({ | ||
_class: ['Finding'], | ||
_key: 'sonarqube-finding:12345', | ||
_rawData: [ | ||
{ | ||
name: 'default', | ||
rawData: { | ||
author: 'sns@vuln.in', | ||
component: '12345', | ||
creationDate: '2008-07-31T16:52:52+0000', | ||
debt: '5min', | ||
effort: '5min', | ||
flows: [], | ||
hash: '81234598a9eac8', | ||
key: '12345', | ||
line: 235, | ||
message: 'Unexpected var, use let or const instead.', | ||
project: 'Test12345hvhIyiTjXiMI4DI', | ||
quickFixAvailable: true, | ||
rule: 'javascript:S1234', | ||
scope: 'MAIN', | ||
severity: 'CRITICAL', | ||
status: 'OPEN', | ||
textRange: { | ||
endLine: 235, | ||
endOffset: 110, | ||
startLine: 235, | ||
startOffset: 2, | ||
}, | ||
type: 'CODE_SMELL', | ||
updateDate: '2012-12-15T02:41:31+0000', | ||
}, | ||
}, | ||
], | ||
_type: 'sonarqube_finding', | ||
active: undefined, | ||
author: 'sns@vuln.in', | ||
category: 'CODE_SMELL', | ||
component: '12345', | ||
createdOn: 1217523172000, | ||
debt: '5min', | ||
displayName: '81234598a9eac8', | ||
effort: '5min', | ||
hash: '81234598a9eac8', | ||
key: '12345', | ||
line: 235, | ||
message: 'Unexpected var, use let or const instead.', | ||
name: '81234598a9eac8', | ||
numericSeverity: 8, | ||
open: true, | ||
project: 'Test12345hvhIyiTjXiMI4DI', | ||
quickFixAvailable: true, | ||
rule: 'javascript:S1234', | ||
scope: 'MAIN', | ||
severity: 'critical', | ||
status: 'OPEN', | ||
textRangeEndLine: 235, | ||
textRangeEndOffset: 110, | ||
textRangeStartLine: 235, | ||
textRangeStartOffset: 2, | ||
updatedOn: 1355539291000, | ||
}), | ||
); | ||
}); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
import { | ||
createIntegrationEntity, | ||
Entity, | ||
parseTimePropertyValue, | ||
} from '@jupiterone/integration-sdk-core'; | ||
|
||
import { Entities } from '../constants'; | ||
import { SonarqubeFinding } from '../../provider/types'; | ||
|
||
const FINDING_KEY_PREFIX = 'sonarqube-finding'; | ||
export function createFindingEntityIdentifier(key: string): string { | ||
return `${FINDING_KEY_PREFIX}:${key}`; | ||
} | ||
|
||
const severityToNumericSeverity = (severity: string): number => { | ||
switch (severity) { | ||
case 'BLOCKER': | ||
return 10; | ||
case 'CRITICAL': | ||
return 8; | ||
case 'MAJOR': | ||
return 6; | ||
case 'MINOR': | ||
return 4; | ||
case 'INFO': | ||
return 2; | ||
default: | ||
return 0; | ||
} | ||
}; | ||
|
||
export function createFindingEntity(finding: SonarqubeFinding): Entity { | ||
// console.log("finding", finding) | ||
const { tags, ...rest } = finding; | ||
|
||
return createIntegrationEntity({ | ||
entityData: { | ||
source: rest, | ||
assign: { | ||
_key: createFindingEntityIdentifier(finding.key), | ||
_type: Entities.FINDING._type, | ||
_class: Entities.FINDING._class, | ||
key: finding.key, | ||
name: finding.hash, | ||
rule: finding.rule, | ||
severity: finding.severity.toLowerCase(), | ||
component: finding.component, | ||
project: finding.project, | ||
line: finding.line, | ||
hash: finding.hash, | ||
textRangeStartLine: finding.textRange.startLine, | ||
textRangeEndLine: finding.textRange.endLine, | ||
textRangeStartOffset: finding.textRange.startOffset, | ||
textRangeEndOffset: finding.textRange.endOffset, | ||
status: finding.status, | ||
message: finding.message, | ||
effort: finding.effort, | ||
debt: finding.debt, | ||
author: finding.author, | ||
createdOn: parseTimePropertyValue(finding.creationDate), | ||
updatedOn: parseTimePropertyValue(finding.updateDate), | ||
scope: finding.scope, | ||
quickFixAvailable: finding.quickFixAvailable, | ||
category: finding.type, | ||
open: finding.status === 'OPEN', | ||
numericSeverity: severityToNumericSeverity(finding.severity), | ||
}, | ||
}, | ||
}); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,68 @@ | ||
import { | ||
createMockStepExecutionContext, | ||
Recording, | ||
setupRecording, | ||
} from '@jupiterone/integration-sdk-testing'; | ||
import { fetchFindings } from '.'; | ||
import { fetchAccount } from '../account'; | ||
import { fetchProjects } from '../project'; | ||
|
||
describe('#fetchFindings', () => { | ||
let recording: Recording; | ||
|
||
afterEach(async () => { | ||
await recording.stop(); | ||
}); | ||
|
||
test('should collect data', async () => { | ||
recording = setupRecording({ | ||
directory: __dirname, | ||
name: 'fetchFindingsShouldCollectData', | ||
options: { | ||
matchRequestsBy: { | ||
url: { | ||
hostname: false, | ||
}, | ||
}, | ||
}, | ||
}); | ||
|
||
const context = createMockStepExecutionContext({ | ||
instanceConfig: { | ||
baseUrl: process.env.BASE_URL || 'http://localhost:9000', | ||
apiToken: process.env.API_TOKEN || 'string-value', | ||
}, | ||
}); | ||
|
||
await fetchAccount(context); | ||
await fetchProjects(context); | ||
await fetchFindings(context); | ||
|
||
const entities = context.jobState.collectedEntities.filter( | ||
(p) => p._type === 'sonarqube_finding', | ||
); | ||
const relationships = context.jobState.collectedRelationships.filter( | ||
(r) => r._type === 'sonarqube_project_has_finding', | ||
); | ||
|
||
expect(entities).toMatchGraphObjectSchema({ | ||
_class: ['Finding'], | ||
schema: { | ||
additionalProperties: true, | ||
properties: { | ||
_type: { const: 'sonarqube_finding' }, | ||
_key: { type: 'string' }, | ||
key: { type: 'string' }, | ||
name: { type: 'string' }, | ||
_rawData: { | ||
type: 'array', | ||
items: { type: 'object' }, | ||
}, | ||
}, | ||
}, | ||
}); | ||
|
||
expect(entities.length).toBeGreaterThan(0); | ||
expect(relationships.length).toBeGreaterThan(0); | ||
}); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
import { | ||
createDirectRelationship, | ||
getRawData, | ||
IntegrationStep, | ||
IntegrationStepExecutionContext, | ||
RelationshipClass, | ||
} from '@jupiterone/integration-sdk-core'; | ||
|
||
import { Entities, Relationships, Steps } from '../constants'; | ||
import { createSonarqubeClient } from '../../provider'; | ||
import { SonarqubeIntegrationConfig } from '../../types'; | ||
import { SonarqubeProject } from '../../provider/types'; | ||
import { createFindingEntity } from './converter'; | ||
|
||
export async function fetchFindings({ | ||
instance, | ||
jobState, | ||
logger, | ||
}: IntegrationStepExecutionContext<SonarqubeIntegrationConfig>) { | ||
const client = createSonarqubeClient(instance.config); | ||
|
||
await jobState.iterateEntities( | ||
{ _type: Entities.PROJECT._type }, | ||
async (projectEntity) => { | ||
const project = getRawData<SonarqubeProject>(projectEntity); | ||
|
||
if (!project) { | ||
logger.warn(`Can not get raw data for entity ${projectEntity._key}`); | ||
return; | ||
} | ||
|
||
await client.iterateProjectFindings( | ||
async (finding) => { | ||
const findingEntity = createFindingEntity(finding); | ||
|
||
if (!(await jobState.hasKey(findingEntity._key))) { | ||
await jobState.addEntity(findingEntity); | ||
} | ||
|
||
await jobState.addRelationship( | ||
createDirectRelationship({ | ||
_class: RelationshipClass.HAS, | ||
from: projectEntity, | ||
to: findingEntity, | ||
}), | ||
); | ||
}, | ||
{ componentKeys: project.key }, | ||
); | ||
}, | ||
); | ||
} | ||
|
||
export const findingSteps: IntegrationStep<SonarqubeIntegrationConfig>[] = [ | ||
{ | ||
id: Steps.FINDINGS, | ||
name: 'Fetch Project Findings', | ||
entities: [Entities.FINDING], | ||
executionHandler: fetchFindings, | ||
relationships: [Relationships.PROJECT_HAS_FINDING], | ||
dependsOn: [Steps.PROJECTS], | ||
}, | ||
]; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters