diff --git a/commands/documentPermissions.ts b/commands/documentPermissions.ts new file mode 100644 index 00000000..e433f65d --- /dev/null +++ b/commands/documentPermissions.ts @@ -0,0 +1,107 @@ +import { promises as fs } from 'fs'; +import * as path from 'path'; +import { Command } from 'commander'; +import { invocationConfig } from '../src'; +import { GoogleCloudIntegrationStep } from '../src/types'; + +const table = require('markdown-table'); + +const documentPermissionsCommand = new Command(); + +interface DocumentCommandArgs { + outputFile: string; +} + +const J1_PERMISSIONS_DOCUMENTATION_MARKER_START = + ''; +const J1_PERMISSIONS_DOCUMENTATION_MARKER_END = + ''; + +documentPermissionsCommand + .command('documentPermissions') + .description('Generate GCP permissions list') + .option( + '-o, --output-file ', + 'project relative path to generated Markdown file', + path.join('docs', 'jupiterone.md'), + ) + .action(executeDocumentPermissionsAction); + +documentPermissionsCommand.parse(); + +async function executeDocumentPermissionsAction(options: DocumentCommandArgs) { + const { outputFile } = options; + const documentationFilePath = path.join(process.cwd(), outputFile); + const oldDocumentationFile = await getDocumentationFile( + documentationFilePath, + ); + + const newGeneratedDocumentationSection = getNewDocumentationVersion(); + + if (!newGeneratedDocumentationSection) return; + + const newDocumentationFile = replaceBetweenDocumentMarkers( + oldDocumentationFile, + newGeneratedDocumentationSection, + ); + + await fs.writeFile(documentationFilePath, newDocumentationFile, { + encoding: 'utf-8', + }); +} + +function getDocumentationFile(documentationFilePath: string): Promise { + return fs.readFile(documentationFilePath, { + encoding: 'utf-8', + }); +} + +function getNewDocumentationVersion(): string | undefined { + const { integrationSteps } = invocationConfig; + + const permissionsList = integrationSteps.reduce( + (accumulatedPermissions, step) => { + const googleCloudIntegrationStep = step as GoogleCloudIntegrationStep; + return googleCloudIntegrationStep.permissions + ? [...accumulatedPermissions, ...googleCloudIntegrationStep.permissions] + : accumulatedPermissions; + }, + [] as string[], + ); + + const tableMarkdown = getTableMarkdown(permissionsList); + + return `${J1_PERMISSIONS_DOCUMENTATION_MARKER_START}\n${tableMarkdown}\n${J1_PERMISSIONS_DOCUMENTATION_MARKER_END}`; +} + +function getTableMarkdown(permissionsList: string[]): string { + return table([ + ['Permissions List'], + ...permissionsList.map((permission) => [`\`${permission}\``]), + ]); +} + +function replaceBetweenDocumentMarkers( + oldDocumentationFile: string, + newGeneratedDocumentationSection: string, +): string { + const startIndex = oldDocumentationFile.indexOf( + J1_PERMISSIONS_DOCUMENTATION_MARKER_START, + ); + + if (startIndex === -1) { + return `${oldDocumentationFile}\n\n${newGeneratedDocumentationSection}`; + } + + const endIndex = oldDocumentationFile.indexOf( + J1_PERMISSIONS_DOCUMENTATION_MARKER_END, + ); + + return ( + oldDocumentationFile.substring(0, startIndex) + + newGeneratedDocumentationSection + + oldDocumentationFile.substring( + endIndex + J1_PERMISSIONS_DOCUMENTATION_MARKER_END.length, + ) + ); +} diff --git a/docs/jupiterone.md b/docs/jupiterone.md index 8b23791c..90519699 100644 --- a/docs/jupiterone.md +++ b/docs/jupiterone.md @@ -563,3 +563,12 @@ END OF GENERATED DOCUMENTATION AFTER BELOW MARKER + +#### Google Cloud Specific Permissions List + +If you prefer not to use Google managed roles, the following list of specific +permissions can be used to provision only the required ones: + + + + diff --git a/package.json b/package.json index 333bbc82..897ebb68 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,8 @@ "prepush": "yarn lint && yarn type-check && jest --changedSince main", "postversion": "cp package.json ./dist/package.json", "tf": "cd terraform && env `grep -v '^#' .env` terraform $1", - "create-env-file": "yarn ts-node ./scripts/createEnvFile $1" + "create-env-file": "yarn ts-node ./scripts/createEnvFile $1", + "document:permissions": " yarn ts-node commands/documentPermissions.ts documentPermissions" }, "peerDependencies": { "@jupiterone/integration-sdk-core": "^8.24.1" @@ -55,7 +56,8 @@ "gaxios": "^4.2.1", "google-auth-library": "^7.1.0", "googleapis": "94.0.0", - "lodash.get": "^4.4.2" + "lodash.get": "^4.4.2", + "commander": "^9.4.1" }, "auto": { "plugins": [ diff --git a/src/steps/access-context-manager/index.ts b/src/steps/access-context-manager/index.ts index 16fb9825..de37aa6d 100644 --- a/src/steps/access-context-manager/index.ts +++ b/src/steps/access-context-manager/index.ts @@ -2,12 +2,14 @@ import { createDirectRelationship, createMappedRelationship, Entity, - IntegrationStep, JobState, RelationshipClass, RelationshipDirection, } from '@jupiterone/integration-sdk-core'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { AccessContextManagerClient } from './client'; import { STEP_ACCESS_CONTEXT_MANAGER_ACCESS_POLICIES, @@ -383,7 +385,7 @@ export async function fetchServicePerimeters( ); } -export const accessPoliciesSteps: IntegrationStep[] = [ +export const accessPoliciesSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_ACCESS_CONTEXT_MANAGER_ACCESS_POLICIES, name: 'Access Context Manager Access Policies', diff --git a/src/steps/api-gateway/index.ts b/src/steps/api-gateway/index.ts index f5c2d0d3..b7230e7d 100644 --- a/src/steps/api-gateway/index.ts +++ b/src/steps/api-gateway/index.ts @@ -1,10 +1,12 @@ import { createDirectRelationship, - IntegrationStep, RelationshipClass, } from '@jupiterone/integration-sdk-core'; import { apigateway_v1 } from 'googleapis'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { isMemberPublic } from '../../utils/iam'; import { ApiGatewayClient } from './client'; import { @@ -181,7 +183,7 @@ export async function fetchApiGatewayGateways( }); } -export const apiGatewaySteps: IntegrationStep[] = [ +export const apiGatewaySteps: GoogleCloudIntegrationStep[] = [ { id: STEP_API_GATEWAY_APIS, name: 'Api Gateway APIs', diff --git a/src/steps/app-engine/index.ts b/src/steps/app-engine/index.ts index 85e6ff47..c6d8155e 100644 --- a/src/steps/app-engine/index.ts +++ b/src/steps/app-engine/index.ts @@ -4,12 +4,14 @@ import { Entity, getRawData, IntegrationLogger, - IntegrationStep, RelationshipClass, RelationshipDirection, } from '@jupiterone/integration-sdk-core'; import { appengine_v1 } from 'googleapis'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { publishMissingPermissionEvent } from '../../utils/events'; import { AppEngineClient } from './client'; import { @@ -377,7 +379,7 @@ export async function fetchAppEngineVersionInstances( ); } -export const appEngineSteps: IntegrationStep[] = [ +export const appEngineSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_APP_ENGINE_APPLICATION, name: 'AppEngine Application', diff --git a/src/steps/big-query/index.ts b/src/steps/big-query/index.ts index 2051b014..ede7cbf5 100644 --- a/src/steps/big-query/index.ts +++ b/src/steps/big-query/index.ts @@ -1,12 +1,14 @@ import { createDirectRelationship, createMappedRelationship, - IntegrationStep, RelationshipClass, RelationshipDirection, } from '@jupiterone/integration-sdk-core'; import { bigquery_v2 } from 'googleapis'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { isMemberPublic } from '../../utils/iam'; import { getKmsGraphObjectKeyFromKmsKeyName } from '../../utils/kms'; import { ENTITY_TYPE_KMS_KEY, STEP_CLOUD_KMS_KEYS } from '../kms'; @@ -220,7 +222,7 @@ export async function fetchBigQueryTables( ); } -export const bigQuerySteps: IntegrationStep[] = [ +export const bigQuerySteps: GoogleCloudIntegrationStep[] = [ { id: STEP_BIG_QUERY_DATASETS, name: 'Big Query Datasets', diff --git a/src/steps/big-table/index.ts b/src/steps/big-table/index.ts index 37951cd2..443592fd 100644 --- a/src/steps/big-table/index.ts +++ b/src/steps/big-table/index.ts @@ -1,9 +1,11 @@ import { createDirectRelationship, - IntegrationStep, RelationshipClass, } from '@jupiterone/integration-sdk-core'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { getKmsGraphObjectKeyFromKmsKeyName } from '../../utils/kms'; import { ENTITY_TYPE_KMS_KEY, STEP_CLOUD_KMS_KEYS } from '../kms'; import { BigTableClient } from './client'; @@ -219,7 +221,7 @@ export async function fetchTables( ); } -export const bigTableSteps: IntegrationStep[] = [ +export const bigTableSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_BIG_TABLE_INSTANCES, name: 'Bigtable Instances', diff --git a/src/steps/billing-budgets/index.ts b/src/steps/billing-budgets/index.ts index 877e0aae..df68a0ce 100644 --- a/src/steps/billing-budgets/index.ts +++ b/src/steps/billing-budgets/index.ts @@ -1,11 +1,13 @@ import { createDirectRelationship, createMappedRelationship, - IntegrationStep, RelationshipClass, RelationshipDirection, } from '@jupiterone/integration-sdk-core'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { PROJECT_ENTITY_TYPE, STEP_RESOURCE_MANAGER_ORGANIZATION, @@ -199,7 +201,7 @@ export async function buildAdditionalProjectBudgetRelationships( ); } -export const billingBudgetsSteps: IntegrationStep[] = [ +export const billingBudgetsSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_BILLING_BUDGETS, name: 'Billing Budgets', diff --git a/src/steps/binary-authorization/index.ts b/src/steps/binary-authorization/index.ts index ae6fceec..dadabc67 100644 --- a/src/steps/binary-authorization/index.ts +++ b/src/steps/binary-authorization/index.ts @@ -1,10 +1,12 @@ import { createDirectRelationship, - IntegrationStep, RelationshipClass, } from '@jupiterone/integration-sdk-core'; import { binaryauthorization_v1 } from 'googleapis'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { PROJECT_ENTITY_TYPE, STEP_RESOURCE_MANAGER_PROJECT, @@ -72,7 +74,7 @@ export async function fetchBinaryAuthorizationPolicy( } } -export const binaryAuthorizationSteps: IntegrationStep[] = [ +export const binaryAuthorizationSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_BINARY_AUTHORIZATION_POLICY, name: 'Binary Authorization Policy', diff --git a/src/steps/cloud-asset/index.ts b/src/steps/cloud-asset/index.ts index 5949dfea..decab38c 100644 --- a/src/steps/cloud-asset/index.ts +++ b/src/steps/cloud-asset/index.ts @@ -7,7 +7,6 @@ import { getRawData, IntegrationError, IntegrationLogger, - IntegrationStep, JobState, MappedRelationship, PrimitiveEntity, @@ -16,8 +15,10 @@ import { RelationshipDirection, } from '@jupiterone/integration-sdk-core'; import { cloudasset_v1, cloudresourcemanager_v3 } from 'googleapis'; -import { IntegrationConfig } from '../..'; -import { IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { publishMissingPermissionEvent } from '../../utils/events'; import { getProjectIdFromName } from '../../utils/jobState'; import { IAM_ROLE_ENTITY_CLASS, IAM_ROLE_ENTITY_TYPE } from '../iam'; @@ -792,7 +793,7 @@ export async function createApiServiceToAnyResourceRelationships( ); } -export const cloudAssetSteps: IntegrationStep[] = [ +export const cloudAssetSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_IAM_BINDINGS, name: 'IAM Bindings', diff --git a/src/steps/cloud-billing/index.ts b/src/steps/cloud-billing/index.ts index 3c8b5182..29657a99 100644 --- a/src/steps/cloud-billing/index.ts +++ b/src/steps/cloud-billing/index.ts @@ -1,5 +1,7 @@ -import { IntegrationStep } from '@jupiterone/integration-sdk-core'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { CloudBillingClient } from './client'; import { STEP_BILLING_ACCOUNTS, @@ -22,7 +24,7 @@ export async function fetchBillingAccounts( }); } -export const cloudBillingSteps: IntegrationStep[] = [ +export const cloudBillingSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_BILLING_ACCOUNTS, name: 'Billing Accounts', diff --git a/src/steps/cloud-build/index.ts b/src/steps/cloud-build/index.ts index 48905721..a8e2f428 100644 --- a/src/steps/cloud-build/index.ts +++ b/src/steps/cloud-build/index.ts @@ -1,5 +1,4 @@ -import { IntegrationStep } from '@jupiterone/integration-sdk-core'; -import { IntegrationConfig } from '../../types'; +import { GoogleCloudIntegrationStep } from '../../types'; import { buildCloudBuildTriggerTriggersBuildRelationshipsStep } from './steps/build-cloud-build-trigger-triggers-build-relationships'; import { buildCloudBuildTriggerUsesGithubRepositoryStep } from './steps/build-cloud-build-trigger-uses-github-repo-relationships'; import { buildCloudBuildUsesSourceRepositoryRelationshipsStep } from './steps/build-cloud-build-uses-source-repo-relationships'; @@ -11,7 +10,7 @@ import { fetchCloudBuildTriggerStep } from './steps/fetch-cloud-build-triggers'; import { fetchCloudBuildWorkerPoolsStep } from './steps/fetch-cloud-build-worker-pools'; import { fetchCloudBuildStep } from './steps/fetch-cloud-builds'; -export const cloudBuildSteps: IntegrationStep[] = [ +export const cloudBuildSteps: GoogleCloudIntegrationStep[] = [ fetchCloudBuildStep, fetchCloudBuildTriggerStep, fetchCloudBuildWorkerPoolsStep, diff --git a/src/steps/cloud-run/index.ts b/src/steps/cloud-run/index.ts index d47a29af..af114895 100644 --- a/src/steps/cloud-run/index.ts +++ b/src/steps/cloud-run/index.ts @@ -1,9 +1,11 @@ import { createDirectRelationship, - IntegrationStep, RelationshipClass, } from '@jupiterone/integration-sdk-core'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { cacheCloudRunServiceKeyAndUid, getCloudRunServiceKeyFromUid, @@ -145,7 +147,7 @@ export async function fetchCloudRunConfigurations( }); } -export const cloudRunSteps: IntegrationStep[] = [ +export const cloudRunSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_CLOUD_RUN_SERVICES, name: 'Cloud Run Services', diff --git a/src/steps/cloud-source-repositories/index.ts b/src/steps/cloud-source-repositories/index.ts index abdb727b..7f0f9896 100644 --- a/src/steps/cloud-source-repositories/index.ts +++ b/src/steps/cloud-source-repositories/index.ts @@ -1,6 +1,6 @@ -import { IntegrationStep } from '@jupiterone/integration-sdk-core'; -import { IntegrationConfig } from '../../types'; +import { GoogleCloudIntegrationStep } from '../../types'; import { fetchCloudSourceRepositoriesStep } from './steps/fetch-cloud-source-repositories'; -export const cloudSourceRepositoriesSteps: IntegrationStep[] = - [fetchCloudSourceRepositoriesStep]; +export const cloudSourceRepositoriesSteps: GoogleCloudIntegrationStep[] = [ + fetchCloudSourceRepositoriesStep, +]; diff --git a/src/steps/compute/index.ts b/src/steps/compute/index.ts index 003aba74..9fad0fb0 100644 --- a/src/steps/compute/index.ts +++ b/src/steps/compute/index.ts @@ -1,5 +1,4 @@ import { - IntegrationStep, JobState, Entity, createDirectRelationship, @@ -8,7 +7,10 @@ import { getRawData, } from '@jupiterone/integration-sdk-core'; import { ComputeClient } from './client'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { createComputeDiskEntity, createComputeInstanceEntity, @@ -1877,7 +1879,7 @@ export async function fetchComputeSslPolicies( }); } -export const computeSteps: IntegrationStep[] = [ +export const computeSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_COMPUTE_NETWORKS, name: 'Compute Networks', diff --git a/src/steps/containers/index.ts b/src/steps/containers/index.ts index ba03ca7a..f0d84476 100644 --- a/src/steps/containers/index.ts +++ b/src/steps/containers/index.ts @@ -1,9 +1,11 @@ import { createDirectRelationship, - IntegrationStep, RelationshipClass, } from '@jupiterone/integration-sdk-core'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { ENTITY_TYPE_COMPUTE_INSTANCE_GROUP, STEP_COMPUTE_INSTANCE_GROUPS, @@ -96,7 +98,7 @@ export async function fetchContainerClusters( }); } -export const containerSteps: IntegrationStep[] = [ +export const containerSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_CONTAINER_CLUSTERS, name: 'Container Clusters', diff --git a/src/steps/dataproc/index.ts b/src/steps/dataproc/index.ts index c1307450..c2720a13 100644 --- a/src/steps/dataproc/index.ts +++ b/src/steps/dataproc/index.ts @@ -2,12 +2,14 @@ import { createDirectRelationship, createMappedRelationship, getRawData, - IntegrationStep, RelationshipClass, RelationshipDirection, } from '@jupiterone/integration-sdk-core'; import { dataproc_v1 } from 'googleapis'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { getKmsGraphObjectKeyFromKmsKeyName } from '../../utils/kms'; import { ENTITY_TYPE_COMPUTE_IMAGE, STEP_COMPUTE_IMAGES } from '../compute'; import { ENTITY_TYPE_KMS_KEY, STEP_CLOUD_KMS_KEYS } from '../kms'; @@ -183,7 +185,7 @@ export async function createClusterStorageRelationships( ); } -export const dataprocSteps: IntegrationStep[] = [ +export const dataprocSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_DATAPROC_CLUSTERS, name: 'Dataproc Clusters', diff --git a/src/steps/dns/index.ts b/src/steps/dns/index.ts index 25d080d0..8aca6071 100644 --- a/src/steps/dns/index.ts +++ b/src/steps/dns/index.ts @@ -1,9 +1,11 @@ import { createDirectRelationship, - IntegrationStep, RelationshipClass, } from '@jupiterone/integration-sdk-core'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { ENTITY_TYPE_COMPUTE_NETWORK, STEP_COMPUTE_NETWORKS } from '../compute'; import { DNSClient } from './client'; import { @@ -71,7 +73,7 @@ export async function fetchDNSPolicies( }); } -export const dnsManagedZonesSteps: IntegrationStep[] = [ +export const dnsManagedZonesSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_DNS_MANAGED_ZONES, name: 'DNS Managed Zones', diff --git a/src/steps/functions/index.ts b/src/steps/functions/index.ts index 239324ab..3102ad9a 100644 --- a/src/steps/functions/index.ts +++ b/src/steps/functions/index.ts @@ -1,11 +1,13 @@ import { createDirectRelationship, getRawData, - IntegrationStep, RelationshipClass, } from '@jupiterone/integration-sdk-core'; import { CloudFunctionsClient } from './client'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { createCloudFunctionEntity } from './converters'; import { STEP_IAM_SERVICE_ACCOUNTS } from '../iam'; import { @@ -145,7 +147,7 @@ export async function buildCloudFunctionStorageBucketRelationships( ); } -export const functionsSteps: IntegrationStep[] = [ +export const functionsSteps: GoogleCloudIntegrationStep[] = [ { id: FunctionStepsSpec.FETCH_CLOUD_FUNCTIONS.id, name: FunctionStepsSpec.FETCH_CLOUD_FUNCTIONS.name, diff --git a/src/steps/iam/index.ts b/src/steps/iam/index.ts index b70a133e..57b0f382 100644 --- a/src/steps/iam/index.ts +++ b/src/steps/iam/index.ts @@ -1,11 +1,13 @@ import { createDirectRelationship, IntegrationLogger, - IntegrationStep, JobState, } from '@jupiterone/integration-sdk-core'; import { IamClient } from './client'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { createIamRoleEntity, createIamServiceAccountEntity, @@ -228,7 +230,7 @@ export async function fetchIamServiceAccounts( }); } -export const iamSteps: IntegrationStep[] = [ +export const iamSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_IAM_CUSTOM_ROLES, name: 'Identity and Access Management (IAM) Custom Roles', diff --git a/src/steps/kms/index.ts b/src/steps/kms/index.ts index 248bdac2..5401ea8f 100644 --- a/src/steps/kms/index.ts +++ b/src/steps/kms/index.ts @@ -1,10 +1,12 @@ import { createDirectRelationship, - IntegrationStep, RelationshipClass, } from '@jupiterone/integration-sdk-core'; import { CloudKmsClient } from './client'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { ENTITY_CLASS_KMS_KEY, ENTITY_CLASS_KMS_KEY_RING, @@ -24,22 +26,25 @@ export async function fetchKmsKeyRings( const { jobState, instance, logger } = context; const client = new CloudKmsClient({ config: instance.config }); - await client.iterateKeyRings(async (keyRing) => { - await jobState.addEntity(createKmsKeyRingEntity(keyRing)); - }, ({ - totalRequestsMade, - totalResourcesReturned, - maximumResourcesPerPage - }) => { - logger.info( - { - totalRequestsMade, - totalResourcesReturned, - maximumResourcesPerPage - }, - 'KMS Key Rings API Requests summary', - ); - }); + await client.iterateKeyRings( + async (keyRing) => { + await jobState.addEntity(createKmsKeyRingEntity(keyRing)); + }, + ({ + totalRequestsMade, + totalResourcesReturned, + maximumResourcesPerPage, + }) => { + logger.info( + { + totalRequestsMade, + totalResourcesReturned, + maximumResourcesPerPage, + }, + 'KMS Key Rings API Requests summary', + ); + }, + ); } export async function fetchKmsCryptoKeys( @@ -90,7 +95,7 @@ export async function fetchKmsCryptoKeys( ); } -export const kmsSteps: IntegrationStep[] = [ +export const kmsSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_CLOUD_KMS_KEY_RINGS, name: 'KMS Key Rings', diff --git a/src/steps/logging/index.ts b/src/steps/logging/index.ts index 07102d83..1ea26ff5 100644 --- a/src/steps/logging/index.ts +++ b/src/steps/logging/index.ts @@ -1,10 +1,12 @@ import { createDirectRelationship, getRawData, - IntegrationStep, RelationshipClass, } from '@jupiterone/integration-sdk-core'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { LoggingClient } from './client'; import { LOGGING_METRIC_ENTITY_CLASS, @@ -139,7 +141,7 @@ export async function fetchMetrics( }); } -export const loggingSteps: IntegrationStep[] = [ +export const loggingSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_LOGGING_PROJECT_SINKS, name: 'Logging Project Sinks', diff --git a/src/steps/memcache/index.ts b/src/steps/memcache/index.ts index 12ecc5a5..7b82c8c6 100644 --- a/src/steps/memcache/index.ts +++ b/src/steps/memcache/index.ts @@ -1,11 +1,13 @@ import { createDirectRelationship, getRawData, - IntegrationStep, RelationshipClass, } from '@jupiterone/integration-sdk-core'; import { memcache_v1 } from 'googleapis'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { ENTITY_TYPE_COMPUTE_NETWORK, STEP_COMPUTE_NETWORKS } from '../compute'; import { MemcacheClient } from './client'; import { @@ -104,7 +106,7 @@ export async function buildMemcacheInstancesUsesNetworkRelationships( ); } -export const memcacheSteps: IntegrationStep[] = [ +export const memcacheSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_MEMCACHE_INSTANCES, name: 'Memcache Instances', diff --git a/src/steps/monitoring/index.ts b/src/steps/monitoring/index.ts index f33efa3d..ccf5464f 100644 --- a/src/steps/monitoring/index.ts +++ b/src/steps/monitoring/index.ts @@ -1,5 +1,7 @@ -import { IntegrationStep } from '@jupiterone/integration-sdk-core'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { publishUnsupportedConfigEvent } from '../../utils/events'; import { MonitoringClient } from './client'; import { @@ -39,7 +41,7 @@ export async function fetchAlertPolicies( } } -export const monitoringSteps: IntegrationStep[] = [ +export const monitoringSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_MONITORING_ALERT_POLICIES, name: 'Monitoring Alert Policies', diff --git a/src/steps/privateca/index.ts b/src/steps/privateca/index.ts index 154f9303..e964c092 100644 --- a/src/steps/privateca/index.ts +++ b/src/steps/privateca/index.ts @@ -1,11 +1,13 @@ import { createDirectRelationship, getRawData, - IntegrationStep, RelationshipClass, } from '@jupiterone/integration-sdk-core'; import { privateca_v1beta1 } from 'googleapis'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { isMemberPublic } from '../../utils/iam'; import { PrivateCaClient } from './client'; import { @@ -158,7 +160,7 @@ export async function fetchAuthorityCertificates( ); } -export const privateCaSteps: IntegrationStep[] = [ +export const privateCaSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_PRIVATE_CA_CERTIFICATE_AUTHORITIES, name: 'Private CA Certificate Authorities', diff --git a/src/steps/pub-sub/index.ts b/src/steps/pub-sub/index.ts index 76ec8b3c..a4c3702d 100644 --- a/src/steps/pub-sub/index.ts +++ b/src/steps/pub-sub/index.ts @@ -2,12 +2,14 @@ import { createDirectRelationship, createMappedRelationship, getRawData, - IntegrationStep, RelationshipClass, RelationshipDirection, } from '@jupiterone/integration-sdk-core'; import { pubsub_v1 } from 'googleapis'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { ENTITY_TYPE_KMS_KEY, STEP_CLOUD_KMS_KEYS } from '../kms'; import { PubSubClient } from './client'; import { @@ -162,7 +164,7 @@ export async function fetchPubSubSubscriptions( }); } -export const pubSubSteps: IntegrationStep[] = [ +export const pubSubSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_PUBSUB_TOPICS, name: 'PubSub Topics', diff --git a/src/steps/redis/index.ts b/src/steps/redis/index.ts index fb5a27d7..e3fb2e79 100644 --- a/src/steps/redis/index.ts +++ b/src/steps/redis/index.ts @@ -1,10 +1,12 @@ import { createDirectRelationship, getRawData, - IntegrationStep, RelationshipClass, } from '@jupiterone/integration-sdk-core'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { STEP_COMPUTE_NETWORKS } from '../compute'; import { RedisClient } from './client'; import { @@ -78,7 +80,7 @@ export async function buildRedisInstanceUsesNetworkRelationships( ); } -export const redisSteps: IntegrationStep[] = [ +export const redisSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_REDIS_INSTANCES, name: 'Redis Instances', diff --git a/src/steps/resource-manager/index.ts b/src/steps/resource-manager/index.ts index 15fb4fb2..957a2f50 100644 --- a/src/steps/resource-manager/index.ts +++ b/src/steps/resource-manager/index.ts @@ -1,12 +1,14 @@ import { - IntegrationStep, Entity, createDirectRelationship, createMappedRelationship, RelationshipDirection, } from '@jupiterone/integration-sdk-core'; import { ResourceManagerClient } from './client'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { createAuditConfigEntity, createFolderEntity, @@ -347,7 +349,7 @@ export async function fetchIamPolicyAuditConfig( }); } -export const resourceManagerSteps: IntegrationStep[] = [ +export const resourceManagerSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_RESOURCE_MANAGER_ORGANIZATION, name: 'Resource Manager Organization', diff --git a/src/steps/secret-manager/index.ts b/src/steps/secret-manager/index.ts index 13c367fc..1fd51fe2 100644 --- a/src/steps/secret-manager/index.ts +++ b/src/steps/secret-manager/index.ts @@ -1,11 +1,13 @@ import { createDirectRelationship, getRawData, - IntegrationStep, RelationshipClass, } from '@jupiterone/integration-sdk-core'; import { secretmanager_v1 } from 'googleapis'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { SecretManagerClient } from './client'; import { SecretManagerEntities, @@ -58,7 +60,7 @@ export async function fetchSecretVersions( ); } -export const secretManagerSteps: IntegrationStep[] = [ +export const secretManagerSteps: GoogleCloudIntegrationStep[] = [ { ...SecretManagerSteps.FETCH_SECRETS, entities: [SecretManagerEntities.SECRET], diff --git a/src/steps/service-usage/index.ts b/src/steps/service-usage/index.ts index cf2919bb..2e8e55fa 100644 --- a/src/steps/service-usage/index.ts +++ b/src/steps/service-usage/index.ts @@ -1,10 +1,12 @@ import { - IntegrationStep, createDirectRelationship, RelationshipClass, } from '@jupiterone/integration-sdk-core'; import { ServiceUsageClient } from './client'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { createApiServiceEntity } from './converters'; import { ServiceUsageStepIds, @@ -103,7 +105,7 @@ export async function fetchApiServices( ); } -export const serviceUsageSteps: IntegrationStep[] = [ +export const serviceUsageSteps: GoogleCloudIntegrationStep[] = [ { id: ServiceUsageStepIds.FETCH_API_SERVICES, name: 'API Services', diff --git a/src/steps/spanner/index.ts b/src/steps/spanner/index.ts index 15657f75..ae626774 100644 --- a/src/steps/spanner/index.ts +++ b/src/steps/spanner/index.ts @@ -1,12 +1,14 @@ import { createDirectRelationship, createMappedRelationship, - IntegrationStep, RelationshipClass, RelationshipDirection, } from '@jupiterone/integration-sdk-core'; import { spanner_v1 } from 'googleapis'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { isMemberPublic } from '../../utils/iam'; import { getKmsGraphObjectKeyFromKmsKeyName } from '../../utils/kms'; import { RELATIONSHIP_TYPE_DATASET_USES_KMS_CRYPTO_KEY } from '../big-query'; @@ -174,7 +176,7 @@ export async function fetchSpannerInstanceDatabases( ); } -export const spannerSteps: IntegrationStep[] = [ +export const spannerSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_SPANNER_INSTANCE_CONFIGS, name: 'Spanner Instance Configs', diff --git a/src/steps/sql-admin/index.ts b/src/steps/sql-admin/index.ts index be89a448..553fd90a 100644 --- a/src/steps/sql-admin/index.ts +++ b/src/steps/sql-admin/index.ts @@ -3,11 +3,13 @@ import { createDirectRelationship, createMappedRelationship, getRawData, - IntegrationStep, RelationshipClass, RelationshipDirection, } from '@jupiterone/integration-sdk-core'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { getKmsGraphObjectKeyFromKmsKeyName } from '../../utils/kms'; import { ENTITY_TYPE_KMS_KEY, @@ -154,7 +156,7 @@ export async function buildSqlAdminInstanceKmsKeyRelationships( } } -export const sqlAdminSteps: IntegrationStep[] = [ +export const sqlAdminSteps: GoogleCloudIntegrationStep[] = [ { id: STEP_SQL_ADMIN_INSTANCES, name: 'SQL Admin Instances', diff --git a/src/steps/steps.ts b/src/steps/steps.ts index 28c48f42..20b20bfe 100644 --- a/src/steps/steps.ts +++ b/src/steps/steps.ts @@ -1,11 +1,10 @@ import { ExecutionHandlerFunction, IntegrationProviderAuthorizationError, - IntegrationStep, StepExecutionContext, } from '@jupiterone/integration-sdk-core'; import { createErrorProps } from '../google-cloud/utils/createErrorProps'; -import { IntegrationConfig } from '../types'; +import { GoogleCloudIntegrationStep } from '../types'; import { accessPoliciesSteps } from './access-context-manager'; import { apiGatewaySteps } from './api-gateway'; import { appEngineSteps } from './app-engine'; @@ -38,7 +37,7 @@ import { spannerSteps } from './spanner'; import { sqlAdminSteps } from './sql-admin'; import { storageSteps } from './storage'; -const steps: IntegrationStep[] = wrapStepExecutionHandlers([ +const steps: GoogleCloudIntegrationStep[] = wrapStepExecutionHandlers([ ...functionsSteps, ...storageSteps, ...serviceUsageSteps, @@ -73,8 +72,8 @@ const steps: IntegrationStep[] = wrapStepExecutionHandlers([ ]); function wrapStepExecutionHandlers( - steps: IntegrationStep[], -): IntegrationStep[] { + steps: GoogleCloudIntegrationStep[], +): GoogleCloudIntegrationStep[] { return steps.map((step) => { step.executionHandler; return { diff --git a/src/steps/storage/index.ts b/src/steps/storage/index.ts index 9b89c849..e2810c43 100644 --- a/src/steps/storage/index.ts +++ b/src/steps/storage/index.ts @@ -1,6 +1,8 @@ -import { IntegrationStep } from '@jupiterone/integration-sdk-core'; import { CloudStorageClient } from './client'; -import { IntegrationConfig, IntegrationStepContext } from '../../types'; +import { + GoogleCloudIntegrationStep, + IntegrationStepContext, +} from '../../types'; import { createCloudStorageBucketEntity } from './converters'; import { StorageStepsSpec, StorageEntitiesSpec } from './constants'; import { storage_v1 } from 'googleapis'; @@ -25,9 +27,17 @@ export async function fetchStorageBuckets( publicAccessPreventionPolicy = await orgPolicyClient.fetchOrganizationPublicAccessPreventionPolicy(); } catch (err) { - logger.warn({ err }, 'Error fetching organization public access prevention policy'); - - if (err.code === 403 && (err.message as string).includes(`Permission 'orgpolicy.policy.get' denied on resource`)) { + logger.warn( + { err }, + 'Error fetching organization public access prevention policy', + ); + + if ( + err.code === 403 && + (err.message as string).includes( + `Permission 'orgpolicy.policy.get' denied on resource`, + ) + ) { logger.publishEvent({ name: 'missing_permission', description: @@ -83,7 +93,7 @@ export async function fetchStorageBuckets( } } -export const storageSteps: IntegrationStep[] = [ +export const storageSteps: GoogleCloudIntegrationStep[] = [ { id: StorageStepsSpec.FETCH_STORAGE_BUCKETS.id, name: StorageStepsSpec.FETCH_STORAGE_BUCKETS.name, diff --git a/src/types.ts b/src/types.ts index 09d3b9ee..02430944 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,6 @@ import { IntegrationInstanceConfig, + IntegrationStep, IntegrationStepExecutionContext, } from '@jupiterone/integration-sdk-core'; import { ParsedServiceAccountKeyFile } from './utils/parseServiceAccountKeyFile'; @@ -31,3 +32,8 @@ export interface IntegrationConfig extends SerializedIntegrationConfig { // HACK - used to prevent binding step ingestion for large accounts. Think twice before using. markBindingStepsAsPartial?: boolean; } + +export interface GoogleCloudIntegrationStep + extends IntegrationStep { + permissions?: Array; +} diff --git a/yarn.lock b/yarn.lock index 072d2599..fcdb135f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2150,6 +2150,11 @@ commander@^9.4.0: resolved "https://registry.yarnpkg.com/commander/-/commander-9.4.0.tgz#bc4a40918fefe52e22450c111ecd6b7acce6f11c" integrity sha512-sRPT+umqkz90UA8M1yqYfnHlZA7fF6nSphDtxeywPZ49ysjxDQybzk13CL+mXekDRG92skbcqCLVovuCusNmFw== +commander@^9.4.1: + version "9.5.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-9.5.0.tgz#bc08d1eb5cedf7ccb797a96199d41c7bc3e60d30" + integrity sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ== + compare-versions@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/compare-versions/-/compare-versions-3.6.0.tgz#1a5689913685e5a87637b8d3ffca75514ec41d62"