INT-9871: add ingest sources

src/ingestSources.ts

@@ -0,0 +1,69 @@
+import { IntegrationIngestionConfigFieldMap } from '@jupiterone/integration-sdk-core';
+import { AccessContextManagerIngestionConfig } from './steps/access-context-manager/constants';
+import { ApiGatewayIngestionConfig } from './steps/api-gateway/constants';
+import { AppEngineIngestionConfig } from './steps/app-engine/constants';
+import { BigQueryIngestionConfig } from './steps/big-query';
+import { BigTableIngestionConfig } from './steps/big-table/constants';
+import { BillingBudgetsIngestionConfig } from './steps/billing-budgets/constants';
+import { BinaryAuthorizationIngestionConfig } from './steps/binary-authorization/constants';
+import { CloudAssetIngestionConfig } from './steps/cloud-asset/constants';
+import { CloudBillingIngestionConfig } from './steps/cloud-billing/constants';
+import { CloudBuildIngestionConfig } from './steps/cloud-build/constants';
+import { CloudRunIngestionConfig } from './steps/cloud-run/constants';
+import { CloudSourceRepositoriesIngestionConfig } from './steps/cloud-source-repositories/constants';
+import { ComputeIngestionConfig } from './steps/compute';
+import { ContainersIngestionConfig } from './steps/containers';
+import { DataprocIngestionConfig } from './steps/dataproc/constants';
+import { DnsIngestionConfig } from './steps/dns/constants';
+import { FunctionsIngestionConfig } from './steps/functions';
+import { IamIngestionConfig } from './steps/iam';
+import { KmsIngestionConfig } from './steps/kms';
+import { LoggingIngestionConfig } from './steps/logging/constants';
+import { MemcacheIngestionConfig } from './steps/memcache/constants';
+import { MonitoringIngestionConfig } from './steps/monitoring/constants';
+import { PrivatecaIngestionConfig } from './steps/privateca/constants';
+import { PubSubIngestionConfig } from './steps/pub-sub/constants';
+import { RedisIngestionConfig } from './steps/redis/constants';
+import { ResourceManagerIngestionConfig } from './steps/resource-manager';
+import { SecretManagerIngestionConfig } from './steps/secret-manager/constants';
+import { ServiceUsageIngestionConfig } from './steps/service-usage/constants';
+import { SpannerIngestionConfig } from './steps/spanner/constants';
+import { SQLAdminIngestionConfig } from './steps/sql-admin';
+import { StorageIngestionConfig } from './steps/storage/constants';
+import { WebSecurityScannerIngestionConfig } from './steps/web-security-scanner/constants';
+
+export const ingestionConfig: IntegrationIngestionConfigFieldMap = {
+  ...AccessContextManagerIngestionConfig,
+  ...ApiGatewayIngestionConfig,
+  ...ApiGatewayIngestionConfig,
+  ...AppEngineIngestionConfig,
+  ...BigQueryIngestionConfig,
+  ...BigTableIngestionConfig,
+  ...BillingBudgetsIngestionConfig,
+  ...BinaryAuthorizationIngestionConfig,
+  ...CloudAssetIngestionConfig,
+  ...CloudBillingIngestionConfig,
+  ...CloudBuildIngestionConfig,
+  ...CloudRunIngestionConfig,
+  ...CloudSourceRepositoriesIngestionConfig,
+  ...ComputeIngestionConfig,
+  ...ContainersIngestionConfig,
+  ...DataprocIngestionConfig,
+  ...DnsIngestionConfig,
+  ...FunctionsIngestionConfig,
+  ...IamIngestionConfig,
+  ...KmsIngestionConfig,
+  ...LoggingIngestionConfig,
+  ...MemcacheIngestionConfig,
+  ...MonitoringIngestionConfig,
+  ...PrivatecaIngestionConfig,
+  ...PubSubIngestionConfig,
+  ...RedisIngestionConfig,
+  ...ResourceManagerIngestionConfig,
+  ...SecretManagerIngestionConfig,
+  ...ServiceUsageIngestionConfig,
+  ...SpannerIngestionConfig,
+  ...SQLAdminIngestionConfig,
+  ...StorageIngestionConfig,
+  ...WebSecurityScannerIngestionConfig,
+};

src/steps/access-context-manager/constants.ts

@@ -57,3 +57,27 @@ export const RELATIONSHIP_TYPE_SERVICE_PERIMETER_HAS_INGRESS_POLICY =
   'google_access_context_manager_service_perimeter_has_ingress_policy';
 export const RELATIONSHIP_TYPE_INGRESS_POLICY_HAS_API_OPERATION =
   'google_access_context_manager_service_perimeter_ingress_policy_has_api_operation';
+
+export const IngestionSources = {
+  ACCESS_CONTEXT_MANAGER_ACCESS_POLICIES: 'acm-access-policies',
+  ACCESS_CONTEXT_MANAGER_ACCESS_LEVELS: 'acm-access-levels',
+  ACCESS_CONTEXT_MANAGER_SERVICE_PERIMETERS: 'acm-service-perimeters',
+};
+
+export const AccessContextManagerIngestionConfig = {
+  [IngestionSources.ACCESS_CONTEXT_MANAGER_ACCESS_POLICIES]: {
+    title: 'Google Cloud Access Context Manager Access Policies',
+    description: 'Defines secure access boundaries in GCP.',
+    defaultsToDisabled: false,
+  },
+  [IngestionSources.ACCESS_CONTEXT_MANAGER_ACCESS_LEVELS]: {
+    title: 'Google Cloud Access Context Manager Access Levels',
+    description: 'Manages access via hierarchical levels in GCP.',
+    defaultsToDisabled: false,
+  },
+  [IngestionSources.ACCESS_CONTEXT_MANAGER_SERVICE_PERIMETERS]: {
+    title: 'Google Cloud Access Context Manager Service Perimeters',
+    description: 'Secures resources within network boundaries.',
+    defaultsToDisabled: false,
+  },
+};

src/steps/access-context-manager/index.ts

@@ -38,6 +38,7 @@ import {
   RELATIONSHIP_TYPE_SERVICE_PERIMETER_HAS_INGRESS_POLICY,
   RELATIONSHIP_TYPE_INGRESS_POLICY_HAS_API_OPERATION,
   ENTITY_CLASS_ACCESS_CONTEXT_MANAGER_SERVICE_PERIMETER_INGRESS_POLICY,
+  IngestionSources,
 } from './constants';
 import {
   PROJECT_ENTITY_TYPE,
@@ -391,6 +392,7 @@ export async function fetchServicePerimeters(
 export const accessPoliciesSteps: GoogleCloudIntegrationStep[] = [
   {
     id: STEP_ACCESS_CONTEXT_MANAGER_ACCESS_POLICIES,
+    ingestionSourceId: IngestionSources.ACCESS_CONTEXT_MANAGER_ACCESS_POLICIES,
     name: 'Access Context Manager Access Policies',
     entities: [
       {
@@ -407,6 +409,7 @@ export const accessPoliciesSteps: GoogleCloudIntegrationStep[] = [
   },
   {
     id: STEP_ACCESS_CONTEXT_MANAGER_ACCESS_LEVELS,
+    ingestionSourceId: IngestionSources.ACCESS_CONTEXT_MANAGER_ACCESS_LEVELS,
     name: 'Access Context Manager Access Levels',
     entities: [
       {
@@ -430,6 +433,8 @@ export const accessPoliciesSteps: GoogleCloudIntegrationStep[] = [
   },
   {
     id: STEP_ACCESS_CONTEXT_MANAGER_SERVICE_PERIMETERS,
+    ingestionSourceId:
+      IngestionSources.ACCESS_CONTEXT_MANAGER_SERVICE_PERIMETERS,
     name: 'Access Context Manager Service Perimeters',
     entities: [
       {

src/steps/api-gateway/constants.ts

@@ -18,3 +18,27 @@ export const RELATIONSHIP_TYPE_API_GATEWAY_API_HAS_GATEWAY =
   'google_api_gateway_api_has_gateway';
 export const RELATIONSHIP_TYPE_API_GATEWAY_API_CONFIG_USES_SERVICE_ACCOUNT =
   'google_api_gateway_api_config_uses_iam_service_account';
+
+export const IngestionSources = {
+  API_GATEWAY_APIS: 'api-gateway-apis',
+  API_GATEWAY_API_CONFIGS: 'api-gateway-api-configs',
+  API_GATEWAY_GATEWAYS: 'api-gateway-gateways',
+};
+
+export const ApiGatewayIngestionConfig = {
+  [IngestionSources.API_GATEWAY_APIS]: {
+    title: 'Google Cloud API Gateway APIs',
+    description: 'Endpoint management for API gateways.',
+    defaultsToDisabled: false,
+  },
+  [IngestionSources.API_GATEWAY_API_CONFIGS]: {
+    title: 'Google Cloud API Gateway API Configurations',
+    description: 'Config settings for API interfaces.',
+    defaultsToDisabled: false,
+  },
+  [IngestionSources.API_GATEWAY_GATEWAYS]: {
+    title: 'Google Cloud API Gateway Gateways',
+    description: 'Networking gateways for API management.',
+    defaultsToDisabled: false,
+  },
+};

src/steps/api-gateway/index.ts

@@ -22,6 +22,7 @@ import {
   RELATIONSHIP_TYPE_API_GATEWAY_API_USES_CONFIG,
   RELATIONSHIP_TYPE_API_GATEWAY_API_HAS_GATEWAY,
   RELATIONSHIP_TYPE_API_GATEWAY_API_CONFIG_USES_SERVICE_ACCOUNT,
+  IngestionSources,
 } from './constants';
 import {
   createApiGatewayApiConfigEntity,
@@ -188,6 +189,7 @@ export async function fetchApiGatewayGateways(
 export const apiGatewaySteps: GoogleCloudIntegrationStep[] = [
   {
     id: STEP_API_GATEWAY_APIS,
+    ingestionSourceId: IngestionSources.API_GATEWAY_APIS,
     name: 'Api Gateway APIs',
     entities: [
       {
@@ -203,6 +205,7 @@ export const apiGatewaySteps: GoogleCloudIntegrationStep[] = [
   },
   {
     id: STEP_API_GATEWAY_API_CONFIGS,
+    ingestionSourceId: IngestionSources.API_GATEWAY_API_CONFIGS,
     name: 'Api Gateway Api Configs',
     entities: [
       {
@@ -235,6 +238,7 @@ export const apiGatewaySteps: GoogleCloudIntegrationStep[] = [
   },
   {
     id: STEP_API_GATEWAY_GATEWAYS,
+    ingestionSourceId: IngestionSources.API_GATEWAY_GATEWAYS,
     name: 'Api Gateway Gateways',
     entities: [
       {

src/steps/app-engine/constants.ts

@@ -35,3 +35,33 @@ export const RELATIONSHIP_TYPE_GOOGLE_USER_CREATED_VERSION =
 
 export const RELATIONSHIP_TYPE_SERVICE_ACCOUNT_CREATED_VERSION =
   'google_iam_service_account_created_app_engine_version';
+
+export const IngestionSources = {
+  APP_ENGINE_APPLICATION: 'app-engine-application',
+  APP_ENGINE_SERVICES: 'app-engine-services',
+  APP_ENGINE_VERSIONS: 'app-engine-versions',
+  APP_ENGINE_INSTANCES: 'app-engine-instances',
+};
+
+export const AppEngineIngestionConfig = {
+  [IngestionSources.APP_ENGINE_APPLICATION]: {
+    title: 'Google Cloud App Engine Application',
+    description: 'Platform for building scalable web apps.',
+    defaultsToDisabled: false,
+  },
+  [IngestionSources.APP_ENGINE_SERVICES]: {
+    title: 'Google Cloud App Engine Services',
+    description: 'Modular components of App Engine apps.',
+    defaultsToDisabled: false,
+  },
+  [IngestionSources.APP_ENGINE_VERSIONS]: {
+    title: 'Google Cloud App Engine Versions',
+    description: 'Versioning for App Engine application components.',
+    defaultsToDisabled: false,
+  },
+  [IngestionSources.APP_ENGINE_INSTANCES]: {
+    title: 'Google Cloud App Engine Instances',
+    description: 'Running instances of App Engine services.',
+    defaultsToDisabled: false,
+  },
+};

src/steps/app-engine/index.ts

@@ -34,6 +34,7 @@ import {
   RELATIONSHIP_TYPE_GOOGLE_USER_CREATED_VERSION,
   RELATIONSHIP_TYPE_SERVICE_ACCOUNT_CREATED_VERSION,
   STEP_CREATE_APP_ENGINE_BUCKET_RELATIONSHIPS,
+  IngestionSources,
 } from './constants';
 import {
   createAppEngineApplicationEntity,
@@ -382,6 +383,7 @@ export async function fetchAppEngineVersionInstances(
 export const appEngineSteps: GoogleCloudIntegrationStep[] = [
   {
     id: STEP_APP_ENGINE_APPLICATION,
+    ingestionSourceId: IngestionSources.APP_ENGINE_APPLICATION,
     name: 'AppEngine Application',
     entities: [
       {
@@ -416,6 +418,7 @@ export const appEngineSteps: GoogleCloudIntegrationStep[] = [
   },
   {
     id: STEP_APP_ENGINE_SERVICES,
+    ingestionSourceId: IngestionSources.APP_ENGINE_SERVICES,
     name: 'AppEngine Services',
     entities: [
       {
@@ -439,6 +442,7 @@ export const appEngineSteps: GoogleCloudIntegrationStep[] = [
   },
   {
     id: STEP_APP_ENGINE_VERSIONS,
+    ingestionSourceId: IngestionSources.APP_ENGINE_VERSIONS,
     name: 'AppEngine Versions',
     entities: [
       {
@@ -473,6 +477,7 @@ export const appEngineSteps: GoogleCloudIntegrationStep[] = [
   },
   {
     id: STEP_APP_ENGINE_INSTANCES,
+    ingestionSourceId: IngestionSources.APP_ENGINE_INSTANCES,
     name: 'AppEngine Instances',
     entities: [
       {

src/steps/big-query/constants.ts

@@ -19,3 +19,27 @@ export const RELATIONSHIP_TYPE_DATASET_HAS_TABLE =
   'google_bigquery_dataset_has_table';
 export const RELATIONSHIP_TYPE_DATASET_HAS_MODEL =
   'google_bigquery_dataset_has_model';
+
+export const IngestionSources = {
+  BIG_QUERY_DATASETS: 'big-query-datasets',
+  BIG_QUERY_MODELS: 'big-query-models',
+  BIG_QUERY_TABLES: 'big-query-tables',
+};
+
+export const BigQueryIngestionConfig = {
+  [IngestionSources.BIG_QUERY_DATASETS]: {
+    title: 'Google Cloud BigQuery Datasets',
+    description: 'Organized collections of BigQuery data.',
+    defaultsToDisabled: true,
+  },
+  [IngestionSources.BIG_QUERY_MODELS]: {
+    title: 'Google Cloud BigQuery Models',
+    description: 'Machine learning models in BigQuery.',
+    defaultsToDisabled: true,
+  },
+  [IngestionSources.BIG_QUERY_TABLES]: {
+    title: 'Google Cloud BigQuery Tables',
+    description: 'Structured data tables in BigQuery.',
+    defaultsToDisabled: true,
+  },
+};

src/steps/big-query/index.ts

@@ -30,6 +30,7 @@ import {
   BIG_QUERY_MODEL_ENTITY_CLASS,
   RELATIONSHIP_TYPE_DATASET_HAS_MODEL,
   STEP_BUILD_BIG_QUERY_DATASET_KMS_RELATIONSHIPS,
+  IngestionSources,
 } from './constants';
 import {
   createBigQueryDatasetEntity,
@@ -257,6 +258,7 @@ export function handleDatasetError(
 export const bigQuerySteps: GoogleCloudIntegrationStep[] = [
   {
     id: STEP_BIG_QUERY_DATASETS,
+    ingestionSourceId: IngestionSources.BIG_QUERY_DATASETS,
     name: 'Big Query Datasets',
     entities: [
       {
@@ -288,6 +290,7 @@ export const bigQuerySteps: GoogleCloudIntegrationStep[] = [
   },
   {
     id: STEP_BIG_QUERY_MODELS,
+    ingestionSourceId: IngestionSources.BIG_QUERY_MODELS,
     name: 'Big Query Models',
     entities: [
       {
@@ -315,6 +318,7 @@ export const bigQuerySteps: GoogleCloudIntegrationStep[] = [
   },
   {
     id: STEP_BIG_QUERY_TABLES,
+    ingestionSourceId: IngestionSources.BIG_QUERY_TABLES,
     name: 'Big Query Tables',
     entities: [
       {

src/steps/big-table/constants.ts

@@ -36,3 +36,39 @@ export const RELATIONSHIP_TYPE_TABLE_HAS_BACKUP =
 
 export const RELATIONSHIP_TYPE_CLUSTER_USES_KMS_KEY =
   'google_bigtable_cluster_uses_kms_key';
+
+export const IngestionSources = {
+  BIG_TABLE_INSTANCES: 'big-table-instances',
+  BIG_TABLE_APP_PROFILES: 'big-table-app-profiles',
+  BIG_TABLE_CLUSTERS: 'big-table-clusters',
+  BIG_TABLE_BACKUPS: 'big-table-backups',
+  BIG_TABLE_TABLES: 'big-table-tables',
+};
+
+export const BigTableIngestionConfig = {
+  [IngestionSources.BIG_TABLE_INSTANCES]: {
+    title: 'Google Cloud BigTable Instances',
+    description: 'Managed NoSQL database instances.',
+    defaultsToDisabled: true,
+  },
+  [IngestionSources.BIG_TABLE_APP_PROFILES]: {
+    title: 'Google Cloud BigTable App Profiles',
+    description: 'App profiles for BigTable configuration.',
+    defaultsToDisabled: true,
+  },
+  [IngestionSources.BIG_TABLE_CLUSTERS]: {
+    title: 'Google Cloud BigTable Clusters',
+    description: 'Cluster management in BigTable.',
+    defaultsToDisabled: true,
+  },
+  [IngestionSources.BIG_TABLE_BACKUPS]: {
+    title: 'Google Cloud BigTable Backups',
+    description: 'Backup solutions for BigTable data.',
+    defaultsToDisabled: true,
+  },
+  [IngestionSources.BIG_TABLE_TABLES]: {
+    title: 'Google Cloud BigTable Tables',
+    description: 'Data tables within BigTable.',
+    defaultsToDisabled: true,
+  },
+};