upgrade sdk (#116)

.github/workflows/build.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
         os: [ubuntu-latest]
 
     steps:
@@ -44,7 +44,7 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v1
         with:
-          node-version: 14.x
+          node-version: 18.x
       - name: Check out repo
         uses: actions/checkout@v2
         with:

.gitignore

@@ -1,7 +1,8 @@
 node_modules/
 dist/
-.env*
-!.env.example
-.j1-integration
+.j1-integration/
+.j1-integration-cache/
+.env
 .eslintcache
-.npmrc
+tsconfig.tsbuildinfo
+.npmrc

.node-version

@@ -1 +1 @@
-14
+18

.prettierignore

@@ -2,4 +2,5 @@ dist
 coverage/
 .j1-integration
 .j1-integration-cache
-CHANGELOG.md
+.gitleaks.yml
+CHANGELOG.md

Dockerfile

@@ -1,11 +1,25 @@
-FROM node:14-alpine
+FROM node:18-bullseye-slim as builder
 
 ENV JUPITERONE_INTEGRATION_DIR=/opt/jupiterone/integration
 
-COPY package.json yarn.lock tsconfig.json LICENSE ${JUPITERONE_INTEGRATION_DIR}/
+COPY package.json yarn.lock tsconfig.dist.json tsconfig.json LICENSE ${JUPITERONE_INTEGRATION_DIR}/
 COPY src/ ${JUPITERONE_INTEGRATION_DIR}/src
+WORKDIR  ${JUPITERONE_INTEGRATION_DIR}
+RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/api/lists/*
+RUN yarn install
+RUN yarn build:docker
+
 
+FROM node:18-bullseye-slim
+ENV JUPITERONE_INTEGRATION_DIR=/opt/jupiterone/integration
+COPY --from=builder --chown=node:node ${JUPITERONE_INTEGRATION_DIR}/dist ${JUPITERONE_INTEGRATION_DIR}
+COPY --from=builder --chown=node:node ${JUPITERONE_INTEGRATION_DIR}/yarn.lock ${JUPITERONE_INTEGRATION_DIR}
+COPY scripts/ ${JUPITERONE_INTEGRATION_DIR}/scripts
 WORKDIR ${JUPITERONE_INTEGRATION_DIR}
-RUN yarn install
+RUN apt-get update && apt-get install -y python3
+RUN yarn install --production --fronzen-lockfile --cache-folder ./ycache && yarn global add --cache-folder ./ycache @jupiterone/integration-sdk-cli && rm -rf ./ycache  && chown -R node:node /opt/jupiterone
+RUN export PATH="$(yarn global bin):$PATH"
+
+USER node
 
-ENTRYPOINT /usr/local/bin/yarn --silent j1-integration run -i ${INTEGRATION_INSTANCE_ID} --disable-schema-validation --api-base-url ${JUPITERONE_API_BASE_URL:-https://api.us.jupiterone.io} --account ${JUPITERONE_ACCOUNT} --api-key ${JUPITERONE_API_KEY}
+CMD ["sh", "scripts/execute.sh"]

README.md

@@ -1,90 +1,85 @@
 # JupiterOne Integration
 
-## Development Environment
+Learn about the data ingested, benefits of this integration, and how to use it
+with JupiterOne in the [integration documentation](docs/jupiterone.md).
 
-### Prerequisites
-
-You must have Node.JS installed to run this project. If you don't already have
-it installed, you can can download the installer
-[here](https://nodejs.org/en/download/). You can alternatively install Node.JS
-using a version manager like [fnm](https://github.com/Schniz/fnm) or
-[nvm](https://github.com/nvm-sh/nvm).
-
-### Setup
+## Development
 
-#### Installing dependencies
-
-First, you'll need to
-[install `yarn`](https://yarnpkg.com/getting-started/install). Then, from the
-root of this project, run `yarn install` to install dependencies.
+### Prerequisites
 
-#### Loading configuration
+1. Install [Node.js](https://nodejs.org/) using the
+   [installer](https://nodejs.org/en/download/) or a version manager such as
+   [nvm](https://github.com/nvm-sh/nvm) or [fnm](https://github.com/Schniz/fnm).
+2. Install [`yarn`](https://yarnpkg.com/getting-started/install) or
+   [`npm`](https://github.com/npm/cli#installation) to install dependencies.
+3. Install dependencies with `yarn install`.
+4. Register an account in the system this integration targets for ingestion and
+   obtain API credentials.
+5. `cp .env.example .env` and add necessary values for runtime configuration.
 
-An integration executes against a configuration that provides credentials and
-any other information necessary to ingest data from the provider. The
-configuration fields are defined in `src/instanceConfigFields.ts` while the
-configuration values are stored in a `.env` file at the root of this project.
-This allows the integration to automatically load the field values and complain
-when they're not provided.
+   When an integration executes, it needs API credentials and any other
+   configuration parameters necessary for its work (provider API credentials,
+   data ingestion parameters, etc.). The names of these parameters are defined
+   by the `IntegrationInstanceConfigFieldMap`in `src/config.ts`. When the
+   integration is executed outside the JupiterOne managed environment (local
+   development or on-prem), values for these parameters are read from Node's
+   `process.env` by converting config field names to constant case. For example,
+   `clientId` is read from `process.env.CLIENT_ID`.
 
-Create a `.env` file at the root of this project and add environment variables
-to match what is in `src/instanceConfigFields.ts`. The `.env` file is ignored by
-git, so you won't have to worry about accidentally pushing credentials.
+   The `.env` file is loaded into `process.env` before the integration code is
+   executed. This file is not required should you configure the environment
+   another way. `.gitignore` is configured to avoid committing the `.env` file.
 
-Given this example configuration:
+### Running the integration
 
-```typescript
-import { IntegrationInstanceConfigFieldMap } from '@jupiterone/integration-sdk-core';
+#### Running directly
 
-const instanceConfigFields: IntegrationInstanceConfigFieldMap = {
-  clientId: {
-    type: 'string',
-  },
-  clientSecret: {
-    type: 'string',
-    mask: true,
-  },
-};
+1. `yarn start` to collect data
+2. `yarn graph` to show a visualization of the collected data
+3. `yarn j1-integration -h` for additional commands
 
-export default instanceConfigFields;
-```
+#### Running with Docker
 
-You would provide a `.env` file like this:
+Create an integration instance for the integration in JupiterOne. With an
+**JupiterOne API Key** scoped to the integration or an API Key with permissions
+to synchronize data and the **Integration Instance ID**:
 
-```bash
-CLIENT_ID="client-id"
-CLIENT_SECRET="supersecret"
-```
+1. `docker build -t $IMAGE_NAME .`
+2. `docker run -e "JUPITERONE_API_KEY=<JUPITERONE_API_KEY>" -e "JUPITERONE_ACCOUNT=<JUPITERONE_ACCOUNT> -e "INTEGRATION_INSTANCE_ID=<INTEGRATION_INSTANCE_ID>" "JUPITERONE_API_BASE_URL=<JUPITERONE_API_BASE_URL>" $IMAGE_NAME`
 
-The snake cased environment variables will automatically be converted and
-applied to the camel cased configuration field. So for example, `CLIENT_ID` will
-apply to the `clientId` config field, `CLIENT_SECRET` will apply to
-`clientSecret`, and `MY_SUPER_SECRET_CONFIGURATION_VALUE` will apply to a
-`mySuperSecretConfigurationValue` configuration field.
+### Making Contributions
 
-## Running the integration
+Start by taking a look at the source code. The integration is basically a set of
+functions called steps, each of which ingests a collection of resources and
+relationships. The goal is to limit each step to as few resource types as
+possible so that should the ingestion of one type of data fail, it does not
+necessarily prevent the ingestion of other, unrelated data. That should be
+enough information to allow you to get started coding!
 
-To start collecting data, run `yarn start` from the root of the project. This
-will load in your configuration from `src/index.ts`.
+See the
+[SDK development documentation](https://github.com/JupiterOne/sdk/blob/main/docs/integrations/development.md)
+for a deep dive into the mechanics of how integrations work.
 
-## Documentation
+See [docs/development.md](docs/development.md) for any additional details about
+developing this integration.
 
-### Development
+## Testing the integration
 
-Please reference the JupiterOne integration
-[development documentation](https://github.com/JupiterOne/sdk/blob/main/docs/integrations/development.md)
-for more information on how to use the SDK.
+Ideally, all major calls to the API and converter functions would be tested. You
+can run the tests with `yarn test`, and you can run the tests as they execute in
+the CI/CD environment with `yarn test:ci` (adds linting and type-checking to
+`yarn test`). If you have a valid runtime configuration, you can run the tests
+with your credentials using `yarn test:env`.
 
-See [docs/development.md](docs/development.md) for details about how to get
-started with developing this integration.
+For more details on setting up tests, and specifically on using recordings to
+simulate API responses, see `test/README.md`.
 
-### Integration usage and resource coverage
+### Changelog
 
-More information about the resources covered by this integration and how to
-setup the integration in JupiterOne can be found in
-[docs/jupiterone.md](docs/jupiterone.md).
+The history of this integration's development can be viewed at
+[CHANGELOG.md](CHANGELOG.md).
 
-### Versioning this project
+## Versioning this project
 
 This project is versioned using [auto](https://intuit.github.io/auto/).
 
@@ -105,7 +100,15 @@ system that we need to publish to NPM and will correctly version based on the
 highest degree of change since the last release, package the project, and
 publish it to NPM.
 
-### Changelog
-
-The history of this integration's development can be viewed at
-[CHANGELOG.md](CHANGELOG.md).
+In order to successfully version and publish to NPM we need access to two
+secrets: a valid NPM token for publishing and a GitHub token for querying the
+repo and pushing version changes. For JupiterOne projects please put in a ticket
+with security to have the repository correctly granted access. For external
+projects, please provide secrets with access to your own NPM and GitHub
+accounts. The secret names should be set to NPM_AUTH_TOKEN and
+AUTO_GITHUB_PAT_TOKEN respectively (or the action can be updated to accommodate
+different naming conventions).
+
+We are not currently using the functionality for auto to update the CHANGELOG.
+As such, please remember to update CHANGELOG.md with the appropriate version,
+date, and changes.

jest.config.js

@@ -1,4 +1 @@
-module.exports = {
-  ...require('@jupiterone/integration-sdk-dev-tools/config/jest'),
-  setupFiles: ['dotenv/config'],
-};
+module.exports = require('@jupiterone/integration-sdk-dev-tools/config/jest');

jupiterone/questions/questions.yaml

@@ -0,0 +1,16 @@
+---
+sourceId: managed:gitlab
+integrationDefinitionId: '${integration_definition_id}'
+questions:
+  []
+  # - id: integration-question-template-replace-me
+  #   title: What kinds of questions will this integration support?
+  #   description:
+  #     TODO Every integration should contribute questions! Please be careful
+  #     to replace this before deploying the integration to JupiterOne.
+  #   queries:
+  #     - name: good
+  #       query: |
+  #         find * with _integrationDefinitionId = '${integration_definition_id}'
+  #   tags:
+  #     - template

package.json

@@ -10,51 +10,61 @@
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "files": [
-    "dist"
+    "src",
+    "jupiterone"
   ],
   "publishConfig": {
     "access": "public"
   },
+  "engines": {
+    "node": ">=18.0.0 <19.x"
+  },
   "scripts": {
     "start": "j1-integration collect",
     "graph": "j1-integration visualize",
+    "graph:types": "j1-integration visualize-types",
+    "graph:spec": "j1-integration visualize-types --project-path docs/spec --output-file ./.j1-integration/types-graph/index.spec.html",
+    "graph:dependencies": "j1-integration visualize-dependencies",
+    "validate:questions:dry": "j1-integration validate-question-file --dry-run",
+    "validate:questions": "j1-integration validate-question-file -a $MANAGED_QUESTIONS_JUPITERONE_ACCOUNT_ID -k $MANAGED_QUESTIONS_JUPITERONE_API_KEY",
     "lint": "eslint . --cache --fix --ext .ts,.tsx",
-    "format": "prettier --write '**/*.{ts,js,json,css,md,yml}'",
-    "format:check": "prettier --check '**/*.{ts,js,json,css,md,yml}'",
+    "format": "prettier --write \"**/*.{ts,js,json,css,md,yml}\"",
+    "format:check": "prettier --check \"**/*.{ts,js,json,css,md,yml}\"",
     "type-check": "tsc",
     "test": "jest",
-    "test:env": "LOAD_ENV=1 jest",
+    "test:env": "LOAD_ENV=1 yarn test",
     "test:ci": "yarn format:check && yarn lint && yarn type-check && yarn test",
-    "prebuild": "yarn test:ci",
-    "prepush": "yarn test:ci",
-    "build": "tsc -p tsconfig.dist.json --declaration",
-    "prepack": "yarn build"
+    "build": "tsc -p tsconfig.dist.json --declaration && cp README.md dist/README.md && cp -r jupiterone/ dist/jupiterone/",
+    "build:docker": "tsc --declaration false --emitDeclarationOnly false -p tsconfig.dist.json",
+    "prepush": "yarn format:check && yarn lint && yarn type-check && jest --changedSince main",
+    "postversion": "cp package.json ./dist/package.json"
   },
   "dependencies": {
     "@lifeomic/attempt": "^3.0.3",
     "node-fetch": "^2.6.1",
     "uuid": "^8.3.2"
   },
   "devDependencies": {
-    "@jupiterone/integration-sdk-core": "^8.33.1",
-    "@jupiterone/integration-sdk-dev-tools": "^8.33.1",
-    "@jupiterone/integration-sdk-testing": "^8.33.1",
+    "@jupiterone/integration-sdk-core": "^13.2.0",
+    "@jupiterone/integration-sdk-dev-tools": "^13.2.0",
+    "@jupiterone/integration-sdk-testing": "^13.2.0",
+    "@types/node": "^20.14.10",
     "@types/node-fetch": "^2.5.9",
     "@types/uuid": "^8.3.0",
     "auto": "^10.37.4",
     "dotenv": "^16.0.3",
     "type-fest": "^0.18.1"
   },
   "peerDependencies": {
-    "@jupiterone/integration-sdk-core": "^8.33.1"
+    "@jupiterone/integration-sdk-core": "^13.2.0"
   },
   "auto": {
     "plugins": [
       [
         "npm",
         {
           "setRcToken": false,
-          "publishFolder": "."
+          "publishFolder": "./dist"
         }
       ],
       "released"

src/constants.ts

@@ -26,7 +26,7 @@ export const Entities = {
   ACCOUNT: {
     resourceName: 'Account',
     _type: 'gitlab_account',
-    _class: 'Account',
+    _class: ['Account'],
   },
   USER: {
     resourceName: 'User',
@@ -36,7 +36,7 @@ export const Entities = {
   GROUP: {
     resourceName: 'Group',
     _type: 'gitlab_group',
-    _class: 'Group',
+    _class: ['Group'],
   },
   FINDING: {
     resourceName: 'Finding',

src/converters/index.ts

@@ -33,6 +33,7 @@ export function createAccountEntity(
         version: systemVersion.version,
         revision: systemVersion.revision,
         enterprise: systemVersion.enterprise,
+        vendor: 'GitLab',
       },
     },
   });
@@ -193,6 +194,7 @@ export function createMergeRequestEntity(
 
 export function createMergeRequestCommitEntity(
   mergeRequestCommit: GitLabMergeCommitRequest,
+  sourceBranch: string,
 ): Entity {
   const key = createCommitIdentifier(mergeRequestCommit.id);
 
@@ -220,6 +222,7 @@ export function createMergeRequestCommitEntity(
         committerEmail: mergeRequestCommit.committer_email,
         authorName: mergeRequestCommit.author_name,
         authorEmail: mergeRequestCommit.author_email,
+        branch: sourceBranch,
       },
     },
   });

src/index.ts

@@ -5,8 +5,9 @@ import { integrationSteps } from './steps';
 import { GitlabIntegrationConfig } from './types';
 import validateInvocation from './validateInvocation';
 
-export const invocationConfig: IntegrationInvocationConfig<GitlabIntegrationConfig> = {
-  instanceConfigFields,
-  validateInvocation,
-  integrationSteps,
-};
+export const invocationConfig: IntegrationInvocationConfig<GitlabIntegrationConfig> =
+  {
+    instanceConfigFields,
+    validateInvocation,
+    integrationSteps,
+  };