test: support probe e2e tests (#509)

* feat: move to single setup file

* feat: make e2e folder independent

* feat: move e2e tests

* feat: move .mocharc e2e file inside e2e folder

* feat: rename mocharc

* feat: use ralative paths in e2e tests

* fix: sync workers

* feat: await all workers in other utils

* feat: use src logger

* feat: use src client

* feat: calculate number of responses dynamically

* feat: remove adopted-probe.ts dependency

* feat: pass number of processes to api

* feat: small updates

* feat: move .mocharc.e2e.cjs back to the root folder

* feat: add blacklist e2e tests

* fix: fix comment docker command

.mocharc.e2e.cjs

@@ -0,0 +1,43 @@
+const path = require('path');
+
+module.exports = {
+	'exit': true,
+	'timeout': 20000,
+	'check-leaks': true,
+	'file': [
+		path.join(__dirname, 'test/e2e/setup.ts'),
+	],
+	'spec': [
+		path.join(__dirname, 'test/e2e/cases/**/*.test.ts'),
+	],
+	'node-option': [
+		'experimental-specifier-resolution=node',
+		'loader=ts-node/esm',
+	],
+	'globals': [
+		'__extends',
+		'__assign',
+		'__rest',
+		'__decorate',
+		'__param',
+		'__metadata',
+		'__awaiter',
+		'__generator',
+		'__exportStar',
+		'__createBinding',
+		'__values',
+		'__read',
+		'__spread',
+		'__spreadArrays',
+		'__spreadArray',
+		'__await',
+		'__asyncGenerator',
+		'__asyncDelegator',
+		'__asyncValues',
+		'__makeTemplateObject',
+		'__importStar',
+		'__importDefault',
+		'__classPrivateFieldGet',
+		'__classPrivateFieldSet',
+	],
+};

knexfile.js

@@ -1,6 +1,10 @@
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
 import _ from 'lodash';
 import config from 'config';
 
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
 const dbConfig = config.get('db');
 
 /**
@@ -19,7 +23,10 @@ export default _.merge({}, ...[ 'development', 'production', 'staging', 'test' ]
 			},
 			acquireConnectionTimeout: 10000,
 			seeds: {
-				directory: `./seeds/${environment}`,
+				directory: path.join(__dirname, `./seeds/${environment}`),
+			},
+			migrations: {
+				directory: path.join(__dirname, `./migrations`),
 			},
 		},
 	};

package.json

@@ -137,10 +137,9 @@
 		"test:portman": "TEST_DONT_RESTART_WORKERS=1 start-test 'npm run start:test' http://localhost:3000/health 'npm run test:portman:create && npm run test:portman:run' 2> /dev/null || E=$?; rm -rf tmp; exit $E;",
 		"test:portman:create": "mkdir -p tmp && redocly bundle public/v1/spec.yaml > tmp/spec.yaml && portman --cliOptionsFile test/tests/contract/portman-cli.json",
 		"test:portman:run": "newman run tmp/converted/globalpingApi.json -e test/tests/contract/newman-env.json --ignore-redirects",
-		"test:e2e": "npm run test:e2e:docker && npm run test:e2e:cases",
-		"test:e2e:api": "NODE_ENV=test knex migrate:latest && PORT=80 NODE_ENV=test TEST_MODE=e2e NEW_RELIC_ENABLED=false tsx src/index.ts",
+		"test:e2e": "npm run test:e2e:docker && npm run test:e2e:run",
 		"test:e2e:docker": "docker build -t globalping-api-e2e . && docker build -t globalping-probe-e2e https://github.com/jsdelivr/globalping-probe.git",
-		"test:e2e:cases": "TS_NODE_TRANSPILE_ONLY=true NODE_ENV=test mocha --config .mocharc.e2e.json"
+		"test:e2e:run": "TS_NODE_TRANSPILE_ONLY=true NODE_ENV=test mocha --config .mocharc.e2e.cjs"
 	},
 	"lint-staged": {
 		"*.{cjs,js,json,ts}": "eslint --cache --fix"

test/tests/e2e/cases/adopted-probes.test.ts → test/e2e/cases/adopted-probes.test.ts

@@ -1,9 +1,10 @@
 import got from 'got';
 import { expect } from 'chai';
-import { client } from '../../../../src/lib/sql/client.js';
-import { ADOPTED_PROBES_TABLE } from '../../../../src/lib/adopted-probes.js';
+import { client } from '../../../src/lib/sql/client.js';
 import { waitProbeInCity } from '../utils.js';
 
+const ADOPTED_PROBES_TABLE = 'gp_adopted_probes';
+
 describe('adopted probe', () => {
 	before(async function () {
 		this.timeout(80000);

test/tests/e2e/cases/dns.test.ts → test/e2e/cases/dns.test.ts

@@ -33,4 +33,25 @@ describe('dns mesurement', () => {
 		expect(response.body.results[0].result.hops.length).to.be.above(0);
 		expect(response).to.matchApiSchema();
 	});
+
+	it('should return 400 for blacklisted target', async () => {
+		const response = await got.post('http://localhost:80/v1/measurements', { json: {
+			target: 'dpd.96594345154.xyz',
+			type: 'dns',
+		}, throwHttpErrors: false });
+
+		expect(response.statusCode).to.equal(400);
+	});
+
+	it('should return 400 for blacklisted resolver', async () => {
+		const response = await got.post('http://localhost:80/v1/measurements', { json: {
+			target: 'www.jsdelivr.com',
+			type: 'dns',
+			measurementOptions: {
+				resolver: '113.24.166.134',
+			},
+		}, throwHttpErrors: false });
+
+		expect(response.statusCode).to.equal(400);
+	});
 });

test/tests/e2e/cases/http.test.ts → test/e2e/cases/http.test.ts

@@ -35,4 +35,39 @@ describe('http mesurement', () => {
 		expect(response.body.results[0].result.rawBody.length).to.be.above(0);
 		expect(response).to.matchApiSchema();
 	});
+
+	it('should return 400 for blacklisted target', async () => {
+		const response = await got.post('http://localhost:80/v1/measurements', { json: {
+			target: 'dpd.96594345154.xyz',
+			type: 'http',
+		}, throwHttpErrors: false });
+
+		expect(response.statusCode).to.equal(400);
+	});
+
+	it('should return 400 for blacklisted request host', async () => {
+		const response = await got.post('http://localhost:80/v1/measurements', { json: {
+			target: 'www.jsdelivr.com',
+			type: 'http',
+			measurementOptions: {
+				request: {
+					host: 'dpd.96594345154.xyz',
+				},
+			},
+		}, throwHttpErrors: false });
+
+		expect(response.statusCode).to.equal(400);
+	});
+
+	it('should return 400 for blacklisted resolver', async () => {
+		const response = await got.post('http://localhost:80/v1/measurements', { json: {
+			target: 'www.jsdelivr.com',
+			type: 'http',
+			measurementOptions: {
+				resolver: '113.24.166.134',
+			},
+		}, throwHttpErrors: false });
+
+		expect(response.statusCode).to.equal(400);
+	});
 });

test/tests/e2e/cases/mtr.test.ts → test/e2e/cases/mtr.test.ts

@@ -15,4 +15,13 @@ describe('mtr mesurement', () => {
 		expect(response.body.results[0].result.status).to.equal('finished');
 		expect(response).to.matchApiSchema();
 	});
+
+	it('should return 400 for blacklisted target', async () => {
+		const response = await got.post('http://localhost:80/v1/measurements', { json: {
+			target: 'dpd.96594345154.xyz',
+			type: 'mtr',
+		}, throwHttpErrors: false });
+
+		expect(response.statusCode).to.equal(400);
+	});
 });

test/tests/e2e/cases/ping.test.ts → test/e2e/cases/ping.test.ts

@@ -15,4 +15,13 @@ describe('ping mesurement', () => {
 		expect(response.body.results[0].result.status).to.equal('finished');
 		expect(response).to.matchApiSchema();
 	});
+
+	it('should return 400 for blacklisted target', async () => {
+		const response = await got.post('http://localhost:80/v1/measurements', { json: {
+			target: 'dpd.96594345154.xyz',
+			type: 'ping',
+		}, throwHttpErrors: false });
+
+		expect(response.statusCode).to.equal(400);
+	});
 });

test/tests/e2e/cases/traceroute.test.ts → test/e2e/cases/traceroute.test.ts

@@ -15,4 +15,13 @@ describe('traceroute mesurement', () => {
 		expect(response.body.results[0].result.status).to.equal('finished');
 		expect(response).to.matchApiSchema();
 	});
+
+	it('should return 400 for blacklisted target', async () => {
+		const response = await got.post('http://localhost:80/v1/measurements', { json: {
+			target: 'dpd.96594345154.xyz',
+			type: 'traceroute',
+		}, throwHttpErrors: false });
+
+		expect(response.statusCode).to.equal(400);
+	});
 });

test/tests/e2e/docker.ts → test/e2e/docker.ts

@@ -1,7 +1,7 @@
 import Docker from 'dockerode';
 import config from 'config';
 
-import { scopedLogger } from '../../../src/lib/logger.js';
+import { scopedLogger } from '../../src/lib/logger.js';
 
 const logger = scopedLogger('docker-manager');
 
@@ -16,6 +16,7 @@ class DockerManager {
 		let networkMode = 'host';
 		let redisUrl = config.get<string>('redis.url');
 		let dbConnectionHost = config.get<string>('db.connection.host');
+		const processes = config.get<string>('server.processes');
 
 		const isLinux = await this.isLinuxHost();
 
@@ -35,6 +36,7 @@ class DockerManager {
 				'NEW_RELIC_ENABLED=false',
 				`REDIS_URL=${redisUrl}`,
 				`DB_CONNECTION_HOST=${dbConnectionHost}`,
+				`SERVER_PROCESSES=${processes}`,
 			],
 			HostConfig: {
 				PortBindings: {
@@ -59,7 +61,7 @@ class DockerManager {
 			apiHost = apiHost.replace('localhost', 'host.docker.internal');
 		}
 
-		// docker run -e API_HOST=ws://host.docker.internal:80 --name globalping-probe-e2e globalping-api-e2e
+		// docker run -e API_HOST=ws://host.docker.internal:80 --name globalping-probe-e2e globalping-probe-e2e
 		const container = await this.docker.createContainer({
 			Image: 'globalping-probe-e2e',
 			name: 'globalping-probe-e2e',

test/e2e/setup.ts

@@ -0,0 +1,55 @@
+import config from 'config';
+import Bluebird from 'bluebird';
+import type { Knex } from 'knex';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import chai from 'chai';
+import { createClient } from 'redis';
+
+import { waitProbeToConnect } from './utils.js';
+import chaiOas from '../plugins/oas/index.js';
+import { docker } from './docker.js';
+import { client as sql } from '../../src/lib/sql/client.js';
+
+before(async () => {
+	chai.use(await chaiOas({ specPath: path.join(fileURLToPath(new URL('.', import.meta.url)), '../../public/v1/spec.yaml') }));
+
+	await docker.removeProbeContainer();
+	await docker.removeApiContainer();
+
+	await flushRedis();
+
+	await dropAllTables(sql);
+	await sql.migrate.latest();
+	await sql.seed.run();
+
+	await docker.createApiContainer();
+	await docker.createProbeContainer();
+
+	await waitProbeToConnect();
+});
+
+after(async () => {
+	await docker.removeProbeContainer();
+	await docker.removeApiContainer();
+});
+
+const dropAllTables = async (sql: Knex) => {
+	const allTables = (await sql('information_schema.tables')
+		.whereRaw(`table_schema = database()`)
+		.select(`table_name as table`)
+	).map(({ table }: { table: string }) => table);
+	await Bluebird.map(allTables, table => sql.schema.raw(`drop table \`${table}\``));
+};
+
+const flushRedis = async () => {
+	const dbs = [ 0, 1, 2 ];
+	await Promise.all(dbs.map(async (database) => {
+		const client = createClient({
+			...config.util.toObject(config.get('redis')),
+			database,
+		});
+		await client.connect();
+		await client.flushDb();
+	}));
+};