Skip to content

Commit 82a568c

Browse files
daneryldaneryl
committed
multi tenant db, WIP
1 parent a3a1dc7 commit 82a568c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+990
-533
lines changed

‎app/api/auth/routes.js

+2-4
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,13 @@
1-
/** @format */
2-
31
import Joi from 'joi';
42
import cookieParser from 'cookie-parser';
53
import mongoConnect from 'connect-mongo';
6-
import mongoose from 'mongoose';
74
import passport from 'passport';
85
import session from 'express-session';
96
import uniqueID from 'shared/uniqueID';
107
import svgCaptcha from 'svg-captcha';
118
import settings from 'api/settings';
129
import urljoin from 'url-join';
10+
import { DB } from 'api/odm';
1311

1412
import { validation } from '../utils';
1513

@@ -24,7 +22,7 @@ export default app => {
2422
session({
2523
secret: app.get('env') === 'production' ? uniqueID() : 'harvey&lola',
2624
store: new MongoStore({
27-
mongooseConnection: mongoose.connection,
25+
mongooseConnection: DB.getConnection(),
2826
}),
2927
resave: false,
3028
saveUninitialized: false,

‎app/api/auth/specs/routes.spec.js

+3-5
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
/** @format */
2-
31
import express from 'express';
42
import bodyParser from 'body-parser';
53
import request from 'supertest';
@@ -20,12 +18,12 @@ describe('Auth Routes', () => {
2018
let app;
2119

2220
beforeEach(async () => {
23-
routes = instrumentRoutes(authRoutes);
2421
await db.clearAllAndLoad(fixtures);
22+
routes = instrumentRoutes(authRoutes);
2523
});
2624

27-
afterAll(done => {
28-
db.disconnect().then(done);
25+
afterAll(async () => {
26+
await db.disconnect();
2927
});
3028

3129
describe('/login', () => {

‎app/api/auth2fa/routes.ts

-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
11
import Joi from 'joi';
2-
import { Application } from 'express';
3-
42
import { Application } from 'express';
53
import needsAuthorization from 'api/auth/authMiddleware';
64
import * as usersUtils from 'api/auth2fa/usersUtils';

‎app/api/config.ts

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import { Tenant } from './odm/tenantContext';
2+
3+
export const config: {
4+
DBHOST: string;
5+
defaultTenant: Tenant;
6+
} = {
7+
DBHOST: process.env.DBHOST ? `mongodb://${process.env.DBHOST}/` : 'mongodb://localhost/',
8+
9+
defaultTenant: {
10+
name: 'default',
11+
dbName: process.env.DATABASE_NAME || 'uwazi_development',
12+
indexName: process.env.INDEX_NAME || 'uwazi_development',
13+
},
14+
};

‎app/api/config/elasticIndexes.js

+12-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,19 @@
1+
import { tenants } from 'api/odm/tenantContext';
2+
13
const { DATABASE_NAME, INDEX_NAME } = process.env;
24

3-
// Keep in sync with database.js!
4-
export default {
5+
const indexes = {
56
demo: INDEX_NAME || DATABASE_NAME || 'uwazi_demo',
67
development: INDEX_NAME || DATABASE_NAME || 'uwazi_development',
78
testing: INDEX_NAME || DATABASE_NAME || 'testing',
89
production: INDEX_NAME || DATABASE_NAME || 'uwazi_development',
10+
getIndex: () => {
11+
const tenant = tenants.current();
12+
if (tenant === 'default') {
13+
return indexes.index;
14+
}
15+
return tenant;
16+
},
917
};
18+
19+
export default indexes;

‎app/api/csv/csvLoader.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import csv, { CSVRow } from './csv';
1212
import importFile from './importFile';
1313
import { importEntity, translateEntity } from './importEntity';
1414
import { extractEntity, toSafeName } from './entityRow';
15+
import { ObjectId } from 'mongodb';
1516

1617
export class CSVLoader extends EventEmitter {
1718
stopOnError: boolean;
@@ -39,7 +40,7 @@ export class CSVLoader extends EventEmitter {
3940
}
4041
}
4142

42-
async load(csvPath: string, templateId: string, options = { language: 'en' }) {
43+
async load(csvPath: string, templateId: ObjectId | string, options = { language: 'en' }) {
4344
const template = await templates.getById(templateId);
4445
if (!template) {
4546
throw new Error('template not found!');

‎app/api/entities/entities.js

+5-5
Original file line numberDiff line numberDiff line change
@@ -524,7 +524,7 @@ export default {
524524

525525
let dbUpdate = Promise.resolve();
526526
if (actions.$unset || actions.$rename) {
527-
dbUpdate = model.db.updateMany({ template: template._id }, actions);
527+
dbUpdate = model.updateMany({ template: template._id }, actions);
528528
}
529529

530530
await dbUpdate;
@@ -620,7 +620,7 @@ export default {
620620
});
621621

622622
const entitiesToReindex = await this.get(query, { _id: 1 });
623-
await model.db.updateMany(query, { $set: changes });
623+
await model.updateMany(query, { $set: changes });
624624
return search.indexEntities({ _id: { $in: entitiesToReindex.map(e => e._id.toString()) } });
625625
},
626626

@@ -645,7 +645,7 @@ export default {
645645
return;
646646
}
647647
const entities = await this.get(query, { _id: 1 });
648-
await model.db.updateMany(query, { $pull: changes });
648+
await model.updateMany(query, { $pull: changes });
649649
await search.indexEntities({ _id: { $in: entities.map(e => e._id.toString()) } }, null, 1000);
650650
},
651651

@@ -681,7 +681,7 @@ export default {
681681

682682
await Promise.all(
683683
properties.map(property =>
684-
model.db.update(
684+
model.updateMany(
685685
{ language: restrictLanguage, [`metadata.${property.name}.value`]: valueId },
686686
{
687687
$set: Object.keys(changes).reduce(
@@ -692,7 +692,7 @@ export default {
692692
{}
693693
),
694694
},
695-
{ arrayFilters: [{ 'valueObject.value': valueId }], multi: true }
695+
{ arrayFilters: [{ 'valueObject.value': valueId }] }
696696
)
697697
)
698698
);

‎app/api/entities/entitiesModel.ts

-8
Original file line numberDiff line numberDiff line change
@@ -40,14 +40,6 @@ const mongoSchema = new mongoose.Schema(
4040
mongoSchema.index({ title: 'text' }, { language_override: 'mongoLanguage' });
4141

4242
const Model = instanceModel<EntitySchema>('entities', mongoSchema);
43-
Model.db.collection.dropIndex('title_text', () => {
44-
// We deliberately kick this promise into the void and ignore the result,
45-
// because it's usually fast and we can't await here...
46-
Model.db.ensureIndexes().then(
47-
() => {},
48-
() => {}
49-
);
50-
});
5143

5244
const supportedLanguages = [
5345
'da',

‎app/api/entities/specs/entities.spec.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -892,12 +892,12 @@ describe('entities', () => {
892892
});
893893

894894
it('should do nothing when there is no changed or deleted properties', done => {
895-
spyOn(entitiesModel.db, 'updateMany');
895+
spyOn(entitiesModel, 'updateMany');
896896

897897
entities
898898
.updateMetadataProperties(currentTemplate, currentTemplate)
899899
.then(() => {
900-
expect(entitiesModel.db.updateMany).not.toHaveBeenCalled();
900+
expect(entitiesModel.updateMany).not.toHaveBeenCalled();
901901
done();
902902
})
903903
.catch(catchErrors(done));

‎app/api/migrations/migrate.js

-6
This file was deleted.

‎app/api/migrations/migrate.ts

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
import { DB } from 'api/odm';
2+
import { tenants } from 'api/odm/tenantContext';
3+
4+
import migrator from './migrator';
5+
6+
const run = async () => {
7+
await DB.connect();
8+
await tenants.run(async () => {
9+
await migrator.migrate();
10+
});
11+
await DB.disconnect();
12+
};
13+
14+
run().catch(async e => {
15+
await DB.disconnect();
16+
throw e;
17+
});

‎app/api/odm/DB.ts

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
import mongoose, { Connection } from 'mongoose';
2+
import { config } from 'api/config';
3+
4+
import { tenants } from './tenantContext';
5+
6+
type dbAuth = {
7+
user: string;
8+
pass: string;
9+
};
10+
11+
let connection: Connection;
12+
13+
// setting this on createConnection directly is not working, maybe mongoose bug?
14+
mongoose.set('useCreateIndex', true);
15+
16+
const DB = {
17+
async connect(uri: string = config.DBHOST, auth?: dbAuth) {
18+
connection = await mongoose.createConnection(uri, {
19+
...auth,
20+
useUnifiedTopology: true,
21+
useNewUrlParser: true,
22+
});
23+
24+
tenants.add(config.defaultTenant);
25+
26+
return this.getConnection();
27+
},
28+
29+
async disconnect() {
30+
return mongoose.disconnect();
31+
},
32+
33+
connectionForDB(dbName: string) {
34+
//mongoose types not updated yet for useCache
35+
//@ts-ignore
36+
return this.getConnection().useDb(dbName, { useCache: true });
37+
},
38+
39+
getConnection() {
40+
return connection;
41+
},
42+
};
43+
44+
export { DB };

‎app/api/odm/MultiTenantMongooseModel.ts

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
import mongoose, { Schema, UpdateQuery, ModelUpdateOptions } from 'mongoose';
2+
import { createError } from 'api/utils';
3+
4+
import { WithId, UwaziFilterQuery } from './models';
5+
import { tenants, Tenant } from './tenantContext';
6+
import { DB } from './DB';
7+
8+
class MultiTenantMongooseModel<T> {
9+
dbs: { [k: string]: mongoose.Model<WithId<T> & mongoose.Document> };
10+
11+
constructor(collectionName: string, schema: Schema) {
12+
this.dbs = {};
13+
14+
Object.keys(tenants.tenants).forEach(tenantName => {
15+
if (!this.dbs[tenantName]) {
16+
this.dbs[tenantName] = DB.connectionForDB(tenants.tenants[tenantName].dbName).model<
17+
WithId<T> & mongoose.Document
18+
>(collectionName, schema);
19+
}
20+
});
21+
22+
tenants.on('newTenant', (tenant: Tenant) => {
23+
if (!this.dbs[tenant.name]) {
24+
this.dbs[tenant.name] = DB.connectionForDB(tenant.dbName).model<WithId<T> & mongoose.Document>(
25+
collectionName,
26+
schema
27+
);
28+
}
29+
});
30+
}
31+
32+
dbForCurrentTenant() {
33+
const currentTenantName = tenants.current().name;
34+
if (!this.dbs[currentTenantName]) {
35+
throw createError(`tenant "${currentTenantName}" db connection is not available`, 503);
36+
}
37+
38+
return this.dbs[currentTenantName];
39+
}
40+
41+
findById(id: any | string | number, select = {}) {
42+
return this.dbForCurrentTenant().findById(id, select, { lean: true });
43+
}
44+
45+
find(query: UwaziFilterQuery<T>, select = '', options = {}) {
46+
return this.dbForCurrentTenant().find(query, select, options);
47+
}
48+
49+
async findOneAndUpdate(
50+
query: UwaziFilterQuery<T>,
51+
update: Readonly<Partial<T>> & { _id?: any },
52+
options: any = {}
53+
) {
54+
return this.dbForCurrentTenant().findOneAndUpdate(query, update, options);
55+
}
56+
57+
async create(data: Readonly<Partial<T>> & { _id?: any }) {
58+
return this.dbForCurrentTenant().create(data);
59+
}
60+
61+
async _updateMany(
62+
conditions: UwaziFilterQuery<T>,
63+
doc: UpdateQuery<T>,
64+
options: ModelUpdateOptions = {}
65+
) {
66+
return this.dbForCurrentTenant().updateMany(conditions, doc, options);
67+
}
68+
69+
async findOne(conditions: UwaziFilterQuery<T>, projection: any) {
70+
return this.dbForCurrentTenant().findOne(conditions, projection);
71+
}
72+
73+
async replaceOne(conditions: UwaziFilterQuery<T>, replacement: any) {
74+
return this.dbForCurrentTenant().replaceOne(conditions, replacement);
75+
}
76+
77+
async countDocuments(query: UwaziFilterQuery<T> = {}) {
78+
return this.dbForCurrentTenant().countDocuments(query);
79+
}
80+
81+
async distinct(field: string, query: UwaziFilterQuery<T> = {}) {
82+
return this.dbForCurrentTenant().distinct(field, query);
83+
}
84+
85+
async deleteMany(query: UwaziFilterQuery<T>) {
86+
return this.dbForCurrentTenant().deleteMany(query);
87+
}
88+
89+
async aggregate(aggregations?: any[]) {
90+
return this.dbForCurrentTenant().aggregate(aggregations);
91+
}
92+
93+
async updateOne(conditions: UwaziFilterQuery<T>, doc: UpdateQuery<T>) {
94+
return this.dbForCurrentTenant().updateOne(conditions, doc);
95+
}
96+
}
97+
98+
export { MultiTenantMongooseModel };

‎app/api/odm/index.ts

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
/** @format */
2-
31
export * from './model';
42
export * from './models';
3+
export { DB } from './DB';

0 commit comments

Comments
 (0)