Skip to content

Commit 7fd3fbd

Browse files
MrBartusekMrBartusek
committed
Add s3 cache
1 parent 5edb86c commit 7fd3fbd

12 files changed

+120
-34
lines changed

‎apps/api/src/app.module.ts

+2
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import { WarehousesModule } from './models/warehouses/warehouses.module';
1313
import { S3Module } from './s3/s3.module';
1414
import { ImagesModule } from './images/images.module';
1515
import { RedisModule } from './redis/redis.module';
16+
import { S3CacheModule } from './s3-cache/s3-cache.module';
1617

1718
const FrontendModule = ServeStaticModule.forRoot({
1819
rootPath: join(__dirname, '../../..', 'client', 'dist'),
@@ -31,6 +32,7 @@ const FrontendModule = ServeStaticModule.forRoot({
3132
S3Module,
3233
ImagesModule,
3334
RedisModule,
35+
S3CacheModule,
3436
],
3537
controllers: [AppController],
3638
providers: [AppService],

‎apps/api/src/helpers/utils.ts

+9
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { Logger } from '@nestjs/common';
22
import { SchemaOptions } from '@nestjs/mongoose';
3+
import { Readable } from 'node:stream';
34

45
const { BASE_API_URL, NODE_ENV } = process.env;
56
const logger = new Logger('Utils');
@@ -36,6 +37,14 @@ class Utils {
3637
},
3738
};
3839
}
40+
41+
public static async streamToBuffer(stream: Readable) {
42+
const chunks = [];
43+
for await (const chunk of stream) {
44+
chunks.push(chunk);
45+
}
46+
return Buffer.concat(chunks);
47+
}
3948
}
4049

4150
export default Utils;

‎apps/api/src/images/images.controller.spec.ts

+11-11
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,18 @@ import { ImagesController } from './images.controller';
33
import { ImagesService } from './images.service';
44

55
describe('ImagesController', () => {
6-
let controller: ImagesController;
6+
let controller: ImagesController;
77

8-
beforeEach(async () => {
9-
const module: TestingModule = await Test.createTestingModule({
10-
controllers: [ImagesController],
11-
providers: [ImagesService],
12-
}).compile();
8+
beforeEach(async () => {
9+
const module: TestingModule = await Test.createTestingModule({
10+
controllers: [ImagesController],
11+
providers: [ImagesService],
12+
}).compile();
1313

14-
controller = module.get<ImagesController>(ImagesController);
15-
});
14+
controller = module.get<ImagesController>(ImagesController);
15+
});
1616

17-
it('should be defined', () => {
18-
expect(controller).toBeDefined();
19-
});
17+
it('should be defined', () => {
18+
expect(controller).toBeDefined();
19+
});
2020
});

‎apps/api/src/images/images.controller.ts

+2-10
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,6 @@
1-
import {
2-
Controller,
3-
Get,
4-
NotFoundException,
5-
Param,
6-
Redirect,
7-
HttpRedirectResponse,
8-
Res,
9-
} from '@nestjs/common';
10-
import { ImagesService } from './images.service';
1+
import { Controller, Get, Param, Redirect, Res } from '@nestjs/common';
112
import { Response } from 'express';
3+
import { ImagesService } from './images.service';
124

135
@Controller('images')
146
export class ImagesController {

‎apps/api/src/images/images.module.ts

+3-2
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@ import { S3Module } from 'nestjs-s3';
33
import { ImagesController } from './images.controller';
44
import { ImagesService } from './images.service';
55
import { S3Service } from '../s3/s3.service';
6+
import { S3CacheModule } from '../s3-cache/s3-cache.module';
67

78
@Module({
89
controllers: [ImagesController],
9-
providers: [ImagesService, S3Service],
10-
imports: [S3Module],
10+
providers: [ImagesService],
11+
imports: [S3CacheModule],
1112
})
1213
export class ImagesModule {}

‎apps/api/src/images/images.service.ts

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
import { Injectable } from '@nestjs/common';
22
import { S3Service } from '../s3/s3.service';
3+
import { S3CacheService } from '../s3-cache/s3-cache.service';
34

45
@Injectable()
56
export class ImagesService {
6-
constructor(private readonly s3Service: S3Service) {}
7+
constructor(private readonly s3CacheService: S3CacheService) {}
78

89
getObject(key: string) {
9-
return this.s3Service.getObject(key);
10+
return this.s3CacheService.getObject(key);
1011
}
1112
}

‎apps/api/src/redis/redis.service.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ describe('RedisService', () => {
2323
});
2424

2525
it('should retrieve client', async () => {
26-
const client = service.getClient();
26+
const client = service.client;
2727
await client.set('test', 'test');
2828
expect(setSpy).toBeCalledWith('test', 'test');
2929
});

‎apps/api/src/redis/redis.service.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ import { Inject, Injectable } from '@nestjs/common';
22
import Redis from 'ioredis';
33
@Injectable()
44
export class RedisService {
5-
constructor(@Inject('CLIENT') private readonly client: Redis) {}
5+
constructor(@Inject('CLIENT') private readonly _client: Redis) {}
66

7-
getClient(): Redis {
8-
return this.client;
7+
get client(): Redis {
8+
return this._client;
99
}
1010
}

‎apps/api/src/s3-cache/s3-cache.module.ts

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
import { Module } from '@nestjs/common';
2+
import { S3Module } from 'nestjs-s3';
3+
import { RedisModule } from '../redis/redis.module';
4+
import { S3Service } from '../s3/s3.service';
5+
import { S3CacheService } from './s3-cache.service';
6+
7+
@Module({
8+
imports: [S3Module, RedisModule],
9+
providers: [S3CacheService, S3Service],
10+
exports: [S3CacheService],
11+
})
12+
export class S3CacheModule {}

‎apps/api/src/s3-cache/s3-cache.service.spec.ts

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import { Test, TestingModule } from '@nestjs/testing';
2+
import { S3CacheService } from './s3-cache.service';
3+
4+
describe('S3CacheService', () => {
5+
let service: S3CacheService;
6+
7+
beforeEach(async () => {
8+
const module: TestingModule = await Test.createTestingModule({
9+
providers: [S3CacheService],
10+
}).compile();
11+
12+
service = module.get<S3CacheService>(S3CacheService);
13+
});
14+
15+
it('should be defined', () => {
16+
expect(service).toBeDefined();
17+
});
18+
});

‎apps/api/src/s3-cache/s3-cache.service.ts

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import { Injectable, Logger } from '@nestjs/common';
2+
import { S3Service } from '../s3/s3.service';
3+
import { RedisService } from '../redis/redis.service';
4+
import { Readable } from 'stream';
5+
import Utils from '../helpers/utils';
6+
7+
const ONE_DAY = 24 * 60 * 60;
8+
const REDIS_PREFIX = 's3-cache:';
9+
const CACHE_TIME = 30 * ONE_DAY;
10+
11+
@Injectable()
12+
export class S3CacheService {
13+
private readonly logger = new Logger(S3CacheService.name);
14+
15+
constructor(
16+
private readonly s3: S3Service,
17+
private readonly redis: RedisService,
18+
) {}
19+
20+
async getObject(key: string): Promise<Readable | null> {
21+
const cachedObject = await this.getCachedObject(key);
22+
if (cachedObject) {
23+
return cachedObject;
24+
}
25+
26+
const object = await this.s3.getObjectBody(key);
27+
if (!object) return null;
28+
29+
await this.cacheObject(key, object);
30+
const newObject = await this.getCachedObject(key);
31+
return newObject;
32+
}
33+
34+
private async getCachedObject(key: string): Promise<Readable | null> {
35+
const redisKey = `${REDIS_PREFIX}${key}`;
36+
37+
const exist = await this.redis.client.exists(redisKey);
38+
if (exist == 0) return null;
39+
40+
const cachedObject = await this.redis.client.getBuffer(redisKey);
41+
const stream = Readable.from(cachedObject);
42+
return stream;
43+
}
44+
45+
private async cacheObject(key: string, object: Readable): Promise<void> {
46+
const redisKey = `${REDIS_PREFIX}${key}`;
47+
48+
const buffer = await Utils.streamToBuffer(object);
49+
await this.redis.client.set(redisKey, buffer);
50+
await this.redis.client.expire(redisKey, CACHE_TIME);
51+
this.logger.log(`Cached s3 object with key "${key}" for ${CACHE_TIME}s`);
52+
}
53+
}

‎apps/api/src/s3/s3.service.ts

+3-5
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,11 @@ import { Readable } from 'node:stream';
1414

1515
const { AWS_BUCKET_NAME } = process.env;
1616

17-
const SINGED_URL_EXPIRE = 24 * 60 * 60;
18-
1917
@Injectable()
2018
export class S3Service {
2119
constructor(@InjectS3() private readonly s3: S3) {}
2220

23-
async uploadFile(file: Express.Multer.File): Promise<string> {
21+
async uploadObject(file: Express.Multer.File): Promise<string> {
2422
const key = this.generateFileKey();
2523
const params: PutObjectCommandInput = {
2624
Bucket: AWS_BUCKET_NAME,
@@ -32,7 +30,7 @@ export class S3Service {
3230
return key;
3331
}
3432

35-
async getObject(key: string) {
33+
async getObjectBody(key: string): Promise<Readable | null> {
3634
const params: GetObjectCommandInput = {
3735
Bucket: AWS_BUCKET_NAME,
3836
Key: key,
@@ -43,7 +41,7 @@ export class S3Service {
4341
return body;
4442
}
4543

46-
async deleteFile(key: string): Promise<void> {
44+
async deleteObject(key: string): Promise<void> {
4745
const params: DeleteObjectCommandInput = {
4846
Bucket: AWS_BUCKET_NAME,
4947
Key: key,

0 commit comments

Comments
 (0)