Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
65 commits
Select commit Hold shift + click to select a range
18fcc79
feat: cache warmer implementation
JivusAyrus Jan 8, 2025
4e6ff40
fix: remove unnecessary code
JivusAyrus Jan 9, 2025
bbc7f87
fix: lint
JivusAyrus Jan 9, 2025
0318842
Merge branch 'main' into suvij/eng-5580-create-new-optimized-clickhou…
Noroth Jan 9, 2025
342c55e
feat(router): implement cache warming from CDN source (#1491)
Noroth Jan 9, 2025
8764ff0
fix: path
JivusAyrus Jan 9, 2025
e66b761
feat: add cdn as an option for the source of cache operations
JivusAyrus Jan 9, 2025
a812141
Merge branch 'main' into suvij/eng-5580-create-new-optimized-clickhou…
Noroth Jan 10, 2025
6598124
Merge branch 'suvij/eng-5580-create-new-optimized-clickhouse-views-fo…
JivusAyrus Jan 10, 2025
822793c
feat: add cache operations ui
JivusAyrus Jan 10, 2025
33887f4
fix: use p90 and order operations by descending order
JivusAyrus Jan 10, 2025
5aa9575
feat: add operation details page
JivusAyrus Jan 10, 2025
0f0271f
chore: remove migrations
JivusAyrus Jan 10, 2025
bb92cc9
Merge branch 'main' of github.com:wundergraph/cosmo into suvij/eng-55…
JivusAyrus Jan 10, 2025
d339d4d
chore: add migrations
JivusAyrus Jan 10, 2025
4d43501
fix: ci
JivusAyrus Jan 10, 2025
3375630
fix: security bot issues
JivusAyrus Jan 10, 2025
223fbf9
fix: url
JivusAyrus Jan 10, 2025
56ceb02
fix: pr suggestions
JivusAyrus Jan 10, 2025
37a61e7
fix: tests
JivusAyrus Jan 10, 2025
45ea000
feat: add option to enable cache warmer
JivusAyrus Jan 11, 2025
4a0bfde
feat: perform cache warming actions only if its enabled
JivusAyrus Jan 11, 2025
300bdb7
fix: rename rpc to configureCacheWarmer
JivusAyrus Jan 13, 2025
7e393bf
fix: pr suggestion
JivusAyrus Jan 13, 2025
ebe4160
Merge branch 'main' of github.com:wundergraph/cosmo into suvij/eng-55…
JivusAyrus Jan 13, 2025
2e60fc2
fix: lint
JivusAyrus Jan 13, 2025
44a135c
fix: ci
JivusAyrus Jan 13, 2025
a7ab522
Merge branch 'main' into suvij/eng-5580-create-new-optimized-clickhou…
Noroth Jan 13, 2025
70c220c
fix: cache warmer configuration ui
JivusAyrus Jan 14, 2025
59cd756
fix: operations table ui
JivusAyrus Jan 14, 2025
b801cce
fix: fetch the operation content of all the hashes in one query
JivusAyrus Jan 14, 2025
7db6963
fix: merge migrations
JivusAyrus Jan 14, 2025
103a21a
fix: remove gql operations migrations
JivusAyrus Jan 14, 2025
ab3a286
fix: revert graphql metrics
JivusAyrus Jan 15, 2025
1c1caf4
fix: cache operations ui
JivusAyrus Jan 15, 2025
5bc7a71
fix: remove cache warmer worker
JivusAyrus Jan 15, 2025
6d44616
fix: check if the org has access to cache warmer
JivusAyrus Jan 15, 2025
dd4e392
fix: remove unnecessary fields from the query
JivusAyrus Jan 15, 2025
42bf8f2
chore: add cache warmer to FeatureIds type
JivusAyrus Jan 15, 2025
c49c3a7
fix: remoce cache warmer worker
JivusAyrus Jan 15, 2025
12474e6
fix: remove subgraph id condition in the mv
JivusAyrus Jan 15, 2025
1b1bab7
fix: check if the org has access to cache-warmer in the rpcs
JivusAyrus Jan 16, 2025
0f38a20
fix: check if the org has access to cahe warmer
JivusAyrus Jan 16, 2025
ed81956
chore: remove obsolte commented code
Noroth Jan 16, 2025
c36be50
feat: add getPersistedOperation function
JivusAyrus Jan 16, 2025
6e96fbd
chore: add tests
JivusAyrus Jan 16, 2025
d25fdeb
fix: lint
JivusAyrus Jan 16, 2025
9abab21
fix: operations with top planning time query
JivusAyrus Jan 16, 2025
50a1995
fix: cache operations ui
JivusAyrus Jan 16, 2025
51ffe16
fix: message
JivusAyrus Jan 16, 2025
94246e6
fix: handle clickhouse optionally
JivusAyrus Jan 16, 2025
6afcc9c
fix: security bug
JivusAyrus Jan 16, 2025
3f3d235
Merge branch 'main' into suvij/eng-5580-create-new-optimized-clickhou…
JivusAyrus Jan 16, 2025
2fe6910
fix: change the status code to upgrade plan
JivusAyrus Jan 16, 2025
c763d86
fix: rpcs
JivusAyrus Jan 16, 2025
b924ccd
fix: remove unused imports
JivusAyrus Jan 16, 2025
73e8df8
chore: use protobuf type
Noroth Jan 16, 2025
c2620c8
feat: add no cache headers to the cdn
JivusAyrus Jan 16, 2025
fcca618
Merge branch 'suvij/eng-5580-create-new-optimized-clickhouse-views-fo…
JivusAyrus Jan 16, 2025
b2c43c1
chore: assign types explicitly
Noroth Jan 16, 2025
ebbf17c
chore: don't allow pushing PO with incorrect values
Noroth Jan 16, 2025
17d20c4
chore: refactor
StarpTech Jan 17, 2025
3d218e4
chore: refactor
StarpTech Jan 17, 2025
616ace9
chore: update lock
StarpTech Jan 17, 2025
ac308b6
chore: fix lint
StarpTech Jan 17, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions cdn-server/cdn/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,36 @@ const draftRouterConfig = (storage: BlobStorage) => {
};
};

const cacheOperations = (storage: BlobStorage) => {
Comment thread
StarpTech marked this conversation as resolved.
return async (c: Context) => {
const organizationId = c.get('authenticatedOrganizationId');
const federatedGraphId = c.get('authenticatedFederatedGraphId');

if (organizationId !== c.req.param('organization_id') || federatedGraphId !== c.req.param('federated_graph_id')) {
return c.text('Bad Request', 400);
}

const key = `${organizationId}/${federatedGraphId}/cache_warmup/operations.json`;
let blobObject: BlobObject;

try {
blobObject = await storage.getObject({ context: c, key, cacheControl: 'no-cache' });
} catch (e: any) {
if (e instanceof BlobNotFoundError) {
return c.notFound();
}
throw e;
}

c.header('Content-Type', 'application/json; charset=UTF-8');
c.header('Cache-Control', 'no-cache, no-store, must-revalidate');

return stream(c, async (stream) => {
await stream.pipe(blobObject.stream);
});
};
};

// eslint-disable-next-line @typescript-eslint/ban-types
export const cdn = <E extends Env, S extends Schema = {}, BasePath extends string = '/'>(
hono: Hono<E, S, BasePath>,
Expand All @@ -235,4 +265,9 @@ export const cdn = <E extends Env, S extends Schema = {}, BasePath extends strin
hono
.use(draftRouterConfigs, jwtMiddleware(opts.authAdmissionJwtSecret))
.get(draftRouterConfigs, draftRouterConfig(opts.blobStorage));

const cacheOperationsPath = '/:organization_id/:federated_graph_id/cache_warmup/operations.json';
hono
.use(cacheOperationsPath, jwtMiddleware(opts.authJwtSecret))
.get(cacheOperationsPath, cacheOperations(opts.blobStorage));
};
97 changes: 97 additions & 0 deletions cdn-server/cdn/test/cdn.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -391,4 +391,101 @@ describe('CDN handlers', () => {
expect(res.status).toBe(404);
});
});

describe('Test cache warmer opeartions handler', async () => {
const federatedGraphId = 'federatedGraphId';
const organizationId = 'organizationId';
const token = await generateToken(organizationId, federatedGraphId, secretKey);
const blobStorage = new InMemoryBlobStorage();
const requestPath = `${organizationId}/${federatedGraphId}/cache_warmup/operations.json`;

const app = new Hono();

cdn(app, {
authJwtSecret: secretKey,
authAdmissionJwtSecret: secretAdmissionKey,
blobStorage,
});

test('it returns a 401 if no Authorization header is provided', async () => {
const res = await app.request(requestPath, {
method: 'GET',
});
expect(res.status).toBe(401);
});

test('it returns a 401 if an invalid Authorization header is provided', async () => {
const res = await app.request(requestPath, {
method: 'GET',
headers: {
Authorization: `Bearer ${token.slice(0, -1)}}`,
},
});
expect(res.status).toBe(401);
});

test('it returns a 401 if the graph or organization ids does not match with the JWT payload', async () => {
const res = await app.request(`/foo/bar/operations/cache_warmup/operations.json`, {
method: 'GET',
headers: {
Authorization: `Bearer ${token}`,
},
});
expect(res.status).toBe(400);
});

test('it returns a 401 if the token has expired', async () => {
const token = await new SignJWT({
organization_id: organizationId,
federated_graph_id: federatedGraphId,
exp: Math.floor(Date.now() / 1000) - 60,
})
.setProtectedHeader({ alg: 'HS256' })
.sign(new TextEncoder().encode(secretKey));
const res = await app.request(`/foo/bar/cache_warmup/operations.json`, {
method: 'GET',
headers: {
Authorization: `Bearer ${token}`,
},
});
expect(res.status).toBe(401);
});

test('it returns the cache warmer operations', async () => {
const operationContents = JSON.stringify({
operations: [
{
request: {
operationName: 'AB',
query: 'query AB($a: Int!){employeeAsList(id: $a){tag id derivedMood products}}',
},
client: { name: 'unknown', version: 'missing' },
},
],
});

blobStorage.objects.set(requestPath, {
buffer: Buffer.from(operationContents),
});

const res = await app.request(requestPath, {
method: 'GET',
headers: {
Authorization: `Bearer ${token}`,
},
});
expect(res.status).toBe(200);
expect(await res.text()).toBe(operationContents);
});

test('it returns a 404 if the persisted operation does not exist', async () => {
const res = await app.request(`${organizationId}/${federatedGraphId}/cache_warmup/does_not_exist.json`, {
method: 'GET',
headers: {
Authorization: `Bearer ${token}`,
},
});
expect(res.status).toBe(404);
});
});
});
84 changes: 84 additions & 0 deletions cli/src/commands/router/commands/cache/commands/push.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
import { existsSync } from 'node:fs';
import { readFile } from 'node:fs/promises';
import { Command, program } from 'commander';
import pc from 'picocolors';
import { resolve } from 'pathe';
import { EnumStatusCode } from '@wundergraph/cosmo-connect/dist/common/common_pb';
import { BaseCommandOptions } from '../../../../../core/types/types.js';
import { getBaseHeaders } from '../../../../../core/config.js';

export default (opts: BaseCommandOptions) => {
const command = new Command('push');
command.description('Pushes new cache warmer operations to the registry');
command.argument(
'<graph_name>',
'The name of the federated graph or monograph on which the check operations are stored.',
);
command.option('-n, --namespace [string]', 'The namespace of the federated graph or monograph.');
command.option(
'-o, --operation-name <operation-name>',
'The name of the operation. Only needed when working with multi-operation document.',
);
command.option(
Comment thread
StarpTech marked this conversation as resolved.
'-p, --persisted-operation-id <persisted-operation-id>',
'The id of the persisted operation to be pushed.',
);
command.option(
'-f, --file <file>',
'The file with the operation to push - supports .graphql, .gql. If both the file and the persisted operation id are provided, the persisted operation id will be used.',
);

command.action(async (name, options) => {
if (!options.file && !options.persistedOperationId) {
command.error(pc.red('No operation file or the id of persisted operation provided'));
}

if (options.persistedOperationId && options.file) {
command.error(pc.red('Only the persisted operation id or the file can be specified'));
}

let operation: string | undefined;
if (options.file) {
const operationFile = resolve(options.file);
if (!existsSync(operationFile)) {
program.error(
pc.red(
pc.bold(
`The operation file '${pc.bold(operationFile)}' does not exist. Please check the path and try again.`,
),
),
);
}

const schemaBuffer = await readFile(operationFile);
operation = new TextDecoder().decode(schemaBuffer);
if (operation.trim().length === 0) {
program.error(
pc.red(pc.bold(`The schema file '${pc.bold(operationFile)}' is empty. Please provide a valid operation.`)),
);
}
}

const result = await opts.client.platform.pushCacheWarmerOperation(
{
federatedGraphName: name,
operationContent: operation,
operationName: options.operationName,
operationPersistedId: options.persistedOperationId,
namespace: options.namespace,
},
{ headers: getBaseHeaders() },
);

if (result.response?.code === EnumStatusCode.OK) {
console.log(pc.green(`The cache warmer operation was pushed successfully.`));
} else {
console.log(pc.red(`Failed to push the cache warmer operation. Please try again.`));
if (result.response?.details) {
console.error(pc.red(pc.bold(result.response?.details)));
}
process.exit(1);
Comment thread
JivusAyrus marked this conversation as resolved.
}
});
return command;
};
14 changes: 14 additions & 0 deletions cli/src/commands/router/commands/cache/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import { Command } from 'commander';
import { BaseCommandOptions } from '../../../../core/types/types.js';
import PushCacheOperation from './commands/push.js';

export default (opts: BaseCommandOptions) => {
const schema = new Command('cache');
schema.description(
'Provides commands for pushing and maintaining router cache warmer operations of a federated graph',
);

schema.addCommand(PushCacheOperation(opts));

return schema;
};
6 changes: 6 additions & 0 deletions cli/src/commands/router/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import ComposeRouterConfig from './commands/compose.js';
import FetchRouterConfig from './commands/fetch.js';
import RouterTokenCommands from './commands/token/index.js';
import DownloadRouterBinaryConfig from './commands/download-binary.js';
import RouterCacheCommands from './commands/cache/index.js';

export default (opts: BaseCommandOptions) => {
const cmd = new Command('router');
Expand All @@ -17,6 +18,11 @@ export default (opts: BaseCommandOptions) => {
}),
);
cmd.addCommand(DownloadRouterBinaryConfig(opts));
cmd.addCommand(
RouterCacheCommands({
client: opts.client,
}),
);

cmd.hook('preAction', async (thisCmd) => {
if (['compose', 'download-binary'].includes(thisCmd.args[0])) {
Expand Down
69 changes: 37 additions & 32 deletions connect-go/gen/proto/wg/cosmo/common/common.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading