diff --git a/CHANGES.md b/CHANGES.md index f26229c36e..1fa9a54507 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,8 @@ ## v2.2.3 - Fixed an issue that zendesk integration feedback form might not always be opened via footer link +- #3449: Gateway ckanRedirection module performance improvement +- related to #3448: make web server module to use read only registry node for site map generation ## v2.2.2 diff --git a/deploy/helm/internal-charts/gateway/README.md b/deploy/helm/internal-charts/gateway/README.md index 90544c9213..5def85e474 100644 --- a/deploy/helm/internal-charts/gateway/README.md +++ b/deploy/helm/internal-charts/gateway/README.md @@ -18,8 +18,8 @@ Kubernetes: `>= 1.14.0-0` | autoscaler.maxReplicas | int | `3` | | | autoscaler.minReplicas | int | `1` | | | autoscaler.targetCPUUtilizationPercentage | int | `80` | | -| ckanRedirectionDomain | string | `nil` | CKAN redirection target CKAN instance domain. See `enableCkanRedirection` for more details. | -| ckanRedirectionPath | string | `nil` | CKAN redirection target CKAN instance path. See `enableCkanRedirection` for more details. | +| ckanRedirectionDomain | string | `nil` | CKAN redirection target CKAN instance domain (e.g. `data.gov.au`). See `enableCkanRedirection` for more details. | +| ckanRedirectionPath | string | `nil` | CKAN redirection target CKAN instance path (e.g. `/data`). See `enableCkanRedirection` for more details. | | cookie | object | default value see `Description` | Session cookie settings.
More info: https://github.com/expressjs/session#cookie
Supported options are:
| | cors.exposedHeaders[0] | string | `"Content-Range"` | | | cors.exposedHeaders[1] | string | `"X-Content-Range"` | | @@ -51,6 +51,8 @@ Kubernetes: `>= 1.14.0-0` | helmet.frameguard | bool | `false` | | | image.name | string | `"magda-gateway"` | | | proxyTimeout | int | nil (120 seconds default value will be used by upstream lib internally) | How long time (in seconds) before upstream service must complete request in order to avoid request timeout error. If not set, the request will time out after 120 seconds. | +| registryQueryCacheMaxKeys | number | `nil` | Specifies a maximum amount of keys that can be stored in the registryQueryCache. By default, it will be set to 500 seconds if leave blank. | +| registryQueryCacheStdTTL | number | `nil` | The standard ttl as number in seconds for every generated cache element in the registryQueryCache. To disable the cache, set this value to `0`. By default, it will be set to 600 seconds if leave blank. | | resources.limits.cpu | string | `"200m"` | | | resources.requests.cpu | string | `"50m"` | | | resources.requests.memory | string | `"40Mi"` | | diff --git a/deploy/helm/internal-charts/gateway/templates/deployment.yaml b/deploy/helm/internal-charts/gateway/templates/deployment.yaml index 439a470d59..135bb56d31 100644 --- a/deploy/helm/internal-charts/gateway/templates/deployment.yaml +++ b/deploy/helm/internal-charts/gateway/templates/deployment.yaml @@ -58,6 +58,12 @@ spec: {{- end }} {{- if .Values.enableCkanRedirection }} "--enableCkanRedirection", {{ .Values.enableCkanRedirection | quote }}, +{{- end }} +{{- if not (kindIs "invalid" .Values.registryQueryCacheStdTTL) }} + "--registryQueryCacheStdTTL", {{ .Values.registryQueryCacheStdTTL | quote }}, +{{- end }} +{{- if .Values.registryQueryCacheMaxKeys }} + "--registryQueryCacheMaxKeys", {{ .Values.registryQueryCacheMaxKeys | quote }}, {{- end }} "--proxyRoutesJson", "/etc/config/routes.json", "--webProxyRoutesJson", "/etc/config/webRoutes.json", diff --git a/deploy/helm/internal-charts/gateway/values.yaml b/deploy/helm/internal-charts/gateway/values.yaml index e039f5dc7d..608234424d 100644 --- a/deploy/helm/internal-charts/gateway/values.yaml +++ b/deploy/helm/internal-charts/gateway/values.yaml @@ -34,10 +34,19 @@ enableWebAccessControl: false # that is specified by config option `ckanRedirectionDomain` and `ckanRedirectionPath`. enableCkanRedirection: false -# -- CKAN redirection target CKAN instance domain. See `enableCkanRedirection` for more details. +# -- (number) The standard ttl as number in seconds for every generated cache element in the registryQueryCache. +# To disable the cache, set this value to `0`. +# By default, it will be set to 600 seconds if leave blank. +registryQueryCacheStdTTL: + +# -- (number) Specifies a maximum amount of keys that can be stored in the registryQueryCache. +# By default, it will be set to 500 seconds if leave blank. +registryQueryCacheMaxKeys: + +# -- CKAN redirection target CKAN instance domain (e.g. `data.gov.au`). See `enableCkanRedirection` for more details. ckanRedirectionDomain: -# -- CKAN redirection target CKAN instance path. See `enableCkanRedirection` for more details. +# -- CKAN redirection target CKAN instance path (e.g. `/data`). See `enableCkanRedirection` for more details. ckanRedirectionPath: diff --git a/deploy/helm/internal-charts/web-server/README.md b/deploy/helm/internal-charts/web-server/README.md index 2028844466..bbe166300c 100644 --- a/deploy/helm/internal-charts/web-server/README.md +++ b/deploy/helm/internal-charts/web-server/README.md @@ -99,7 +99,7 @@ Kubernetes: `>= 1.14.0-0` | openInExternalTerriaMapButtonText | string | `nil` | When set, the string here will replace the text of the `Open in National Map` button in Map Preview area. | | openInExternalTerriaMapTargetUrl | string | `nil` | When set, the `Open in National Map` button in Map Preview area will sent map data to the URL provided and open the map preview there. When not set, UI will by default send to the National Map. | | previewMapFormatPerference | list | `[{"format":"WMS","urlRegex":"^(?!.*(SceneServer)).*$"},{"format":"ESRI MAPSERVER","urlRegex":"MapServer"},{"format":"WFS","urlRegex":"^(?!.*(SceneServer)).*$"},{"format":"ESRI FEATURESERVER","urlRegex":"FeatureServer"},{"format":"GeoJSON","singleFile":true},{"format":"csv-geo-au","singleFile":true},{"format":"KML","singleFile":true},{"format":"KMZ","singleFile":true}]` | Preview map module format perference list The list includes one or more `format perference item`. When there are more than one data source available, "Preview Map module" will use this perference to determine which data soruce will be used. It will go through the perference list. The first matched format (i.e. find a data source with the format ) will be chosen. A `format perference item` can have the following fields: | -| registryApiBaseUrlInternal | string | `"http://registry-api/v0"` | | +| registryApiBaseUrlInternal | string | `"http://registry-api-read-only/v0"` | | | replicas | string | `nil` | no. of replicas required for the deployment. If not set, k8s will assume `1` but allows HPA (autoscaler) alters it. @default 1 | | resources.limits.cpu | string | `"100m"` | | | resources.requests.cpu | string | `"10m"` | | diff --git a/deploy/helm/internal-charts/web-server/values.yaml b/deploy/helm/internal-charts/web-server/values.yaml index 4c6cd1a943..80fd6d3c89 100644 --- a/deploy/helm/internal-charts/web-server/values.yaml +++ b/deploy/helm/internal-charts/web-server/values.yaml @@ -46,7 +46,7 @@ useLocalStyleSheet: false contentApiBaseUrlInternal: "http://content-api/v0/" -registryApiBaseUrlInternal: "http://registry-api/v0" +registryApiBaseUrlInternal: "http://registry-api-read-only/v0" adminApiBaseUrl: diff --git a/magda-gateway/package.json b/magda-gateway/package.json index f389739781..e82ddf20b0 100644 --- a/magda-gateway/package.json +++ b/magda-gateway/package.json @@ -40,6 +40,7 @@ "pg": "^8.7.3", "read-pkg-up": "^3.0.0", "request": "^2.88.0", + "node-cache": "^5.1.2", "tsmonad": "^0.7.2", "urijs": "^1.19.11", "uuid": "^8.2.0", diff --git a/magda-gateway/src/buildApp.ts b/magda-gateway/src/buildApp.ts index 5a395801a6..51e86ab847 100644 --- a/magda-gateway/src/buildApp.ts +++ b/magda-gateway/src/buildApp.ts @@ -29,6 +29,7 @@ import AuthDecisionQueryClient from "magda-typescript-common/src/opa/AuthDecisio type Route = { to: string; auth?: boolean; + methods?: { method: string; target: string }[]; }; type Routes = { @@ -74,6 +75,8 @@ export type Config = { magdaAdminPortalName?: string; proxyTimeout?: string; skipAuth?: boolean; + registryQueryCacheMaxKeys: number; + registryQueryCacheStdTTL: number; }; export default function buildApp(app: express.Application, config: Config) { @@ -220,12 +223,19 @@ export default function buildApp(app: express.Application, config: Config) { "Cannot locate routes.registry for ckan redirection!" ); } else { + const registryReadOnlyApi = routes.registry?.methods?.find( + (item) => item.method.toLowerCase() === "get" + )?.target; mainRouter.use( createCkanRedirectionRouter({ ckanRedirectionDomain: config.ckanRedirectionDomain, ckanRedirectionPath: config.ckanRedirectionPath, - registryApiBaseUrlInternal: routes.registry.to, - tenantId: 0 // FIXME: Rather than being hard-coded to the default tenant, the CKAN router needs to figure out the correct tenant. + registryApiBaseUrlInternal: registryReadOnlyApi + ? registryReadOnlyApi + : routes.registry?.to, + tenantId: 0, // FIXME: Rather than being hard-coded to the default tenant, the CKAN router needs to figure out the correct tenant., + cacheStdTTL: config.registryQueryCacheStdTTL, + cacheMaxKeys: config.registryQueryCacheMaxKeys }) ); } diff --git a/magda-gateway/src/createCkanRedirectionRouter.ts b/magda-gateway/src/createCkanRedirectionRouter.ts index 664c069a3f..441ed807af 100644 --- a/magda-gateway/src/createCkanRedirectionRouter.ts +++ b/magda-gateway/src/createCkanRedirectionRouter.ts @@ -4,12 +4,15 @@ import _ from "lodash"; import Registry from "magda-typescript-common/src/registry/RegistryClient"; import unionToThrowable from "magda-typescript-common/src/util/unionToThrowable"; import { escapeRegExp } from "lodash"; +import NodeCache from "node-cache"; export type CkanRedirectionRouterOptions = { ckanRedirectionDomain: string; ckanRedirectionPath: string; registryApiBaseUrlInternal: string; tenantId: number; + cacheStdTTL: number; + cacheMaxKeys: number; }; export type genericUrlRedirectConfig = @@ -76,8 +79,17 @@ export default function buildCkanRedirectionRouter({ ckanRedirectionDomain, ckanRedirectionPath, registryApiBaseUrlInternal, - tenantId + tenantId, + cacheStdTTL, + cacheMaxKeys }: CkanRedirectionRouterOptions): express.Router { + const registryQueryCache = + cacheMaxKeys === 0 // turn off the cache when cacheMaxKeys==0 + ? null + : new NodeCache({ + stdTTL: cacheStdTTL, + maxKeys: cacheMaxKeys + }); const router = express.Router(); const registry = new Registry({ baseUrl: registryApiBaseUrlInternal, @@ -162,11 +174,31 @@ export default function buildCkanRedirectionRouter({ ); }); + function getQueryRegistryRecordApiCacheKey( + aspectQuery: string[], + aspect: string[], + limit: number + ) { + return JSON.stringify({ + aspectQuery, + aspect, + limit + }); + } + async function queryRegistryRecordApi( aspectQuery: string[], aspect: string[], - limit: number = 1 + limit: number = undefined ) { + const cacheKey = getQueryRegistryRecordApiCacheKey( + aspectQuery, + aspect, + limit + ); + if (registryQueryCache?.has(cacheKey)) { + return registryQueryCache.get(cacheKey); + } const queryParameters: any = { limit }; @@ -188,6 +220,12 @@ export default function buildCkanRedirectionRouter({ ) ); + try { + registryQueryCache?.set(cacheKey, records); + } catch (e) { + console.log("Failed to save registryQueryCache: " + e); + } + return records; } @@ -196,7 +234,7 @@ export default function buildCkanRedirectionRouter({ aspectName: string, retrieveAspectContent: boolean = true, retrieveAspects: string[] = [], - limit: number = 1 + limit: number = undefined ): Promise { const query = `${aspectName}.${ uuidRegEx.test(ckanIdOrName) ? "id" : "name" @@ -232,7 +270,7 @@ export default function buildCkanRedirectionRouter({ ckanIdOrName: string, aspectName: string ) { - const records = await queryCkanAspect(ckanIdOrName, aspectName, false); + const records = await queryCkanAspect(ckanIdOrName, aspectName); if (!records || !records.length || !records[0]["id"]) { return null; } else { diff --git a/magda-gateway/src/defaultConfig.ts b/magda-gateway/src/defaultConfig.ts index fc96451b2f..e83d43a4b1 100644 --- a/magda-gateway/src/defaultConfig.ts +++ b/magda-gateway/src/defaultConfig.ts @@ -10,7 +10,16 @@ export default { }, registry: { to: "http://localhost:6101/v0", - auth: true + auth: true, + methods: [ + { method: "get", target: "http://localhost:6101/v0" }, + { method: "head", target: "http://localhost:6101/v0" }, + { method: "options", target: "http://localhost:6101/v0" }, + { method: "post", target: "http://localhost:6101/v0" }, + { method: "put", target: "http://localhost:6101/v0" }, + { method: "patch", target: "http://localhost:6101/v0" }, + { method: "delete", target: "http://localhost:6101/v0" } + ] }, "registry-read-only": { to: "http://localhost:6101/v0", diff --git a/magda-gateway/src/index.ts b/magda-gateway/src/index.ts index e921033d04..d0bd88e401 100755 --- a/magda-gateway/src/index.ts +++ b/magda-gateway/src/index.ts @@ -139,6 +139,18 @@ const argv = addJwtSecretFromEnvVar( type: "string", default: "" }) + .option("registryQueryCacheStdTTL", { + describe: + "the standard ttl as number in seconds for every generated cache element in the registryQueryCache.", + type: "number", + default: 600 + }) + .option("registryQueryCacheMaxKeys", { + describe: + "specifies a maximum amount of keys that can be stored in the registryQueryCache. To disable the cache, set this value to `0`.", + type: "number", + default: 500 + }) .option("enableWebAccessControl", { describe: "Whether users are required to enter a username & password to access the magda web interface", diff --git a/magda-gateway/src/test/ckanRedirection.spec.ts b/magda-gateway/src/test/ckanRedirection.spec.ts index fe736e8efe..136d34dcfa 100644 --- a/magda-gateway/src/test/ckanRedirection.spec.ts +++ b/magda-gateway/src/test/ckanRedirection.spec.ts @@ -17,6 +17,22 @@ import resCkanDatasetQuery from "./sampleRegistryResponses/ckanDatasetQuery.json import resCkanOrganizationQuery from "./sampleRegistryResponses/ckanOrganizationQuery.json"; import resCkanResource from "./sampleRegistryResponses/ckanResource.json"; +function checkRedirectionDetails(location: string | RegExp) { + return (res: supertest.Response) => { + if (_.isRegExp(location)) { + expect(location.test(res.header["location"])).to.equal(true); + } else { + expect(res.header["location"]).to.equal(location); + } + }; +} + +function checkStatusCode(statusCode: number = 308) { + return (res: supertest.Response) => { + expect(res.status).to.equal(statusCode); + }; +} + describe("ckanRedirectionRouter router", () => { const ckanRedirectionDomain = "ckan.data.gov.au"; const ckanRedirectionPath = ""; @@ -39,7 +55,9 @@ describe("ckanRedirectionRouter router", () => { ckanRedirectionDomain, ckanRedirectionPath, registryApiBaseUrlInternal: registryUrl, - tenantId: 1 + tenantId: 1, + cacheMaxKeys: 500, + cacheStdTTL: 600 }); app = express(); app.use(router); @@ -474,22 +492,6 @@ describe("ckanRedirectionRouter router", () => { }); } - function checkRedirectionDetails(location: string | RegExp) { - return (res: supertest.Response) => { - if (_.isRegExp(location)) { - expect(location.test(res.header["location"])).to.equal(true); - } else { - expect(res.header["location"]).to.equal(location); - } - }; - } - - function checkStatusCode(statusCode: number = 308) { - return (res: supertest.Response) => { - expect(res.status).to.equal(statusCode); - }; - } - function testCkanDomainChangeOnly( targetUrlOrUrls: string | string[], statusCode: number = 308, @@ -570,3 +572,176 @@ describe("ckanRedirectionRouter router", () => { }); } }); + +describe("ckanRedirectionRouter router cache", () => { + const ckanRedirectionDomain = "ckan.data.gov.au"; + const ckanRedirectionPath = ""; + + let app: express.Application; + const registryUrl = "http://registry.example.com"; + let registryScope: nock.Scope; + let registryQueryCount: number = 0; + + function setupRegistryApiForCkanDatasetQuery() { + registryQueryCount = 0; + const errorResponse = `{ + "hasMore": false, + "records": [ ] + }`; + + const okCkanDatasetResponse = resCkanDatasetQuery; + + registryScope + .persist() + .get("/records") + .query(true) + .reply(200, function (uri: string) { + registryQueryCount++; + const uriObj = URI(uri); + const query = uriObj.search(true); + if (!query || !query.aspectQuery) return errorResponse; + const [path, value] = (query.aspectQuery as string).split(":"); + if ( + path === "ckan-dataset.name" && + value === "pg_skafsd0_f___00120141210_11a" + ) { + return okCkanDatasetResponse; + } else if ( + path === "ckan-dataset.id" && + value === "8beb4387-ec03-46f9-8048-3ad76c0416c8" + ) { + return okCkanDatasetResponse; + } else { + return errorResponse; + } + }); + } + + before(() => { + registryScope = nock(registryUrl); + setupRegistryApiForCkanDatasetQuery(); + }); + + after(() => { + nock.cleanAll(); + }); + + beforeEach(() => { + app = express(); + registryQueryCount = 0; + }); + + afterEach(() => { + if ((console.error).restore) { + (console.error).restore(); + } + }); + + function setCkanRedirectRouter( + cacheMaxKeys: number = 500, + cacheStdTTL: number = 600 + ) { + const router = createCkanRedirectionRouter({ + ckanRedirectionDomain, + ckanRedirectionPath, + registryApiBaseUrlInternal: registryUrl, + tenantId: 1, + cacheMaxKeys, + cacheStdTTL + }); + app.use(router); + } + + it("should query registry only once for multiple requests to the same URL", async () => { + setCkanRedirectRouter(); + for (let i = 0; i < 3; i++) { + await supertest(app) + .get("/dataset/pg_skafsd0_f___00120141210_11a") + .expect(303) + .expect( + checkRedirectionDetails( + "/dataset/ds-dga-8beb4387-ec03-46f9-8048-3ad76c0416c8/details" + ) + ); + } + expect(registryQueryCount).to.equal(1); + }); + + it("should query registry once for every request to the same URL", async () => { + // set cacheMaxKeys = 0 to disable the cache + setCkanRedirectRouter(0); + for (let i = 0; i < 3; i++) { + await supertest(app) + .get("/dataset/pg_skafsd0_f___00120141210_11a") + .expect(303) + .expect( + checkRedirectionDetails( + "/dataset/ds-dga-8beb4387-ec03-46f9-8048-3ad76c0416c8/details" + ) + ); + } + expect(registryQueryCount).to.equal(3); + }); + + it("should query registry more than once for different requests to different URLs when cacheMaxKeys = 1", async () => { + setCkanRedirectRouter(1); + await supertest(app) + .get("/dataset/pg_skafsd0_f___00120141210_11a") + .expect(303) + .expect( + checkRedirectionDetails( + "/dataset/ds-dga-8beb4387-ec03-46f9-8048-3ad76c0416c8/details" + ) + ); + await supertest(app) // this request will not be cached + .get("/dataset/8beb4387-ec03-46f9-8048-3ad76c0416c8") + .expect(303) + .expect( + checkRedirectionDetails( + "/dataset/ds-dga-8beb4387-ec03-46f9-8048-3ad76c0416c8/details" + ) + ); + for (let i = 0; i < 3; i++) { + await supertest(app) + .get("/dataset/8beb4387-ec03-46f9-8048-3ad76c0416c8") + .expect(303) + .expect( + checkRedirectionDetails( + "/dataset/ds-dga-8beb4387-ec03-46f9-8048-3ad76c0416c8/details" + ) + ); + } + expect(registryQueryCount).to.equal(5); + }); + + it("should query registry 2 times for different requests to different URLs when cacheMaxKeys = 2", async () => { + setCkanRedirectRouter(2); + await supertest(app) + .get("/dataset/pg_skafsd0_f___00120141210_11a") + .expect(303) + .expect( + checkRedirectionDetails( + "/dataset/ds-dga-8beb4387-ec03-46f9-8048-3ad76c0416c8/details" + ) + ); + await supertest(app) // this request will not be cached + .get("/dataset/8beb4387-ec03-46f9-8048-3ad76c0416c8") + .expect(303) + .expect( + checkRedirectionDetails( + "/dataset/ds-dga-8beb4387-ec03-46f9-8048-3ad76c0416c8/details" + ) + ); + for (let i = 0; i < 3; i++) { + await supertest(app) + .get("/dataset/8beb4387-ec03-46f9-8048-3ad76c0416c8") + .expect(303) + .expect( + checkRedirectionDetails( + "/dataset/ds-dga-8beb4387-ec03-46f9-8048-3ad76c0416c8/details" + ) + ); + } + expect(registryQueryCount).to.equal(2); + }); +}); diff --git a/magda-gateway/src/test/createAuthApiKeyMiddleware.spec.ts b/magda-gateway/src/test/createAuthApiKeyMiddleware.spec.ts index d82917fe48..1687a1b0d6 100644 --- a/magda-gateway/src/test/createAuthApiKeyMiddleware.spec.ts +++ b/magda-gateway/src/test/createAuthApiKeyMiddleware.spec.ts @@ -71,7 +71,9 @@ const defaultAppOptions = { web: "https://127.0.0.1", tenantUrl: "http://tenant", defaultCacheControl: "DEFAULT CACHE CONTROL", - authPluginConfigJson: [] as AuthPluginBasicConfig[] + authPluginConfigJson: [] as AuthPluginBasicConfig[], + registryQueryCacheStdTTL: 600, + registryQueryCacheMaxKeys: 500 }; function getUserIdFromNockReq(req: any, jwtSecret: string) { diff --git a/magda-gateway/src/test/createHttpsRedirectionMiddleware.spec.ts b/magda-gateway/src/test/createHttpsRedirectionMiddleware.spec.ts index a64949bf41..62ad72d07f 100644 --- a/magda-gateway/src/test/createHttpsRedirectionMiddleware.spec.ts +++ b/magda-gateway/src/test/createHttpsRedirectionMiddleware.spec.ts @@ -40,7 +40,9 @@ const defaultAppOptions = { web: "https://127.0.0.1", tenantUrl: "http://tenant", defaultCacheControl: "DEFAULT CACHE CONTROL", - authPluginConfigJson: [] as AuthPluginBasicConfig[] + authPluginConfigJson: [] as AuthPluginBasicConfig[], + registryQueryCacheStdTTL: 600, + registryQueryCacheMaxKeys: 500 }; describe("Test createHttpsRedirectionMiddleware", () => { diff --git a/magda-gateway/src/test/integration.spec.ts b/magda-gateway/src/test/integration.spec.ts index a5573bfbf3..a6c94e94b6 100644 --- a/magda-gateway/src/test/integration.spec.ts +++ b/magda-gateway/src/test/integration.spec.ts @@ -120,6 +120,8 @@ describe("Integration Tests", function (this: Mocha.ISuiteCallbackContext) { defaultCacheControl: "public, max-age=60", openfaasGatewayUrl: undefined, authPluginConfigJson: [], + registryQueryCacheStdTTL: 600, + registryQueryCacheMaxKeys: 500, ...config }); @@ -469,4 +471,92 @@ describe("Integration Tests", function (this: Mocha.ISuiteCallbackContext) { await app.put("/xxx").expect(200, resText2); }); }); + + describe("ckanRedirectionRouter setup", () => { + it("should use read only registry API endpoint when available", async () => { + const app = setupTestApp({ + enableCkanRedirection: true, + proxyRoutesJson: { + registry: { + to: "http://registry-full.com", + auth: true, + statusCheck: false, + methods: [ + { + method: "get", + target: "http://registry-readonly.com" + } + ] + } + }, + enableWebAccessControl: false + }); + + const registryReadOnly = nock("http://registry-readonly.com") + .get("/records") + .query(true) + .reply( + 200, + `{ + hasMore: false, + records: [] + }` + ); + + const registryFull = nock("http://registry-full.com") + .get("/records") + .query(true) + .reply( + 200, + `{ + hasMore: false, + records: [] + }` + ); + + await app + .get("/dataset/pg_skafsd0_f___00120141210_11a") + .expect(303); + + expect(registryFull.isDone()).to.be.false; + expect(registryReadOnly.isDone()).to.be.true; + }); + + it("should use full registry API endpoint when read only node is not available", async () => { + const app = setupTestApp({ + enableCkanRedirection: true, + proxyRoutesJson: { + registry: { + to: "http://registry-full.com", + auth: true, + statusCheck: false + } + }, + enableWebAccessControl: false + }); + + const registryReadOnly = nock("http://registry-readonly.com") + .get("/records") + .query(true) + .reply(200, { + hasMore: false, + records: [] + }); + + const registryFull = nock("http://registry-full.com") + .get("/records") + .query(true) + .reply(200, { + hasMore: false, + records: [] + }); + + await app + .get("/dataset/pg_skafsd0_f___00120141210_11a") + .expect(303); + + expect(registryFull.isDone()).to.be.true; + expect(registryReadOnly.isDone()).to.be.false; + }); + }); }); diff --git a/magda-gateway/src/test/proxy.spec.ts b/magda-gateway/src/test/proxy.spec.ts index 6a585d02a4..551bd83820 100644 --- a/magda-gateway/src/test/proxy.spec.ts +++ b/magda-gateway/src/test/proxy.spec.ts @@ -49,7 +49,9 @@ const defaultAppOptions = { web: "https://127.0.0.1", tenantUrl: "http://tenant", defaultCacheControl: "DEFAULT CACHE CONTROL", - authPluginConfigJson: [] as AuthPluginBasicConfig[] + authPluginConfigJson: [] as AuthPluginBasicConfig[], + registryQueryCacheStdTTL: 600, + registryQueryCacheMaxKeys: 500 }; describe("proxying", () => { diff --git a/yarn.lock b/yarn.lock index f187c53ca3..40964523dd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6979,6 +6979,11 @@ clone-response@^1.0.2: dependencies: mimic-response "^1.0.0" +clone@2.x: + version "2.1.2" + resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f" + integrity sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w== + clone@^1.0.2: version "1.0.4" resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.4.tgz#da309cc263df15994c688ca902179ca3c7cd7c7e" @@ -15262,6 +15267,13 @@ node-addon-api@^3.1.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161" integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A== +node-cache@^5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/node-cache/-/node-cache-5.1.2.tgz#f264dc2ccad0a780e76253a694e9fd0ed19c398d" + integrity sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg== + dependencies: + clone "2.x" + node-ensure@^0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/node-ensure/-/node-ensure-0.0.0.tgz#ecae764150de99861ec5c810fd5d096b183932a7"