Skip to content

Commit

Permalink
feat(jsDelivr): move code, add tests, preload data correctly (#384)
Browse files Browse the repository at this point in the history
  • Loading branch information
@bodinsamuel
bodinsamuel authored Aug 8, 2019
1 parent 7cd0765 commit 373d341
Show file tree
Hide file tree
Showing 8 changed files with 370 additions and 53 deletions.
1 change: 1 addition & 0 deletions src/__tests__/__snapshots__/config.test.js.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ Object {
},
],
"jsDelivrHitsEndpoint": "https://data.jsdelivr.com/v1/stats/packages/npm/month/all",
"jsDelivrPackageEndpoint": "https://data.jsdelivr.com/v1/package/npm",
"maxObjSize": 450000,
"npmDownloadsEndpoint": "https://api.npmjs.org/downloads",
"npmRegistryEndpoint": "https://replicate.npmjs.com/registry",
Expand Down
1 change: 1 addition & 0 deletions src/config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ const defaultConfig = {
npmRootEndpoint: 'https://registry.npmjs.org',
jsDelivrHitsEndpoint:
'https://data.jsdelivr.com/v1/stats/packages/npm/month/all',
jsDelivrPackageEndpoint: 'https://data.jsdelivr.com/v1/package/npm',
unpkgRoot: 'https://unpkg.com',
maxObjSize: 450000,
popularDownloadsRatio: 0.005,
Expand Down
7 changes: 4 additions & 3 deletions src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import config from './config.js';
import * as npm from './npm/index.js';
import log from './log.js';
import datadog from './datadog.js';
import { loadHits } from './jsDelivr.js';
import * as jsDelivr from './jsDelivr/index.js';

log.info('🗿 npm ↔️ Algolia replication starts ⛷ 🐌 🛰');

Expand All @@ -31,6 +31,9 @@ async function main() {
await setSettings(bootstrapIndex);
datadog.timing('main.init_algolia', Date.now() - start);

// Preload some useful data
await jsDelivr.loadHits();

// then we run the bootstrap
// after a bootstrap is done, it's moved to main (with settings)
// if it was already finished, we will set the settings on the main index
Expand Down Expand Up @@ -106,8 +109,6 @@ async function bootstrap(state) {
return state;
}

await loadHits();

const { seq, nbDocs: totalDocs } = await npm.getInfo();
if (!state.bootstrapLastId) {
// Start from 0
Expand Down
48 changes: 0 additions & 48 deletions src/jsDelivr.js
View file

This file was deleted.

183 changes: 183 additions & 0 deletions src/jsDelivr/__test__/__snapshots__/index.test.js.snap
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`files getAllFilesList() should get a flat list of files 1`] = `
Array [
Array [
Object {
"hash": "+uxmYs/88pYWmLwFS3M54NGjE+hX6sBmwNOpzaW1LGk=",
"name": "/bin/jest.js",
"size": 343,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "MvOGr1Lc6r8wEe8GNmscm3Sx/QWEFE4Is1AZ5rQzFr8=",
"name": "/build/jest.d.ts",
"size": 291,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "BEQ5sRqArzHCh5sNbwjxHRQunhxkCD1HXcM9EdYAKPc=",
"name": "/build/jest.d.ts.map",
"size": 171,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "m5wVGuVr5Pq4z5L2vpeMVA3rbLV4kQ0MCPuo0newsmY=",
"name": "/build/jest.js",
"size": 1030,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "m/vOMvpK2FU19W9PYavnEExEToN7HHU1mb/f/ooU3eQ=",
"name": "/LICENSE",
"size": 1099,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "9hWvkPsgtCTc1w0lswu1AO+Q+S19Dppeg5bNklG/Khg=",
"name": "/package.json",
"size": 925,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "MPu0d2f8or6adBXZZLUNh6vL7Yeg34MmOBiupdclu10=",
"name": "/README.md",
"size": 551,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "CycshPBWVvIRZozw+b1pnAvKYC1Q7aPvcT8tS+HPepU=",
"name": "/tsconfig.json",
"size": 162,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "kt2uoTK/NmyQe2OUiNwpdwxV4RxgS2gW9rEgOtj+lZU=",
"name": "/tsconfig.tsbuildinfo",
"size": 220798,
"time": "1985-10-26T08:15:00.000Z",
},
],
]
`;

exports[`files getAllFilesList() should get multiple flat list of files 1`] = `
Array [
Array [
Object {
"hash": "+uxmYs/88pYWmLwFS3M54NGjE+hX6sBmwNOpzaW1LGk=",
"name": "/bin/jest.js",
"size": 343,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "MvOGr1Lc6r8wEe8GNmscm3Sx/QWEFE4Is1AZ5rQzFr8=",
"name": "/build/jest.d.ts",
"size": 291,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "BEQ5sRqArzHCh5sNbwjxHRQunhxkCD1HXcM9EdYAKPc=",
"name": "/build/jest.d.ts.map",
"size": 171,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "m5wVGuVr5Pq4z5L2vpeMVA3rbLV4kQ0MCPuo0newsmY=",
"name": "/build/jest.js",
"size": 1030,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "m/vOMvpK2FU19W9PYavnEExEToN7HHU1mb/f/ooU3eQ=",
"name": "/LICENSE",
"size": 1099,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "9hWvkPsgtCTc1w0lswu1AO+Q+S19Dppeg5bNklG/Khg=",
"name": "/package.json",
"size": 925,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "MPu0d2f8or6adBXZZLUNh6vL7Yeg34MmOBiupdclu10=",
"name": "/README.md",
"size": 551,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "CycshPBWVvIRZozw+b1pnAvKYC1Q7aPvcT8tS+HPepU=",
"name": "/tsconfig.json",
"size": 162,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "kt2uoTK/NmyQe2OUiNwpdwxV4RxgS2gW9rEgOtj+lZU=",
"name": "/tsconfig.tsbuildinfo",
"size": 220798,
"time": "1985-10-26T08:15:00.000Z",
},
],
Array [],
]
`;

exports[`files getFilesList() should get a flat list of files 1`] = `
Array [
Object {
"hash": "+uxmYs/88pYWmLwFS3M54NGjE+hX6sBmwNOpzaW1LGk=",
"name": "/bin/jest.js",
"size": 343,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "MvOGr1Lc6r8wEe8GNmscm3Sx/QWEFE4Is1AZ5rQzFr8=",
"name": "/build/jest.d.ts",
"size": 291,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "BEQ5sRqArzHCh5sNbwjxHRQunhxkCD1HXcM9EdYAKPc=",
"name": "/build/jest.d.ts.map",
"size": 171,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "m5wVGuVr5Pq4z5L2vpeMVA3rbLV4kQ0MCPuo0newsmY=",
"name": "/build/jest.js",
"size": 1030,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "m/vOMvpK2FU19W9PYavnEExEToN7HHU1mb/f/ooU3eQ=",
"name": "/LICENSE",
"size": 1099,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "9hWvkPsgtCTc1w0lswu1AO+Q+S19Dppeg5bNklG/Khg=",
"name": "/package.json",
"size": 925,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "MPu0d2f8or6adBXZZLUNh6vL7Yeg34MmOBiupdclu10=",
"name": "/README.md",
"size": 551,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "CycshPBWVvIRZozw+b1pnAvKYC1Q7aPvcT8tS+HPepU=",
"name": "/tsconfig.json",
"size": 162,
"time": "1985-10-26T08:15:00.000Z",
},
Object {
"hash": "kt2uoTK/NmyQe2OUiNwpdwxV4RxgS2gW9rEgOtj+lZU=",
"name": "/tsconfig.tsbuildinfo",
"size": 220798,
"time": "1985-10-26T08:15:00.000Z",
},
]
`;
84 changes: 84 additions & 0 deletions src/jsDelivr/__test__/index.test.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
import * as api from '../index.js';

describe('hits', () => {
beforeAll(async () => {
await api.loadHits();
});
describe('loadHits()', () => {
it('should download all packages hits', () => {
expect(api.hits.size).toBeGreaterThan(60000); // 66790 (2019-08)
});

it('should get one hit', () => {
expect(api.hits.get('jquery')).toBeGreaterThan(1000000000); // 1065750968 (2019-08)
});

it('should not get one hit', () => {
expect(api.hits.get('thispackagedoesnotexist')).toBe(undefined);
});
});

describe('getHits()', () => {
it('should get one formatted hit', () => {
expect(api.getHits(['jquery'])).toEqual([
{
jsDelivrHits: expect.any(Number),
_searchInternal: {
jsDelivrPopularity: expect.any(Number),
},
},
]);
});
it('should get multiple formatted hits', () => {
expect(api.getHits(['jquery', 'thispackagedoesnotexist'])).toEqual([
{
jsDelivrHits: expect.any(Number),
_searchInternal: {
jsDelivrPopularity: expect.any(Number),
},
},
{
jsDelivrHits: 0,
_searchInternal: {
jsDelivrPopularity: 0,
},
},
]);
});
});
});

describe('files', () => {
describe('getFilesList()', () => {
it('should get a flat list of files', async () => {
const files = await api.getFilesList({ name: '[email protected]' });
expect(files).toMatchSnapshot();
});

it('should not get a files list', async () => {
const files = await api.getFilesList({
name: '[email protected]',
});
expect(files).toEqual([]);
});
});

describe('getAllFilesList()', () => {
it('should get a flat list of files', async () => {
const files = await api.getAllFilesList([{ name: '[email protected]' }]);
expect(files).toMatchSnapshot();
});

it('should get multiple flat list of files', async () => {
const files = await api.getAllFilesList([
{
name: '[email protected]',
},
{
name: '[email protected]',
},
]);
expect(files).toMatchSnapshot();
});
});
});
Loading

0 comments on commit 373d341

Please sign in to comment.