Skip to content

Commit

Permalink
Link domains to host.io lookup, group duplicate accesses per second, …
Browse files Browse the repository at this point in the history 
…and fix dropping multiple files at once.
  • Loading branch information
@johnspurlock
johnspurlock committed Jun 14, 2021
1 parent f78d518 commit e0bd9cd
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 20 deletions.
77 changes: 58 additions & 19 deletions src/app.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ console.log(`HTTP webserver running. Access it at: ${origin}/`);

const dbName = 'reports.db';
const FILENAME = '[a-zA-Z0-9-]+';
const DEBUG = false;

for await (const request of server) {
console.log(`${request.method} ${request.url}`);
Expand All @@ -23,7 +24,7 @@ for await (const request of server) {
doWithDatabase(db => {
const result = handleHtml(db, url);
if (typeof result === 'string') {
request.respond({ status: 200, body: result });
request.respond({ status: 200, body: result, headers: new Headers({ 'Content-Type': 'text/html; charset=utf-8' }) });
} else if (typeof result === 'number') {
request.respond({ status: 404, body: 'not found' });
} else {
Expand Down Expand Up @@ -73,13 +74,15 @@ function computeFilename(request: ServerRequest) {
}

async function handlePost(request: ServerRequest): Promise<Record<string, unknown>> {
for (const [name, value] of request.headers.entries()) {
console.log(`${name}: ${value}`);
if (DEBUG) {
for (const [name, value] of request.headers.entries()) {
console.log(`${name}: ${value}`);
}
}
const filename = computeFilename(request);

const bytes = await readAll(request.body);
console.log(`${bytes.length} bytes`);
if (DEBUG) console.log(`${bytes.length} bytes`);
const decoder = new TextDecoder('utf-8');
const text = decoder.decode(bytes);
doWithDatabase(db => {
Expand Down Expand Up @@ -164,17 +167,35 @@ function handleHtml(db: Database, url: URL): string | { redirectHref: string } |
lines.push(`<tr><td colspan="5"><h3><a href="${computeHref(url, 'date', date)}">${date}</a></h3></td></tr>`);

const showDomains = type === undefined || !type.startsWith('access');
for (const summary of commonSummariesByDate.get(date)!) {
const { accessSummary, domainSummary } = summary;
const commonSummariesForDate = commonSummariesByDate.get(date)!;
for (let i = 0; i < commonSummariesForDate.length; i++) {
const { accessSummary, domainSummary } = commonSummariesForDate[i];
if (accessSummary) {
const type = 'access/' + accessSummary.stream;
const streamLink = `<a href="${computeHref(url, 'type', type)}">${type}</a>`;
let typeLink = `<a href="${computeHref(url, 'type', type)}">${type}</a>`;
const bundleIdLink = `<a href="${computeHref(url, 'bundleId', accessSummary.bundleId)}">${accessSummary.bundleId}</a>`;
lines.push(`<tr><td>${accessSummary.timestampStart ? formatTimestamp(accessSummary.timestampStart) : ""}</td><td>${accessSummary.timestampEnd ? formatTimestamp(accessSummary.timestampEnd) : ''}</td><td>${bundleIdLink}</td><td></td><td>${streamLink}</td></tr>`);
const time = formatTimestamp(accessSummary.timestampStart);
const timeEnd = accessSummary.timestampEnd ? formatTimestamp(accessSummary.timestampEnd) : '';
let count = 1;
while (timeEnd === '' && i < (commonSummariesForDate.length - 1)) {
// coalesce access duplicates for same time second
// there can be multiple address book accesses in the same second, for example
const { accessSummary: nextAccessSummary } = commonSummariesForDate[i + 1];
if (nextAccessSummary && nextAccessSummary.stream === accessSummary.stream && nextAccessSummary.bundleId === accessSummary.bundleId && formatTimestamp(nextAccessSummary.timestampStart) === time) {
count++;
i++;
} else {
break;
}
}
if (count > 1) typeLink += ` x${count}`;
lines.push(`<tr><td>${time}</td><td>${timeEnd}</td><td>${bundleIdLink}</td><td></td><td>${typeLink}</td></tr>`);
}
if (domainSummary && showDomains) {
const bundleIdLink = `<a href="${computeHref(url, 'bundleId', domainSummary.bundleId)}">${domainSummary.bundleId}</a>`;
lines.push(`<tr><td>${formatTimestamp(domainSummary.timestamp)}</td><td></td><td>${bundleIdLink}</td><td>${domainSummary.hits}</td><td>${domainSummary.domain}</td></tr>`);
const domainLocal = domainSummary.domain.endsWith('.local');
const domainHtml = domainLocal ? domainSummary.domain : `<a class="domain" href="https://host.io/${domainSummary.domain}" target="_blank" rel="noreferrer noopener nofollow">${domainSummary.domain}</a>`;
lines.push(`<tr><td>${formatTimestamp(domainSummary.timestamp)}</td><td></td><td>${bundleIdLink}</td><td>${domainSummary.hits}</td><td>${domainHtml}</td></tr>`);
}
}
}
Expand All @@ -196,19 +217,34 @@ function handleHtml(db: Database, url: URL): string | { redirectHref: string } |
const header = document.getElementsByTagName('header')[0];
header.textContent = 'Importing...';
try {
const files = [];
if (event.dataTransfer.items) {
for (let item of event.dataTransfer.items) {
if (item.kind === 'file') {
processFile(item.getAsFile(), header);
files.push(item.getAsFile());
} else {
console.log('Bad item.kind: expected file, found ' + item.kind, item);
}
}
} else {
for (let file of event.dataTransfer.files) {
processFile(file, header);
files.push(file);
}
}
if (files.length === 0) {
header.textContent = 'Nothing to import';
return;
}
Promise.all(files.map(importFile)).then(results => {
const errors = results.filter(v => v.error);
if (errors.length > 0) {
header.textContent = errors.map(v => v.error).join(', ');
}
const lastFilename = results.filter(v => v.filename).map(v => v.filename).pop();
if (lastFilename) {
document.location = '/' + lastFilename;
}
});
} catch (e) {
header.textContent = e;
}
Expand All @@ -218,18 +254,15 @@ function handleHtml(db: Database, url: URL): string | { redirectHref: string } |
event.preventDefault();
}
function processFile(file, header) {
function importFile(file) {
let status = 0;
fetch('/', { method: 'POST', body: file, headers: { 'x-filename': file.name } }).then(v => { status = v.status; return v.json(); }).then(v => {
return fetch('/', { method: 'POST', body: file, headers: { 'x-filename': file.name } }).then(v => { status = v.status; return v.json(); }).then(v => {
if (status !== 200) {
console.error('fetch failed', v.errorDetail);
header.textContent = v.error;
return;
return { error: v.error, errorDetail: v.errorDetail };
}
document.location = '/' + v.filename;
return v;
}).catch(e => {
console.error('fetch failed', e);
header.textContent = e;
return { error: e.toString(), errorDetail: e.stack };
});
}
</script>
Expand Down Expand Up @@ -291,6 +324,12 @@ function handleHtml(db: Database, url: URL): string | { redirectHref: string } |
color: blue;
}
a.domain:hover:after {
position: relative;
content: " lookup ↗︎";
color: #888888;
}
h3 {
margin: 1rem 0;
}
Expand Down
3 changes: 3 additions & 0 deletions src/database.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,9 @@ export class Database {
let date = timestampStart ? timestampStart.substring(0, 10) : undefined;
const existing = summariesByIdentifier.get(identifier);
if (!existing) {
// ensure timestampStart & date have initial defined values (in case we don't get an intervalStart), this is better than nothing
timestampStart = timestamp;
date = timestampStart.substring(0, 10);
summariesByIdentifier.set(identifier, { date, stream: computeStream(stream, tccService), bundleId: accessorIdentifier, timestampStart, timestampEnd });
} else {
timestampStart = timestampStart || existing.timestampStart;
Expand Down
2 changes: 1 addition & 1 deletion src/importer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { isAccessRecord, isDomainRecord } from './model.ts';

export function importReportFile(text: string, filename: string, db: Database) {
const lines = text.split('\n');
console.log(`${lines.length} lines`);
console.log(`importReportFile: ${lines.length} lines`);

db.clearAccess(filename);

Expand Down

0 comments on commit e0bd9cd

Please sign in to comment.