Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Generate and analyze documents with Khoj using the Code sandbox #960

Merged
merged 7 commits into from
Nov 12, 2024
Merged

Generate and analyze documents with Khoj using the Code sandbox #960

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7b39f20
Enable analysing user documents in code sandbox and other improvements
debanjum Nov 5, 2024
af02157
Decode code text output files from b64 to str to ease client processing
debanjum Nov 9, 2024
92c1efe
Fixes to render & save code context with non text based output modes
debanjum Nov 11, 2024
8e9f426
Render code output files with code references in reference section
debanjum Nov 11, 2024
b970cfd
Align styling of reference panel card across code, docs, web results
debanjum Nov 11, 2024
218eed8
Show output file not code on hover. Remove reference card title border
debanjum Nov 12, 2024
a52500d
Show generated code artifacts before notes and online references
debanjum Nov 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 5 additions & 28 deletions src/interface/web/app/common/chatFunctions.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,33 +139,6 @@ export function processMessageChunk(
if (onlineContext) currentMessage.onlineContext = onlineContext;
if (context) currentMessage.context = context;

// Replace file links with base64 data
currentMessage.rawResponse = renderCodeGenImageInline(
currentMessage.rawResponse,
codeContext,
);

// Add code context files to the message
if (codeContext) {
Object.entries(codeContext).forEach(([key, value]) => {
value.results.output_files?.forEach((file) => {
if (file.filename.endsWith(".png") || file.filename.endsWith(".jpg")) {
// Don't add the image again if it's already in the message!
if (!currentMessage.rawResponse.includes(`![${file.filename}](`)) {
currentMessage.rawResponse += `\n\n![${file.filename}](data:image/png;base64,${file.b64_data})`;
}
} else if (
file.filename.endsWith(".txt") ||
file.filename.endsWith(".org") ||
file.filename.endsWith(".md")
) {
const decodedText = atob(file.b64_data);
currentMessage.rawResponse += `\n\n\`\`\`\n${decodedText}\n\`\`\``;
}
});
});
}

// Mark current message streaming as completed
currentMessage.completed = true;
}
Expand Down Expand Up @@ -200,9 +173,13 @@ export function renderCodeGenImageInline(message: string, codeContext: CodeConte
Object.values(codeContext).forEach((contextData) => {
contextData.results.output_files?.forEach((file) => {
const regex = new RegExp(`!?\\[.*?\\]\\(.*${file.filename}\\)`, "g");
if (file.filename.match(/\.(png|jpg|jpeg|gif|webp)$/i)) {
if (file.filename.match(/\.(png|jpg|jpeg)$/i)) {
const replacement = `![${file.filename}](data:image/${file.filename.split(".").pop()};base64,${file.b64_data})`;
message = message.replace(regex, replacement);
} else if (file.filename.match(/\.(txt|org|md|csv|json)$/i)) {
// render output files generated by codegen as downloadable links
const replacement = `![${file.filename}](data:text/plain;base64,${file.b64_data})`;
message = message.replace(regex, replacement);
}
});
});
Expand Down
8 changes: 7 additions & 1 deletion src/interface/web/app/common/iconUtils.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ import {
Leaf,
NewspaperClipping,
OrangeSlice,
Rainbow,
SmileyMelting,
YinYang,
SneakerMove,
Expand Down Expand Up @@ -247,6 +246,13 @@ function getIconFromFilename(
case "doc":
case "docx":
return <MicrosoftWordLogo className={className} />;
case "csv":
case "json":
return <MathOperations className={className} />;
case "txt":
return <Notebook className={className} />;
case "py":
return <Code className={className} />;
case "jpg":
case "jpeg":
case "png":
Expand Down
60 changes: 18 additions & 42 deletions src/interface/web/app/components/chatMessage/chatMessage.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,24 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
message += `\n\n${inferredQueries[0]}`;
}
}

// Replace file links with base64 data
message = renderCodeGenImageInline(message, props.chatMessage.codeContext);

// Add code context files to the message
if (props.chatMessage.codeContext) {
Object.entries(props.chatMessage.codeContext).forEach(([key, value]) => {
value.results.output_files?.forEach((file) => {
if (file.filename.endsWith(".png") || file.filename.endsWith(".jpg")) {
// Don't add the image again if it's already in the message!
if (!message.includes(`![${file.filename}](`)) {
message += `\n\n![${file.filename}](data:image/png;base64,${file.b64_data})`;
}
}
});
});
}

// Handle user attached images rendering
let messageForClipboard = message;
let messageToRender = message;
Expand All @@ -445,48 +463,6 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
messageToRender = `${userImagesInHtml}${messageToRender}`;
}

if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image") {
message = `![generated image](data:image/png;base64,${message})`;
} else if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image2") {
message = `![generated image](${message})`;
} else if (
props.chatMessage.intent &&
props.chatMessage.intent.type == "text-to-image-v3"
) {
message = `![generated image](data:image/webp;base64,${message})`;
}
if (
props.chatMessage.intent &&
props.chatMessage.intent.type.includes("text-to-image") &&
props.chatMessage.intent["inferred-queries"]?.length > 0
) {
message += `\n\n${props.chatMessage.intent["inferred-queries"][0]}`;
}

// Replace file links with base64 data
message = renderCodeGenImageInline(message, props.chatMessage.codeContext);

// Add code context files to the message
if (props.chatMessage.codeContext) {
Object.entries(props.chatMessage.codeContext).forEach(([key, value]) => {
value.results.output_files?.forEach((file) => {
if (file.filename.endsWith(".png") || file.filename.endsWith(".jpg")) {
// Don't add the image again if it's already in the message!
if (!message.includes(`![${file.filename}](`)) {
message += `\n\n![${file.filename}](data:image/png;base64,${file.b64_data})`;
}
} else if (
file.filename.endsWith(".txt") ||
file.filename.endsWith(".org") ||
file.filename.endsWith(".md")
) {
const decodedText = atob(file.b64_data);
message += `\n\n\`\`\`\n${decodedText}\n\`\`\``;
}
});
});
}

// Set the message text
setTextRendered(messageForClipboard);

Expand Down
Loading
Loading