Skip to content

Commit

Permalink
Fixed #118
Browse files Browse the repository at this point in the history 
Fixed #116

Fixed #115

Fixed #98

Fixed #106 again

Implemented #74
(/app/logs)

Implemented #88

Added a loggerService.js to log all messages to a txt and html file
  • Loading branch information
@clusterzx
clusterzx committed Jan 13, 2025
1 parent 39a918d commit 646b8c9
Show file tree
Hide file tree
Showing 9 changed files with 480 additions and 149 deletions.
5 changes: 4 additions & 1 deletion .dockerignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,7 @@ docker-compose-dev.yml
public/images/
delete_all.js
documents.json
apitest.js
apitest.js
logs/*
api_correspondent.js
prompt.txt
5 changes: 4 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,7 @@ docker-compose-dev.yml
public/images/
delete_all.js
documents.json
apitest.js
apitest.js
logs/*
api_correspondent.js
prompt.txt
2 changes: 1 addition & 1 deletion config/config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ console.log('Loaded environment variables:', {
});

module.exports = {
PAPERLESS_AI_VERSION: '2.1.6',
PAPERLESS_AI_VERSION: '2.1.7',
CONFIGURED: false,
predefinedMode: process.env.PROCESS_PREDEFINED_DOCUMENTS,
paperless: {
Expand Down
202 changes: 116 additions & 86 deletions routes/setup.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -512,8 +512,6 @@ router.get('/dashboard', async (req, res) => {
const averageTotalTokens = metrics.length > 0 ? Math.round(metrics.reduce((acc, cur) => acc + cur.totalTokens, 0) / metrics.length) : 0;
const tokensOverall = metrics.length > 0 ? metrics.reduce((acc, cur) => acc + cur.totalTokens, 0) : 0;
const version = configFile.PAPERLESS_AI_VERSION || ' ';
console.log(tagCount);
console.log(correspondentCount);
res.render('dashboard', { paperless_data: { tagCount, correspondentCount, documentCount, processedDocumentCount }, openai_data: { averagePromptTokens, averageCompletionTokens, averageTotalTokens, tokensOverall }, version });
});

Expand Down Expand Up @@ -611,14 +609,15 @@ router.get('/debug/correspondents', async (req, res) => {
router.post('/manual/analyze', express.json(), async (req, res) => {
try {
const { content, existingTags, id } = req.body;

let existingCorrespondentList = await paperlessService.listCorrespondentsNames();
existingCorrespondentList = existingCorrespondentList.map(correspondent => correspondent.name);
if (!content || typeof content !== 'string') {
console.log('Invalid content received:', content);
return res.status(400).json({ error: 'Valid content string is required' });
}

if (process.env.AI_PROVIDER === 'openai') {
const analyzeDocument = await openaiService.analyzeDocument(content, existingTags, id || []);
const analyzeDocument = await openaiService.analyzeDocument(content, existingTags, existingCorrespondentList, id || []);
await documentModel.addOpenAIMetrics(
id,
analyzeDocument.metrics.promptTokens,
Expand Down Expand Up @@ -853,100 +852,131 @@ router.post('/setup', express.json(), async (req, res) => {

router.post('/settings', express.json(), async (req, res) => {
try {
const {
paperlessUrl,
paperlessToken,
aiProvider,
openaiKey,
openaiModel,
ollamaUrl,
ollamaModel,
scanInterval,
systemPrompt,
showTags,
tags,
aiProcessedTag,
aiTagName,
usePromptTags,
promptTags,
paperlessUsername
} = req.body;

const normalizeArray = (value) => {
if (!value) return [];
if (Array.isArray(value)) return value;
if (typeof value === 'string') return value.split(',').filter(Boolean).map(item => item.trim());
return [];
};
const {
paperlessUrl,
paperlessToken,
aiProvider,
openaiKey,
openaiModel,
ollamaUrl,
ollamaModel,
scanInterval,
systemPrompt,
showTags,
tags,
aiProcessedTag,
aiTagName,
usePromptTags,
promptTags,
paperlessUsername
} = req.body;

const currentConfig = {
PAPERLESS_API_URL: process.env.PAPERLESS_API_URL || '',
PAPERLESS_API_TOKEN: process.env.PAPERLESS_API_TOKEN || '',
PAPERLESS_USERNAME: process.env.PAPERLESS_USERNAME || '',
AI_PROVIDER: process.env.AI_PROVIDER || '',
OPENAI_API_KEY: process.env.OPENAI_API_KEY || '',
OPENAI_MODEL: process.env.OPENAI_MODEL || '',
OLLAMA_API_URL: process.env.OLLAMA_API_URL || '',
OLLAMA_MODEL: process.env.OLLAMA_MODEL || '',
SCAN_INTERVAL: process.env.SCAN_INTERVAL || '*/30 * * * *',
SYSTEM_PROMPT: process.env.SYSTEM_PROMPT || '',
PROCESS_PREDEFINED_DOCUMENTS: process.env.PROCESS_PREDEFINED_DOCUMENTS || 'no',
TAGS: process.env.TAGS || '',
ADD_AI_PROCESSED_TAG: process.env.ADD_AI_PROCESSED_TAG || 'no',
AI_PROCESSED_TAG_NAME: process.env.AI_PROCESSED_TAG_NAME || 'ai-processed',
USE_PROMPT_TAGS: process.env.USE_PROMPT_TAGS || 'no',
PROMPT_TAGS: process.env.PROMPT_TAGS || ''
};

const processedPrompt = systemPrompt
? systemPrompt.replace(/\r\n/g, '\n').replace(/\n/g, '\\n')
: '';
const normalizeArray = (value) => {
if (!value) return [];
if (Array.isArray(value)) return value;
if (typeof value === 'string') return value.split(',').filter(Boolean).map(item => item.trim());
return [];
};

// Validate Paperless config
if (paperlessUrl !== currentConfig.PAPERLESS_API_URL?.replace('/api', '') ||
paperlessToken !== currentConfig.PAPERLESS_API_TOKEN) {
const isPaperlessValid = await setupService.validatePaperlessConfig(paperlessUrl, paperlessToken);
if (!isPaperlessValid) {
return res.status(400).json({
error: 'Paperless-ngx connection failed. Please check URL and Token.'
});
return res.status(400).json({
error: 'Paperless-ngx connection failed. Please check URL and Token.'
});
}
}

// Prepare base config
const config = {
PAPERLESS_API_URL: paperlessUrl + '/api',
PAPERLESS_API_TOKEN: paperlessToken,
PAPERLESS_USERNAME: paperlessUsername,
AI_PROVIDER: aiProvider,
SCAN_INTERVAL: scanInterval || '*/30 * * * *',
SYSTEM_PROMPT: processedPrompt,
PROCESS_PREDEFINED_DOCUMENTS: showTags || 'no',
TAGS: normalizeArray(tags),
ADD_AI_PROCESSED_TAG: aiProcessedTag || 'no',
AI_PROCESSED_TAG_NAME: aiTagName || 'ai-processed',
USE_PROMPT_TAGS: usePromptTags || 'no',
PROMPT_TAGS: normalizeArray(promptTags)
};
const updatedConfig = {};

// Validate AI provider config
if (aiProvider === 'openai') {
const isOpenAIValid = await setupService.validateOpenAIConfig(openaiKey);
if (!isOpenAIValid) {
return res.status(400).json({
error: 'OpenAI API Key is not valid. Please check the key.'
});
}
config.OPENAI_API_KEY = openaiKey;
config.OPENAI_MODEL = openaiModel || 'gpt-4o-mini';
} else if (aiProvider === 'ollama') {
const isOllamaValid = await setupService.validateOllamaConfig(ollamaUrl, ollamaModel);
if (!isOllamaValid) {
return res.status(400).json({
error: 'Ollama connection failed. Please check URL and Model.'
});
}
config.OLLAMA_API_URL = ollamaUrl || 'http://localhost:11434';
config.OLLAMA_MODEL = ollamaModel || 'llama3.2';
if (paperlessUrl) updatedConfig.PAPERLESS_API_URL = paperlessUrl + '/api';
if (paperlessToken) updatedConfig.PAPERLESS_API_TOKEN = paperlessToken;
if (paperlessUsername) updatedConfig.PAPERLESS_USERNAME = paperlessUsername;

if (aiProvider) {
updatedConfig.AI_PROVIDER = aiProvider;

if (aiProvider === 'openai' && openaiKey) {
const isOpenAIValid = await setupService.validateOpenAIConfig(openaiKey);
if (!isOpenAIValid) {
return res.status(400).json({
error: 'OpenAI API Key is not valid. Please check the key.'
});
}
updatedConfig.OPENAI_API_KEY = openaiKey;
if (openaiModel) updatedConfig.OPENAI_MODEL = openaiModel;
}
else if (aiProvider === 'ollama' && (ollamaUrl || ollamaModel)) {
const isOllamaValid = await setupService.validateOllamaConfig(
ollamaUrl || currentConfig.OLLAMA_API_URL,
ollamaModel || currentConfig.OLLAMA_MODEL
);
if (!isOllamaValid) {
return res.status(400).json({
error: 'Ollama connection failed. Please check URL and Model.'
});
}
if (ollamaUrl) updatedConfig.OLLAMA_API_URL = ollamaUrl;
if (ollamaModel) updatedConfig.OLLAMA_MODEL = ollamaModel;
}
}

// Save configuration
await setupService.saveConfig(config);
// Send success response
res.json({
success: true,
message: 'Configuration saved successfully.',
restart: true
});
if (scanInterval) updatedConfig.SCAN_INTERVAL = scanInterval;
if (systemPrompt) updatedConfig.SYSTEM_PROMPT = systemPrompt.replace(/\r\n/g, '\n').replace(/\n/g, '\\n');
if (showTags) updatedConfig.PROCESS_PREDEFINED_DOCUMENTS = showTags;
if(tags !== undefined || tags !== null || tags !== ''){
updatedConfig.TAGS = normalizeArray(tags);
}else{
updatedConfig.TAGS = '';
}
if (aiProcessedTag) updatedConfig.ADD_AI_PROCESSED_TAG = aiProcessedTag;
if (aiTagName) updatedConfig.AI_PROCESSED_TAG_NAME = aiTagName;
if (usePromptTags) updatedConfig.USE_PROMPT_TAGS = usePromptTags;
if (promptTags) updatedConfig.PROMPT_TAGS = normalizeArray(promptTags);

const mergedConfig = {
...currentConfig,
...updatedConfig
};

// Trigger application restart
setTimeout(() => {
process.exit(0);
}, 5000);
await setupService.saveConfig(mergedConfig);

res.json({
success: true,
message: 'Configuration saved successfully.',
restart: true
});

// Trigger application restart
setTimeout(() => {
process.exit(0);
}, 5000);

} catch (error) {
console.error('Setup error:', error);
res.status(500).json({
error: 'An error occurred: ' + error.message
});
console.error('Setup error:', error);
res.status(500).json({
error: 'An error occurred: ' + error.message
});
}
});

Expand Down
39 changes: 31 additions & 8 deletions server.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,22 @@ const setupService = require('./services/setupService');
const setupRoutes = require('./routes/setup');
const cors = require('cors');
const cookieParser = require('cookie-parser');
const Logger = require('./services/loggerService');
const { max } = require('date-fns');

const htmlLogger = new Logger({
logFile: 'logs.html',
format: 'html',
timestamp: true,
maxFileSize: 1024 * 1024 * 10
});

const txtLogger = new Logger({
logFile: 'logs.txt',
format: 'txt',
timestamp: true,
maxFileSize: 1024 * 1024 * 10
});

const app = express();
let runningTask = false;
Expand Down Expand Up @@ -49,7 +65,7 @@ async function initializeDataDirectory() {
}

// Document processing functions
async function processDocument(doc, existingTags, ownUserId) {
async function processDocument(doc, existingTags, existingCorrespondentList, ownUserId) {
const isProcessed = await documentModel.isDocumentProcessed(doc.id);
if (isProcessed) return null;

Expand All @@ -75,7 +91,7 @@ async function processDocument(doc, existingTags, ownUserId) {
}

const aiService = AIServiceFactory.getService();
const analysis = await aiService.analyzeDocument(content, existingTags, doc.id);
const analysis = await aiService.analyzeDocument(content, existingTags, existingCorrespondentList, doc.id);

if (analysis.error) {
throw new Error(`[ERROR] Document analysis failed: ${analysis.error}`);
Expand Down Expand Up @@ -140,15 +156,18 @@ async function scanInitial() {
return;
}

const [existingTags, documents, ownUserId] = await Promise.all([
let [existingTags, documents, ownUserId, existingCorrespondentList] = await Promise.all([
paperlessService.getTags(),
paperlessService.getAllDocuments(),
paperlessService.getOwnUserID()
paperlessService.getOwnUserID(),
paperlessService.listCorrespondentsNames()
]);
//get existing correspondent list
existingCorrespondentList = existingCorrespondentList.map(correspondent => correspondent.name);

for (const doc of documents) {
try {
const result = await processDocument(doc, existingTags, ownUserId);
const result = await processDocument(doc, existingTags, existingCorrespondentList, ownUserId);
if (!result) continue;

const { analysis, originalData } = result;
Expand All @@ -171,15 +190,19 @@ async function scanDocuments() {

runningTask = true;
try {
const [existingTags, documents, ownUserId] = await Promise.all([
let [existingTags, documents, ownUserId, existingCorrespondentList] = await Promise.all([
paperlessService.getTags(),
paperlessService.getAllDocuments(),
paperlessService.getOwnUserID()
paperlessService.getOwnUserID(),
paperlessService.listCorrespondentsNames()
]);

//get existing correspondent list
existingCorrespondentList = existingCorrespondentList.map(correspondent => correspondent.name);

for (const doc of documents) {
try {
const result = await processDocument(doc, existingTags, ownUserId);
const result = await processDocument(doc, existingTags, existingCorrespondentList, ownUserId);
if (!result) continue;

const { analysis, originalData } = result;
Expand Down
Loading

0 comments on commit 646b8c9

Please sign in to comment.