diff --git a/.dockerignore b/.dockerignore index 4ec2eb9..f3dc64b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,4 +19,7 @@ docker-compose-dev.yml public/images/ delete_all.js documents.json -apitest.js \ No newline at end of file +apitest.js +logs/* +api_correspondent.js +prompt.txt \ No newline at end of file diff --git a/.gitignore b/.gitignore index 5ff2f5c..e6f6628 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,7 @@ docker-compose-dev.yml public/images/ delete_all.js documents.json -apitest.js \ No newline at end of file +apitest.js +logs/* +api_correspondent.js +prompt.txt \ No newline at end of file diff --git a/config/config.js b/config/config.js index d9e7b2f..ff44c52 100644 --- a/config/config.js +++ b/config/config.js @@ -11,7 +11,7 @@ console.log('Loaded environment variables:', { }); module.exports = { - PAPERLESS_AI_VERSION: '2.1.6', + PAPERLESS_AI_VERSION: '2.1.7', CONFIGURED: false, predefinedMode: process.env.PROCESS_PREDEFINED_DOCUMENTS, paperless: { diff --git a/routes/setup.js b/routes/setup.js index 8cb5883..099d9cc 100644 --- a/routes/setup.js +++ b/routes/setup.js @@ -512,8 +512,6 @@ router.get('/dashboard', async (req, res) => { const averageTotalTokens = metrics.length > 0 ? Math.round(metrics.reduce((acc, cur) => acc + cur.totalTokens, 0) / metrics.length) : 0; const tokensOverall = metrics.length > 0 ? metrics.reduce((acc, cur) => acc + cur.totalTokens, 0) : 0; const version = configFile.PAPERLESS_AI_VERSION || ' '; - console.log(tagCount); - console.log(correspondentCount); res.render('dashboard', { paperless_data: { tagCount, correspondentCount, documentCount, processedDocumentCount }, openai_data: { averagePromptTokens, averageCompletionTokens, averageTotalTokens, tokensOverall }, version }); }); @@ -611,14 +609,15 @@ router.get('/debug/correspondents', async (req, res) => { router.post('/manual/analyze', express.json(), async (req, res) => { try { const { content, existingTags, id } = req.body; - + let existingCorrespondentList = await paperlessService.listCorrespondentsNames(); + existingCorrespondentList = existingCorrespondentList.map(correspondent => correspondent.name); if (!content || typeof content !== 'string') { console.log('Invalid content received:', content); return res.status(400).json({ error: 'Valid content string is required' }); } if (process.env.AI_PROVIDER === 'openai') { - const analyzeDocument = await openaiService.analyzeDocument(content, existingTags, id || []); + const analyzeDocument = await openaiService.analyzeDocument(content, existingTags, existingCorrespondentList, id || []); await documentModel.addOpenAIMetrics( id, analyzeDocument.metrics.promptTokens, @@ -853,100 +852,131 @@ router.post('/setup', express.json(), async (req, res) => { router.post('/settings', express.json(), async (req, res) => { try { - const { - paperlessUrl, - paperlessToken, - aiProvider, - openaiKey, - openaiModel, - ollamaUrl, - ollamaModel, - scanInterval, - systemPrompt, - showTags, - tags, - aiProcessedTag, - aiTagName, - usePromptTags, - promptTags, - paperlessUsername - } = req.body; - - const normalizeArray = (value) => { - if (!value) return []; - if (Array.isArray(value)) return value; - if (typeof value === 'string') return value.split(',').filter(Boolean).map(item => item.trim()); - return []; - }; + const { + paperlessUrl, + paperlessToken, + aiProvider, + openaiKey, + openaiModel, + ollamaUrl, + ollamaModel, + scanInterval, + systemPrompt, + showTags, + tags, + aiProcessedTag, + aiTagName, + usePromptTags, + promptTags, + paperlessUsername + } = req.body; + + const currentConfig = { + PAPERLESS_API_URL: process.env.PAPERLESS_API_URL || '', + PAPERLESS_API_TOKEN: process.env.PAPERLESS_API_TOKEN || '', + PAPERLESS_USERNAME: process.env.PAPERLESS_USERNAME || '', + AI_PROVIDER: process.env.AI_PROVIDER || '', + OPENAI_API_KEY: process.env.OPENAI_API_KEY || '', + OPENAI_MODEL: process.env.OPENAI_MODEL || '', + OLLAMA_API_URL: process.env.OLLAMA_API_URL || '', + OLLAMA_MODEL: process.env.OLLAMA_MODEL || '', + SCAN_INTERVAL: process.env.SCAN_INTERVAL || '*/30 * * * *', + SYSTEM_PROMPT: process.env.SYSTEM_PROMPT || '', + PROCESS_PREDEFINED_DOCUMENTS: process.env.PROCESS_PREDEFINED_DOCUMENTS || 'no', + TAGS: process.env.TAGS || '', + ADD_AI_PROCESSED_TAG: process.env.ADD_AI_PROCESSED_TAG || 'no', + AI_PROCESSED_TAG_NAME: process.env.AI_PROCESSED_TAG_NAME || 'ai-processed', + USE_PROMPT_TAGS: process.env.USE_PROMPT_TAGS || 'no', + PROMPT_TAGS: process.env.PROMPT_TAGS || '' + }; - const processedPrompt = systemPrompt - ? systemPrompt.replace(/\r\n/g, '\n').replace(/\n/g, '\\n') - : ''; + const normalizeArray = (value) => { + if (!value) return []; + if (Array.isArray(value)) return value; + if (typeof value === 'string') return value.split(',').filter(Boolean).map(item => item.trim()); + return []; + }; - // Validate Paperless config + if (paperlessUrl !== currentConfig.PAPERLESS_API_URL?.replace('/api', '') || + paperlessToken !== currentConfig.PAPERLESS_API_TOKEN) { const isPaperlessValid = await setupService.validatePaperlessConfig(paperlessUrl, paperlessToken); if (!isPaperlessValid) { - return res.status(400).json({ - error: 'Paperless-ngx connection failed. Please check URL and Token.' - }); + return res.status(400).json({ + error: 'Paperless-ngx connection failed. Please check URL and Token.' + }); } + } - // Prepare base config - const config = { - PAPERLESS_API_URL: paperlessUrl + '/api', - PAPERLESS_API_TOKEN: paperlessToken, - PAPERLESS_USERNAME: paperlessUsername, - AI_PROVIDER: aiProvider, - SCAN_INTERVAL: scanInterval || '*/30 * * * *', - SYSTEM_PROMPT: processedPrompt, - PROCESS_PREDEFINED_DOCUMENTS: showTags || 'no', - TAGS: normalizeArray(tags), - ADD_AI_PROCESSED_TAG: aiProcessedTag || 'no', - AI_PROCESSED_TAG_NAME: aiTagName || 'ai-processed', - USE_PROMPT_TAGS: usePromptTags || 'no', - PROMPT_TAGS: normalizeArray(promptTags) - }; + const updatedConfig = {}; - // Validate AI provider config - if (aiProvider === 'openai') { - const isOpenAIValid = await setupService.validateOpenAIConfig(openaiKey); - if (!isOpenAIValid) { - return res.status(400).json({ - error: 'OpenAI API Key is not valid. Please check the key.' - }); - } - config.OPENAI_API_KEY = openaiKey; - config.OPENAI_MODEL = openaiModel || 'gpt-4o-mini'; - } else if (aiProvider === 'ollama') { - const isOllamaValid = await setupService.validateOllamaConfig(ollamaUrl, ollamaModel); - if (!isOllamaValid) { - return res.status(400).json({ - error: 'Ollama connection failed. Please check URL and Model.' - }); - } - config.OLLAMA_API_URL = ollamaUrl || 'http://localhost:11434'; - config.OLLAMA_MODEL = ollamaModel || 'llama3.2'; + if (paperlessUrl) updatedConfig.PAPERLESS_API_URL = paperlessUrl + '/api'; + if (paperlessToken) updatedConfig.PAPERLESS_API_TOKEN = paperlessToken; + if (paperlessUsername) updatedConfig.PAPERLESS_USERNAME = paperlessUsername; + + if (aiProvider) { + updatedConfig.AI_PROVIDER = aiProvider; + + if (aiProvider === 'openai' && openaiKey) { + const isOpenAIValid = await setupService.validateOpenAIConfig(openaiKey); + if (!isOpenAIValid) { + return res.status(400).json({ + error: 'OpenAI API Key is not valid. Please check the key.' + }); + } + updatedConfig.OPENAI_API_KEY = openaiKey; + if (openaiModel) updatedConfig.OPENAI_MODEL = openaiModel; + } + else if (aiProvider === 'ollama' && (ollamaUrl || ollamaModel)) { + const isOllamaValid = await setupService.validateOllamaConfig( + ollamaUrl || currentConfig.OLLAMA_API_URL, + ollamaModel || currentConfig.OLLAMA_MODEL + ); + if (!isOllamaValid) { + return res.status(400).json({ + error: 'Ollama connection failed. Please check URL and Model.' + }); + } + if (ollamaUrl) updatedConfig.OLLAMA_API_URL = ollamaUrl; + if (ollamaModel) updatedConfig.OLLAMA_MODEL = ollamaModel; } + } - // Save configuration - await setupService.saveConfig(config); - // Send success response - res.json({ - success: true, - message: 'Configuration saved successfully.', - restart: true - }); + if (scanInterval) updatedConfig.SCAN_INTERVAL = scanInterval; + if (systemPrompt) updatedConfig.SYSTEM_PROMPT = systemPrompt.replace(/\r\n/g, '\n').replace(/\n/g, '\\n'); + if (showTags) updatedConfig.PROCESS_PREDEFINED_DOCUMENTS = showTags; + if(tags !== undefined || tags !== null || tags !== ''){ + updatedConfig.TAGS = normalizeArray(tags); + }else{ + updatedConfig.TAGS = ''; + } + if (aiProcessedTag) updatedConfig.ADD_AI_PROCESSED_TAG = aiProcessedTag; + if (aiTagName) updatedConfig.AI_PROCESSED_TAG_NAME = aiTagName; + if (usePromptTags) updatedConfig.USE_PROMPT_TAGS = usePromptTags; + if (promptTags) updatedConfig.PROMPT_TAGS = normalizeArray(promptTags); + + const mergedConfig = { + ...currentConfig, + ...updatedConfig + }; - // Trigger application restart - setTimeout(() => { - process.exit(0); - }, 5000); + await setupService.saveConfig(mergedConfig); + + res.json({ + success: true, + message: 'Configuration saved successfully.', + restart: true + }); + + // Trigger application restart + setTimeout(() => { + process.exit(0); + }, 5000); } catch (error) { - console.error('Setup error:', error); - res.status(500).json({ - error: 'An error occurred: ' + error.message - }); + console.error('Setup error:', error); + res.status(500).json({ + error: 'An error occurred: ' + error.message + }); } }); diff --git a/server.js b/server.js index d32501c..6c0bde4 100644 --- a/server.js +++ b/server.js @@ -10,6 +10,22 @@ const setupService = require('./services/setupService'); const setupRoutes = require('./routes/setup'); const cors = require('cors'); const cookieParser = require('cookie-parser'); +const Logger = require('./services/loggerService'); +const { max } = require('date-fns'); + +const htmlLogger = new Logger({ + logFile: 'logs.html', + format: 'html', + timestamp: true, + maxFileSize: 1024 * 1024 * 10 +}); + +const txtLogger = new Logger({ + logFile: 'logs.txt', + format: 'txt', + timestamp: true, + maxFileSize: 1024 * 1024 * 10 +}); const app = express(); let runningTask = false; @@ -49,7 +65,7 @@ async function initializeDataDirectory() { } // Document processing functions -async function processDocument(doc, existingTags, ownUserId) { +async function processDocument(doc, existingTags, existingCorrespondentList, ownUserId) { const isProcessed = await documentModel.isDocumentProcessed(doc.id); if (isProcessed) return null; @@ -75,7 +91,7 @@ async function processDocument(doc, existingTags, ownUserId) { } const aiService = AIServiceFactory.getService(); - const analysis = await aiService.analyzeDocument(content, existingTags, doc.id); + const analysis = await aiService.analyzeDocument(content, existingTags, existingCorrespondentList, doc.id); if (analysis.error) { throw new Error(`[ERROR] Document analysis failed: ${analysis.error}`); @@ -140,15 +156,18 @@ async function scanInitial() { return; } - const [existingTags, documents, ownUserId] = await Promise.all([ + let [existingTags, documents, ownUserId, existingCorrespondentList] = await Promise.all([ paperlessService.getTags(), paperlessService.getAllDocuments(), - paperlessService.getOwnUserID() + paperlessService.getOwnUserID(), + paperlessService.listCorrespondentsNames() ]); + //get existing correspondent list + existingCorrespondentList = existingCorrespondentList.map(correspondent => correspondent.name); for (const doc of documents) { try { - const result = await processDocument(doc, existingTags, ownUserId); + const result = await processDocument(doc, existingTags, existingCorrespondentList, ownUserId); if (!result) continue; const { analysis, originalData } = result; @@ -171,15 +190,19 @@ async function scanDocuments() { runningTask = true; try { - const [existingTags, documents, ownUserId] = await Promise.all([ + let [existingTags, documents, ownUserId, existingCorrespondentList] = await Promise.all([ paperlessService.getTags(), paperlessService.getAllDocuments(), - paperlessService.getOwnUserID() + paperlessService.getOwnUserID(), + paperlessService.listCorrespondentsNames() ]); + //get existing correspondent list + existingCorrespondentList = existingCorrespondentList.map(correspondent => correspondent.name); + for (const doc of documents) { try { - const result = await processDocument(doc, existingTags, ownUserId); + const result = await processDocument(doc, existingTags, existingCorrespondentList, ownUserId); if (!result) continue; const { analysis, originalData } = result; diff --git a/services/loggerService.js b/services/loggerService.js new file mode 100644 index 0000000..815a3b4 --- /dev/null +++ b/services/loggerService.js @@ -0,0 +1,241 @@ +const fs = require('fs'); +const util = require('util'); +const path = require('path'); + +class Logger { + constructor(options = {}) { + this.logFile = options.logFile || 'application.log'; + this.logDir = options.logDir || 'logs'; + this.timestamp = options.timestamp !== false; + this.format = options.format || 'txt'; + this.maxFileSize = options.maxFileSize || 1024 * 1024 * 10; // Standard: 10MB + + if (!fs.existsSync(this.logDir)) { + fs.mkdirSync(this.logDir, { recursive: true }); + } + + this.logPath = path.join(this.logDir, this.logFile); + + // Initialisiere Log-Datei + this.initLogFile(); + + this.originalConsole = { + log: console.log, + error: console.error, + warn: console.warn, + info: console.info, + debug: console.debug + }; + + this.overrideConsoleMethods(); + } + + initLogFile() { + // Prüfe ob die Datei die maximale Größe überschreitet + if (this.checkFileSize()) { + // Lösche die alte Datei + try { + fs.unlinkSync(this.logPath); + } catch (error) { + // Ignoriere Fehler wenn Datei nicht existiert + } + } + + // Initialisiere HTML-Datei wenn nötig + if (this.format === 'html') { + this.initHtmlFile(); + } + } + + checkFileSize() { + if (fs.existsSync(this.logPath)) { + const stats = fs.statSync(this.logPath); + return stats.size >= this.maxFileSize; + } + return false; + } + + initHtmlFile() { + const htmlHeader = ` + + +
+ +