llm

[program4results]

hi all. not sure this functionality is already in Stirling <
im trying to add two features . 1 validate pdf before ie those that are corrupt and those that are of PDF/A, PDF/E, and PDF/X, or password password protected that need special attanti9on 2. merge and split thousands of pdf that pass test 1 above into 200 mb chunks ie notebooklm compliant. I also running into problems with hanging stirling that just bombs out whilst processing 1500 pdf's despite max its ram . I have hundreds of thousands of pdf to process . I am unable to allocate more cpu due to my synology nas sa3400 restriction which has about 100 gb free ram . ie Environment Variables → Add:
JAVA_TOOL_OPTIONS=-Xms8g -Xmx28g -XX:+UseG1GC
STIRLING_PDF_THREADS=12
PIPELINE_MAX_CONCURRENT=6

i also need a browse and add pdf , browse and add pdf picker :// NEW CONTROLLER: src/main/java/stirling/software/SPDF/controller/api/FileBrowserController.java

@RestController
@RequestMapping("/api/v1/file-browser")
@tag(name = "File Browser", description = "Server-side file browsing and PDF validation")
public class FileBrowserController {

@Autowired
private PDFValidationService pdfValidationService;

@GetMapping("/browse")
@Operation(summary = "Browse server directories", 
           description = "Browse folders on the server filesystem")
public ResponseEntity<DirectoryListing> browseDirectory(
        @RequestParam(defaultValue = "/") String path) {
    
    // Security check - only allow browsing mounted volumes
    if (!isAllowedPath(path)) {
        return ResponseEntity.forbidden().build();
    }
    
    File directory = new File(path);
    if (!directory.exists() || !directory.isDirectory()) {
        return ResponseEntity.notFound().build();
    }
    
    DirectoryListing listing = new DirectoryListing();
    listing.setCurrentPath(path);
    listing.setParentPath(directory.getParent());
    
    File[] files = directory.listFiles();
    if (files != null) {
        for (File file : files) {
            if (file.isDirectory()) {
                listing.addDirectory(new DirectoryInfo(file.getName(), file.getAbsolutePath()));
            } else if (file.getName().toLowerCase().endsWith(".pdf")) {
                PDFInfo pdfInfo = new PDFInfo();
                pdfInfo.setName(file.getName());
                pdfInfo.setPath(file.getAbsolutePath());
                pdfInfo.setSize(file.length());
                pdfInfo.setLastModified(new Date(file.lastModified()));
                listing.addPdf(pdfInfo);
            }
        }
    }
    
    return ResponseEntity.ok(listing);
}

@PostMapping("/validate-pdfs")
@Operation(summary = "Validate selected PDFs", 
           description = "Check PDF integrity and return validation results")
public ResponseEntity<ValidationResults> validatePDFs(
        @RequestBody List<String> pdfPaths) {
    
    ValidationResults results = new ValidationResults();
    
    for (String path : pdfPaths) {
        if (!isAllowedPath(path)) {
            continue; // Skip unauthorized paths
        }
        
        ValidationResult result = pdfValidationService.validatePDF(path);
        results.addResult(result);
    }
    
    return ResponseEntity.ok(results);
}

@PostMapping("/process-batch")
@Operation(summary = "Process validated PDFs", 
           description = "Copy valid PDFs to output directory and process")
public ResponseEntity<BatchProcessResult> processBatch(
        @RequestBody BatchProcessRequest request) {
    
    // Create custom output directory
    String outputDir = "/pipeline/finishedFolders/" + request.getOutputFolderName();
    File outputDirectory = new File(outputDir);
    outputDirectory.mkdirs();
    
    BatchProcessResult result = new BatchProcessResult();
    result.setOutputDirectory(outputDir);
    
    for (String pdfPath : request.getValidPdfPaths()) {
        try {
            File source = new File(pdfPath);
            File destination = new File(outputDirectory, source.getName());
            Files.copy(source.toPath(), destination.toPath(), 
                      StandardCopyOption.REPLACE_EXISTING);
            result.addProcessedFile(source.getName());
        } catch (IOException e) {
            result.addError("Failed to copy: " + pdfPath);
        }
    }
    
    return ResponseEntity.ok(result);
}

private boolean isAllowedPath(String path) {
    // Security: Only allow browsing specific mounted volumes
    return path.startsWith("/input/") || 
           path.startsWith("/volume1/") || 
           path.startsWith("/data/");
}

}

// NEW SERVICE: src/main/java/stirling/software/SPDF/service/PDFValidationService.java

@service
public class PDFValidationService {

private static final Logger logger = LoggerFactory.getLogger(PDFValidationService.class);

public ValidationResult validatePDF(String filePath) {
    ValidationResult result = new ValidationResult();
    result.setFilePath(filePath);
    result.setFileName(new File(filePath).getName());
    
    try {
        // Method 1: PDFBox validation
        boolean pdfboxValid = validateWithPDFBox(filePath);
        
        // Method 2: QPDF validation (if available)
        boolean qpdfValid = validateWithQPDF(filePath);
        
        // Method 3: Basic file structure check
        boolean structureValid = validateFileStructure(filePath);
        
        result.setPdfboxValid(pdfboxValid);
        result.setQpdfValid(qpdfValid);
        result.setStructureValid(structureValid);
        result.setOverallValid(pdfboxValid && structureValid);
        
        if (result.isOverallValid()) {
            // Get additional PDF info
            result.setPageCount(getPageCount(filePath));
            result.setFileSize(new File(filePath).length());
        }
        
    } catch (Exception e) {
        logger.error("Error validating PDF: " + filePath, e);
        result.setOverallValid(false);
        result.setError(e.getMessage());
    }
    
    return result;
}

private boolean validateWithPDFBox(String filePath) {
    try (PDDocument document = PDDocument.load(new File(filePath))) {
        // If we can load it with PDFBox, it's likely valid
        return document.getNumberOfPages() > 0;
    } catch (Exception e) {
        logger.debug("PDFBox validation failed for: " + filePath);
        return false;
    }
}

private boolean validateWithQPDF(String filePath) {
    try {
        // Execute qpdf --check command if available
        ProcessBuilder pb = new ProcessBuilder("qpdf", "--check", filePath);
        Process process = pb.start();
        int exitCode = process.waitFor();
        return exitCode == 0;
    } catch (Exception e) {
        logger.debug("QPDF validation failed or not available for: " + filePath);
        return true; // Don't fail if qpdf is not available
    }
}

private boolean validateFileStructure(String filePath) {
    try (RandomAccessFile raf = new RandomAccessFile(filePath, "r")) {
        // Check PDF header
        byte[] header = new byte[5];
        raf.read(header);
        String headerStr = new String(header);
        
        if (!headerStr.equals("%PDF-")) {
            return false;
        }
        
        // Check for EOF marker
        raf.seek(Math.max(0, raf.length() - 1024));
        byte[] buffer = new byte[1024];
        int bytesRead = raf.read(buffer);
        String endStr = new String(buffer, 0, bytesRead);
        
        return endStr.contains("%%EOF");
        
    } catch (Exception e) {
        return false;
    }
}

private int getPageCount(String filePath) {
    try (PDDocument document = PDDocument.load(new File(filePath))) {
        return document.getNumberOfPages();
    } catch (Exception e) {
        return -1;
    }
}

}

// NEW MODELS: src/main/java/stirling/software/SPDF/model/

public class DirectoryListing {
private String currentPath;
private String parentPath;
private List directories = new ArrayList<>();
private List pdfs = new ArrayList<>();

// getters, setters, add methods...

}

public class PDFInfo {
private String name;
private String path;
private long size;
private Date lastModified;
private boolean selected;

// getters, setters...

}

public class ValidationResult {
private String filePath;
private String fileName;
private boolean overallValid;
private boolean pdfboxValid;
private boolean qpdfValid;
private boolean structureValid;
private int pageCount;
private long fileSize;
private String error;

// getters, setters...

}

public class BatchProcessRequest {
private List validPdfPaths;
private String outputFolderName;

// getters, setters...

}

// FRONTEND TEMPLATE: src/main/resources/templates/pdf-folder-browser.html

<title>PDF Folder Browser & Validator</title>

PDF Folder Browser & Validator

    <!-- Folder Browser -->
    <div class="folder-browser">
        <div class="path-bar">
            <span id="current-path">/</span>
            <button onclick="navigateUp()">↑ Parent</button>
        </div>
        
        <div class="directory-list" id="directory-list">
            <!-- Populated by JavaScript -->
        </div>
    </div>
    
    <!-- Selected PDFs -->
    <div class="selected-files">
        <h3>Selected PDFs (<span id="selected-count">0</span>)</h3>
        <button onclick="validateSelected()">Validate Selected</button>
        <button onclick="clearSelection()">Clear Selection</button>
        
        <div id="selected-list">
            <!-- Populated by JavaScript -->
        </div>
    </div>
    
    <!-- Validation Results -->
    <div class="validation-results" id="validation-results" style="display:none;">
        <h3>Validation Results</h3>
        <div id="validation-summary"></div>
        <div id="validation-details"></div>
        
        <div class="output-options">
            <label>Output Folder Name:</label>
            <input type="text" id="output-folder-name" placeholder="my-processed-pdfs">
            <button onclick="processValidPDFs()">Process Valid PDFs</button>
        </div>
    </div>
</div>

<script src="/js/pdf-folder-browser.js"></script>

this in pre revalidation step-

python stirling_preprocessor.py
"/path/to/your/1700/pdfs"
"/volume1/docker/stirling/pipeline/watchedFolders"
--max-size 180

this is teh docker compose

version: '3.8'

services:
stirling-pdf:
image: frooodle/s-pdf:latest
container_name: stirling-pdf-turbo
ports:
- '8092:8080'

# MEMORY ALLOCATION ONLY (CPU limits removed for Synology compatibility)
mem_limit: 32g
mem_reservation: 16g

# PERSISTENT VOLUMES - MATCHES YOUR EXISTING SETUP
volumes:
  # Your existing structure (keep these exactly as they are)
  - ${DOCKER_VOLUME:-/volume1/docker/stirling}/trainingData:/usr/share/tessdata:rw
  - ${DOCKER_VOLUME:-/volume1/docker/stirling}/extraConfigs:/configs:rw
  - ${DOCKER_VOLUME:-/volume1/docker/stirling}/customFiles:/customFiles:rw
  - ${DOCKER_VOLUME:-/volume1/docker/stirling}/logs:/logs:rw
  
  # Your existing pipeline directories (already created)
  - ${DOCKER_VOLUME:-/volume1/docker/stirling}/pipeline/watchedFolders:/pipeline/watchedFolders:rw
  - ${DOCKER_VOLUME:-/volume1/docker/stirling}/pipeline/finishedFolders:/pipeline/finishedFolders:rw
  
  # Add error handling directory
  - ${DOCKER_VOLUME:-/volume1/docker/stirling}/pipeline/errorFolders:/pipeline/errorFolders:rw
  
  # Temporary processing space for performance
  - ${DOCKER_VOLUME:-/volume1/docker/stirling}/temp:/tmp/stirling:rw

# MAXIMUM PERFORMANCE ENVIRONMENT VARIABLES
environment:
  # Security (keep your existing settings)
  - DOCKER_ENABLE_SECURITY=true
  - SECURITY_ENABLELOGIN=true
  - SECURITY_USERNAME=${SECURITY_USERNAME:-xxx}
  - SECURITY_PASSWORD=${SECURITY_PASSWORD:-password}
  
  # Language and features
  - LANGS=en_GB
  - INSTALL_BOOK_AND_ADVANCED_HTML_OPS=true              # Enable all features
  
  # JAVA PERFORMANCE TUNING - MAXIMUM MEMORY
  - JAVA_TOOL_OPTIONS=-Xms8g -Xmx28g -XX:MaxMetaspaceSize=2g -XX:+UseG1GC -XX:+UseStringDeduplication -XX:MaxGCPauseMillis=200
  
  # System optimization
  - PUID=1027
  - PGID=100
  - UMASK=022
  - TZ=${TZ:-UTC}
  
  # Performance tuning
  - STIRLING_PDF_MAX_FILE_SIZE=2000MB                    # Allow large files
  - STIRLING_PDF_MAX_REQUEST_SIZE=2000MB
  - STIRLING_PDF_THREADS=12                              # Use all CPU cores
  - TMPDIR=/tmp/stirling                                 # Use dedicated temp space
  
  # Pipeline performance
  - PIPELINE_WATCH_INTERVAL=1                            # Check every 1 second
  - PIPELINE_MAX_CONCURRENT=6                            # Process 6 batches simultaneously
  - PIPELINE_TIMEOUT=3600                                # 1 hour timeout per batch

# Shared memory for large operations
shm_size: 8gb

# Restart policy
restart: unless-stopped

# User configuration
user: "1027:100"

networks:
default:
driver: bridge

here is the preprocessor code :

#!/usr/bin/env python3
"""
Stirling PDF Pre-processor
Validates PDFs and prepares them for Stirling pipeline processing
"""

import os
import sys
import logging
import json
import shutil
from pathlib import Path
from typing import List, Tuple, Dict
import subprocess
from datetime import datetime

try:
import fitz # PyMuPDF
PYMUPDF_AVAILABLE = True
except ImportError:
PYMUPDF_AVAILABLE = False

try:
import pikepdf
PIKEPDF_AVAILABLE = True
except ImportError:
PIKEPDF_AVAILABLE = False

class StirlingPreProcessor:
def init(self, source_dir: str, stirling_watch_dir: str, max_size_mb: int = 180):
self.source_dir = Path(source_dir)
self.stirling_watch_dir = Path(stirling_watch_dir)
self.max_size_bytes = max_size_mb * 1024 * 1024

    # Ensure directories exist
    self.stirling_watch_dir.mkdir(parents=True, exist_ok=True)
    
    # Setup logging
    self.setup_logging()
    
    self.stats = {
        'total_files': 0,
        'valid_files': 0,
        'corrupt_files': 0,
        'batches_created': 0,
        'errors': []
    }

def setup_logging(self):
    """Setup logging"""
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s - %(levelname)s - %(message)s',
        handlers=[logging.StreamHandler(sys.stdout)]
    )
    self.logger = logging.getLogger(__name__)

def validate_pdf_quick(self, pdf_path: Path) -> Tuple[bool, str]:
    """Quick PDF validation using available methods"""
    if not pdf_path.exists() or pdf_path.stat().st_size == 0:
        return False, "File missing or empty"
    
    # Try PyMuPDF first (fastest)
    if PYMUPDF_AVAILABLE:
        try:
            doc = fitz.open(str(pdf_path))
            page_count = len(doc)
            if page_count > 0:
                # Quick page access test
                first_page = doc[0]
            doc.close()
            return True, f"Valid ({page_count} pages)"
        except Exception as e:
            return False, f"PyMuPDF error: {str(e)[:50]}"
    
    # Fallback to QPDF
    try:
        result = subprocess.run([
            'qpdf', '--check', str(pdf_path)
        ], capture_output=True, text=True, timeout=10)
        
        if result.returncode == 0:
            return True, "Valid (QPDF check)"
        else:
            return False, "QPDF validation failed"
    except:
        return False, "No validation method available"

def process_directory(self):
    """
    Process all PDFs in source directory:
    1. Validate each PDF
    2. Rename corrupt files with 'corrupt-' prefix
    3. Create size-based batches for valid files
    4. Copy batches to Stirling watch directory
    """
    self.logger.info(f"Processing directory: {self.source_dir}")
    
    # Find all PDF files
    pdf_files = list(self.source_dir.rglob("*.pdf"))  # Recursive search
    self.stats['total_files'] = len(pdf_files)
    
    self.logger.info(f"Found {len(pdf_files)} PDF files")
    
    valid_pdfs = []
    
    # Step 1: Validate and rename corrupt files
    for pdf_path in pdf_files:
        # Skip already marked corrupt files
        if pdf_path.name.startswith('corrupt-'):
            continue
            
        self.logger.info(f"Validating: {pdf_path.name}")
        
        is_valid, message = self.validate_pdf_quick(pdf_path)
        
        if is_valid:
            file_size = pdf_path.stat().st_size
            valid_pdfs.append((pdf_path, file_size))
            self.stats['valid_files'] += 1
            self.logger.info(f"✓ {pdf_path.name} ({file_size/1024/1024:.1f}MB)")
        else:
            # Rename with corrupt- prefix
            try:
                corrupt_name = f"corrupt-{pdf_path.name}"
                corrupt_path = pdf_path.parent / corrupt_name
                pdf_path.rename(corrupt_path)
                self.stats['corrupt_files'] += 1
                self.logger.warning(f"✗ Renamed to: {corrupt_name} - {message}")
            except Exception as e:
                self.logger.error(f"Failed to rename {pdf_path.name}: {e}")
                self.stats['errors'].append(f"Rename failed: {pdf_path.name}")
    
    if not valid_pdfs:
        self.logger.error("No valid PDFs found!")
        return
    
    # Step 2: Create size-based batches
    self.logger.info("Creating size-optimized batches...")
    batches = self.create_batches(valid_pdfs)
    
    # Step 3: Copy batches to Stirling watch directory
    self.copy_batches_to_stirling(batches)
    
    self.print_summary()

def create_batches(self, valid_pdfs: List[Tuple[Path, int]]) -> List[List[Path]]:
    """Create size-optimized batches"""
    # Sort by size (largest first)
    sorted_pdfs = sorted(valid_pdfs, key=lambda x: x[1], reverse=True)
    
    batches = []
    
    for pdf_path, file_size in sorted_pdfs:
        # If file is too large, create separate batch
        if file_size > self.max_size_bytes:
            batches.append([pdf_path])
            continue
        
        # Find best-fit batch
        best_batch_idx = None
        best_remaining = self.max_size_bytes + 1
        
        for i, batch in enumerate(batches):
            batch_size = sum(p.stat().st_size for p in batch)
            remaining = self.max_size_bytes - batch_size
            
            if remaining >= file_size and remaining < best_remaining:
                best_batch_idx = i
                best_remaining = remaining
        
        if best_batch_idx is not None:
            batches[best_batch_idx].append(pdf_path)
        else:
            batches.append([pdf_path])
    
    # Log batch info
    for i, batch in enumerate(batches):
        batch_size = sum(p.stat().st_size for p in batch)
        self.logger.info(f"Batch {i+1}: {len(batch)} files, {batch_size/1024/1024:.1f}MB")
    
    return batches

def copy_batches_to_stirling(self, batches: List[List[Path]]):
    """Copy batches to your existing Stirling watch directory"""
    self.logger.info(f"Copying {len(batches)} batches to Stirling watch directory...")
    
    for i, batch in enumerate(batches):
        batch_num = i + 1
        batch_dir = self.stirling_watch_dir / f"batch_{batch_num:03d}"
        batch_dir.mkdir(exist_ok=True)
        
        self.logger.info(f"Creating batch {batch_num}/{len(batches)} with {len(batch)} files")
        
        # Copy files to batch directory
        for pdf_path in batch:
            try:
                dest_path = batch_dir / pdf_path.name
                shutil.copy2(pdf_path, dest_path)
                self.logger.debug(f"Copied: {pdf_path.name}")
            except Exception as e:
                self.logger.error(f"Failed to copy {pdf_path.name}: {e}")
                self.stats['errors'].append(f"Copy failed: {pdf_path.name}")
        
        self.stats['batches_created'] += 1
        
        # Create batch info file compatible with your setup
        metadata = {
            'batch_number': batch_num,
            'file_count': len(batch),
            'total_size_mb': sum(p.stat().st_size for p in batch) / 1024 / 1024,
            'created': datetime.now().isoformat(),
            'target_output': f"chunk_{batch_num:03d}.pdf",
            'files': [p.name for p in batch]
        }
        
        metadata_path = batch_dir / "batch_info.json"
        with open(metadata_path, 'w') as f:
            json.dump(metadata, f, indent=2)

def print_summary(self):
    """Print processing summary"""
    print("\n" + "="*60)
    print("STIRLING PRE-PROCESSING SUMMARY")
    print("="*60)
    print(f"Total files scanned: {self.stats['total_files']}")
    print(f"Valid files: {self.stats['valid_files']}")
    print(f"Corrupt files (renamed): {self.stats['corrupt_files']}")
    print(f"Batches created: {self.stats['batches_created']}")
    if self.stats['errors']:
        print(f"Errors: {len(self.stats['errors'])}")
    print(f"Batches ready in: {self.stirling_watch_dir}")
    print("\nNext step: Stirling PDF will automatically process these batches")

def main():
import argparse

parser = argparse.ArgumentParser(description="Stirling PDF Pre-processor")
parser.add_argument("source_dir", help="Source directory with PDF files")
parser.add_argument("stirling_watch_dir", help="Stirling PDF watch directory")
parser.add_argument("--max-size", type=int, default=180, 
                   help="Maximum batch size in MB (default: 180)")

args = parser.parse_args()

processor = StirlingPreProcessor(args.source_dir, args.stirling_watch_dir, args.max_size)
processor.process_directory()

if name == "main":
main()