initial commit

.gitignore

@@ -0,0 +1,13 @@
+public
+tags
+tests
+c
+d
+t
+agostle
+agostle.*
+*.text
+testdata/
+*.swp
+equinox.priv
+vendor

LICENSE

@@ -0,0 +1,13 @@
+Copyright 2013 Tamás Gulácsi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

README.md

@@ -0,0 +1,23 @@
+# Agostle
+Agostle is a kind of _apostle_ - converts everything to PDF.
+Everything:
+
+  * Text, Spreadsheet, HTML and other office-like documents with the help of LibreOffice,
+  * Images with GraphicsMagick,
+  * Email with agostle (by traversing the tree and applying the transformations as needed).
+
+# Install
+
+    go get github.com/tgulacsi/agostle
+
+
+# Usage
+Agostle can be used for converting files, or start a HTTP server on port 8500, and respond
+to requests like `/email/convert`.
+
+# Build
+The `requirements.txt` contains the needed programs, and a Dockerfile is present for Docker users, to be able to have a converter with every needed program installed, without polluting your environment.
+
+Build agostle and the tgulacsi/agostle container:
+
+	  go run make.go

centos/install.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -e
+sudo useradd -l agostle
+yum install libreoffice GraphicsMagick ghostscript
+if ! yum install pdftk-1.44-2.el6.rf.x86_64.rpm runit-2.1.1-6.el6.x86_64.rpm; then
+	. $(dirname $0)/install_pdftk.sh
+	. $(dirname $0)/install_runit.sh
+fi
+cp -pr $(dirname $0)/etc/service/agostle /etc/service/

centos/install_apostle.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+yum install python python-virtualenv
+virtualenv --system-site-packages venv
+. ./venv/bin/activate
+pip install -r apostle/requirements.txt

centos/install_libreoffice.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+wget http://ftp.bme.hu/pub/mirrors/tdf/libreoffice/stable/4.1.1/rpm/x86_64/LibreOffice_4.1.1_Linux_x86-64_rpm.tar.gz
+tar xaf LibreOffice_4.1.1_Linux_x86-64_rpm.tar.gz
+cd LibreOffice_1.4*/
+yum localinstall RMPS/*
+cd /usr/bin
+ln -s libreoffice4* loffice

centos/install_pdftk.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+yum install rpm-build
+wget http://www.pdflabs.com/tools/pdftk-the-pdf-toolkit/pdftk-2.02-src.zip
+unzip pdftk-2.02-src
+cd pdftk-2.02-dist
+

centos/install_runit.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -e
+yum install rpm-build rpmdevtools
+wget -O runit-rpm.zip https://github.com/imeyer/runit-rpm/archive/master.zip
+unzip runit-rpm.zip
+cd runit-rpm-master
+sudo yum install glibc-static
+./build.sh
+sudo yum install $HOME/rpmbuild/RPMS/x86_64/runit-*.rpm

centos/update_agostle.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+set -e
+go get -u github.com/tgulacsi/agostle
+ver=$(cd ~/src/github.com/tgulacsi/agostle/ \
+	&& git log --oneline | head -n1 | cut -d' ' -f1)
+[ -n "$ver" ]
+ver=$(date '+%Y%m%d')-$ver
+echo ver=$ver
+echo "Copying to 192.168.3.110"
+if scp -p bin/agostle 192.168.3.110:prd/agostle-$ver; then
+#ssh 192.168.3.110 sh -c 'cd /home/agostle && sudo -u agostle ln -sf agostle-${ver} agostle && killall agostle'
+#ssh -t 192.168.3.110 sh -c "cd /home/tgulacsi/prd && ln -sf agostle-${ver} $HOME/prd/agostle && sudo -u agostle killall agostle && sudo -u agostle find /var/tmp/agostle/ -delete"
+	ssh -t 192.168.3.110 sh -c "cd /home/tgulacsi/prd && ln -sf agostle-${ver} $HOME/prd/agostle && sudo -u agostle killall agostle"
+fi
+
+GOOS=windows GOARCH=386 go get -u github.com/tgulacsi/agostle
+
+echo "copying agostle.exe to 192.168.1.2:html/"
+set +e
+echo rsync -av bin/agostle.exe 192.168.1.2:html/
+rsync -av bin/windows_386/agostle.exe 192.168.1.2:html/
+

converter/conclimit.go

@@ -0,0 +1,46 @@
+// Copyright 2013 The Agostle Authors. All rights reserved.
+// Use of this source code is governed by an Apache 2.0
+// license that can be found in the LICENSE file.
+
+package converter
+
+// Concurrency is the default concurrent goroutines number
+var Concurrency = int(8)
+
+// RateLimiter is the interface for rate limiting
+type RateLimiter interface {
+	//Acquire acquires a token (blocks if none accessible)
+	Acquire() Token
+	//Release releases the token
+	Release(Token)
+}
+
+// Token is a token
+type Token struct{}
+
+// NewRateLimiter returns a RateLimiter
+func NewRateLimiter(n int) RateLimiter {
+	rl := &rateLimiter{tokens: make(chan Token, n)}
+	var t Token
+	for i := 0; i < n; i++ {
+		rl.tokens <- t
+	}
+	return rl
+}
+
+type rateLimiter struct {
+	tokens chan Token
+}
+
+// Acquire pulls a token
+func (rl *rateLimiter) Acquire() Token {
+	return <-rl.tokens
+}
+
+// Release pushes back the token
+func (rl *rateLimiter) Release(t Token) {
+	select {
+	case rl.tokens <- t:
+	default:
+	}
+}

converter/config.go

@@ -0,0 +1,168 @@
+// Copyright 2013 The Agostle Authors. All rights reserved.
+// Use of this source code is governed by an Apache 2.0
+// license that can be found in the LICENSE file.
+
+// Package converter implements function for converting files to PDF
+package converter
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+
+	"golang.org/x/net/context"
+
+	"github.com/go-kit/kit/log"
+	"github.com/stvp/go-toml-config"
+	"github.com/tgulacsi/go/osgroup"
+)
+
+var Logger *log.Context
+
+func lookPath(fn string) string {
+	path, err := exec.LookPath(fn)
+	if err != nil {
+		return ""
+	}
+	return path
+}
+
+var (
+	// ConfPdftk is the path for PdfTk
+	ConfPdftk = config.String("pdftk", lookPath("pdftk"))
+
+	// ConfPdfseparate is the path for pdfseparate (member of poppler-utils
+	ConfPdfseparate = config.String("pdfseparate", "pdfseparate")
+
+	// ConfLoffice is the path for LibreOffice
+	ConfLoffice = config.String("loffice", lookPath("loffice"))
+
+	// ConfGm is the path for GraphicsMagick
+	ConfGm = config.String("gm", lookPath("gm"))
+
+	// ConfGs is the path for GhostScript
+	ConfGs = config.String("gs", lookPath("gs"))
+
+	// ConfPdfClean is the path for pdfclean
+	ConfPdfClean = config.String("pdfclean", lookPath("pdfclean"))
+
+	// ConfMutool is the path for mutool
+	ConfMutool = config.String("mutool", lookPath("mutool"))
+
+	// ConvWkhtmltopdf is the parth for wkhtmltopdf
+	ConfWkhtmltopdf = config.String("wkhtmltopdf", lookPath("wkhtmltopdf"))
+
+	// ConfSortBeforeMerge should be true if generally we should sort files by filename before merge
+	ConfSortBeforeMerge = config.Bool("sortBeforeMerge", false)
+
+	// ConfChildTimeout is the time before the child gets killed
+	ConfChildTimeout = config.Duration("childTimeout", 1*time.Hour)
+
+	// ConcLimit limits the concurrently running child processes
+	ConcLimit = NewRateLimiter(Concurrency)
+
+	// ConfWorkdir is the working directory (will be os.TempDir() if empty)
+	ConfWorkdir = config.String("workdir", "")
+
+	// ConfListenAddr is a listen address for HTTP requests
+	ConfListenAddr = config.String("listen", ":9500")
+
+	// ConfDefaultIsService decides whether start as service without args
+	ConfDefaultIsService = config.Bool("defaultIsService", false)
+
+	// ConfUseLofficePortLock defines whether to limit Loffice usage by a port lock
+	ConfLofficeUsePortLock = config.Bool("lofficeUsePortLock", !osgroup.IsInsideDocker())
+)
+
+// LoadConfig loads TOML config file
+func LoadConfig(fn string) error {
+	if err := config.Parse(fn); err != nil {
+		Log("msg", "WARN Cannot open config file", "file", fn, "error", err)
+	}
+	if *ConfLoffice != "" {
+		if _, err := exec.LookPath(*ConfLoffice); err != nil {
+			Log("msg", "WARN cannot use as loffice!", "loffice", *ConfLoffice)
+			if fn, err := exec.LookPath("soffice"); err == nil {
+				Log("msg", "Will use as loffice instead.", "soffice", fn)
+				*ConfLoffice = fn
+			}
+		}
+	}
+	if *ConfWorkdir != "" {
+		_ = os.Setenv("TMPDIR", *ConfWorkdir)
+		Workdir = *ConfWorkdir
+	}
+
+	bn := filepath.Base(*ConfPdfseparate)
+	prefix := (*ConfPdfseparate)[:len(*ConfPdfseparate)-len(bn)]
+	for k := range popplerOk {
+		if err := exec.Command(prefix+k, "-h").Run(); err == nil {
+			popplerOk[k] = prefix + k
+		}
+	}
+	Log("popplerOk", popplerOk)
+
+	if !*ConfLofficeUsePortLock {
+		lofficeMu.Lock()
+		lofficePortLock = nil
+		lofficeMu.Unlock()
+	}
+
+	return nil
+}
+
+// Workdir is the main working directory
+var Workdir = os.TempDir()
+
+// LeaveTempFiles should be true only for debugging purposes (leaves temp files)
+var LeaveTempFiles = false
+
+func prepareContext(ctx context.Context, subdir string) (context.Context, string) {
+	const wdKey = "workdir"
+	odir, _ := ctx.Value(wdKey).(string)
+	if odir != "" {
+		if subdir != "" {
+			ctx = context.WithValue(ctx, wdKey, filepath.Join(Workdir, subdir))
+		}
+	} else {
+		if subdir != "" {
+			ctx = context.WithValue(ctx, wdKey, Workdir)
+		} else {
+			ctx = context.WithValue(ctx, wdKey, filepath.Join(Workdir, subdir))
+		}
+	}
+	ndir, ok := ctx.Value(wdKey).(string)
+	if ok && odir != ndir {
+		if err := os.MkdirAll(ndir, 0750); err != nil {
+			panic("cannot create workdir " + ndir + ": " + err.Error())
+		}
+	}
+	return ctx, ndir
+}
+
+// port for LibreOffice locking (only one instance should be running)
+const LofficeLockPort = 27999
+
+// save original html (do not delete it)
+var SaveOriginalHTML = false
+
+// name of errors list in resulting archive
+const ErrTextFn = "ZZZ-errors.txt"
+
+func getLogger(ctx context.Context) *log.Context {
+	if ctx == nil {
+		return Logger
+	}
+	if logger, ok := ctx.Value("logger").(*log.Context); ok {
+		return logger
+	}
+	return Logger
+}
+
+func Log(keyvals ...interface{}) {
+	if Logger == nil {
+		return
+	}
+	Logger.Log(keyvals...)
+}