Skip to content

Commit ebc5181

Browse files
mikedewarmikedewar
committed
Merge pull request #396 from jprobinson/master
added basic toemail block
2 parents 1ef1862 + 651338a commit ebc5181

File tree

3 files changed

+557
-0
lines changed

3 files changed

+557
-0
lines changed

‎st/library/fromEmail.go

+366
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,366 @@
1+
package library
2+
3+
import (
4+
"bytes"
5+
"crypto/tls"
6+
"net/mail"
7+
"time"
8+
9+
"code.google.com/p/go-imap/go1/imap"
10+
11+
"github.com/nytlabs/streamtools/st/blocks"
12+
"github.com/nytlabs/streamtools/st/util"
13+
)
14+
15+
// FromEmail holds channels we're going to use to communicate with streamtools,
16+
// credentials for authenticating with an IMAP server and the IMAP client.
17+
type FromEmail struct {
18+
blocks.Block
19+
queryrule chan chan interface{}
20+
inrule chan interface{}
21+
out chan interface{}
22+
quit chan interface{}
23+
24+
host string
25+
username string
26+
password string
27+
mailbox string
28+
29+
client *imap.Client
30+
idling bool
31+
}
32+
33+
// NewFromEmail is a simple factory for streamtools to make new blocks of this kind.
34+
// By default, the block is configured for GMail.
35+
func NewFromEmail() blocks.BlockInterface {
36+
return &FromEmail{host: "imap.gmail.com", mailbox: "INBOX"}
37+
}
38+
39+
// newIMAPClient will initiate a new IMAP connection with the given creds.
40+
func newIMAPClient(host, username, password, mailbox string) (*imap.Client, error) {
41+
client, err := imap.DialTLS(host, new(tls.Config))
42+
if err != nil {
43+
return client, err
44+
}
45+
46+
_, err = client.Login(username, password)
47+
if err != nil {
48+
return client, err
49+
}
50+
51+
_, err = imap.Wait(client.Select(mailbox, false))
52+
if err != nil {
53+
return client, err
54+
}
55+
56+
return client, nil
57+
}
58+
59+
// idle will initiate an IMAP idle and wait for updates. Any time the connection finds a idle update,
60+
// it will terminate the idle, fetch any unread email messages and kick idle off again. Every 20
61+
// minutes, it will reset the idle to keep it alive.
62+
func (e *FromEmail) idle() {
63+
// keep track of an ongoing idle
64+
e.idling = true
65+
defer func() { e.idling = false }()
66+
67+
var err error
68+
_, err = e.client.Idle()
69+
if err != nil {
70+
e.Error(err.Error())
71+
return
72+
}
73+
74+
// kicks off occasional data check during Idle
75+
poll := make(chan uint, 1)
76+
poll <- 0
77+
78+
// setup ticker to reset the idle every 20 minutes (RFC-2177 recommends every <=29 mins)
79+
reset := time.NewTicker(20 * time.Minute)
80+
81+
for {
82+
select {
83+
case <-poll:
84+
// attempt to fill pipe with new data
85+
err = e.client.Recv(0)
86+
if err != nil {
87+
// imap.ErrTimeout here means 'no data available'
88+
if err == imap.ErrTimeout {
89+
sleep(poll)
90+
continue
91+
} else {
92+
e.Error(err.Error())
93+
return
94+
}
95+
}
96+
97+
// check the pipe for data
98+
if len(e.client.Data) > 0 {
99+
// term idle and fetch unread
100+
_, err = e.client.IdleTerm()
101+
if err != nil {
102+
e.Error(err.Error())
103+
sleep(poll)
104+
return
105+
}
106+
e.idling = false
107+
108+
// put any new unread messages on the channel
109+
err = e.fetchUnread()
110+
if err != nil {
111+
e.Error(err.Error())
112+
sleep(poll)
113+
return
114+
}
115+
116+
// kick off that idle again
117+
_, err = e.client.Idle()
118+
if err != nil {
119+
e.Error(err.Error())
120+
sleep(poll)
121+
return
122+
}
123+
e.idling = true
124+
}
125+
// clean the pipe
126+
e.client.Data = nil
127+
// sleep a bit before checking the pipe again
128+
sleep(poll)
129+
130+
case <-reset.C:
131+
_, err = e.client.IdleTerm()
132+
if err != nil {
133+
e.Error(err.Error())
134+
return
135+
}
136+
137+
_, err = e.client.Idle()
138+
if err != nil {
139+
e.Error(err.Error())
140+
return
141+
}
142+
}
143+
}
144+
}
145+
146+
func sleep(poll chan uint) {
147+
go func() {
148+
time.Sleep(10 * time.Second)
149+
poll <- 1
150+
}()
151+
}
152+
153+
// fetchUnread emails will check the current mailbox for any unread messages. If it finds
154+
// some, it will grab the email bodies, parse them and pass them along the block's out channel.
155+
func (e *FromEmail) fetchUnread() error {
156+
cmd, err := findUnreadEmails(e.client)
157+
if err != nil {
158+
return err
159+
}
160+
161+
var emails []map[string]interface{}
162+
emails, err = getEmails(e.client, cmd)
163+
if err != nil {
164+
return err
165+
}
166+
167+
for _, email := range emails {
168+
e.out <- email
169+
}
170+
171+
return nil
172+
}
173+
174+
// getEmails will fetch the full bodies of all emails listed in the given command.
175+
func getEmails(client *imap.Client, cmd *imap.Command) ([]map[string]interface{}, error) {
176+
var emails []map[string]interface{}
177+
seq := new(imap.SeqSet)
178+
for _, rsp := range cmd.Data {
179+
for _, uid := range rsp.SearchResults() {
180+
seq.AddNum(uid)
181+
}
182+
}
183+
if seq.Empty() {
184+
return emails, nil
185+
}
186+
fCmd, err := imap.Wait(client.UIDFetch(seq, "INTERNALDATE", "BODY[]", "UID", "RFC822.HEADER"))
187+
if err != nil {
188+
return emails, err
189+
}
190+
191+
var email map[string]interface{}
192+
for _, msgData := range fCmd.Data {
193+
msgFields := msgData.MessageInfo().Attrs
194+
email, err = newEmailMessage(msgFields)
195+
if err != nil {
196+
return emails, err
197+
}
198+
emails = append(emails, email)
199+
200+
// mark message as read
201+
fSeq := new(imap.SeqSet)
202+
fSeq.AddNum(imap.AsNumber(msgFields["UID"]))
203+
_, err = imap.Wait(client.UIDStore(fSeq, "+FLAGS", "\\SEEN"))
204+
if err != nil {
205+
return emails, err
206+
}
207+
}
208+
return emails, nil
209+
}
210+
211+
// newEmailMessage will parse an imap.FieldMap into an map[string]interface{}. This
212+
// will expect the message to container the internaldate and the body with
213+
// all headers included.
214+
func newEmailMessage(msgFields imap.FieldMap) (map[string]interface{}, error) {
215+
var email map[string]interface{}
216+
// parse the header
217+
rawHeader := imap.AsBytes(msgFields["RFC822.HEADER"])
218+
msg, err := mail.ReadMessage(bytes.NewReader(rawHeader))
219+
if err != nil {
220+
return email, err
221+
}
222+
223+
email = map[string]interface{}{
224+
"internal_date": imap.AsDateTime(msgFields["INTERNALDATE"]),
225+
"body": imap.AsString(msgFields["BODY[]"]),
226+
"from": msg.Header.Get("From"),
227+
"to": msg.Header.Get("To"),
228+
"subject": msg.Header.Get("Subject"),
229+
}
230+
231+
return email, nil
232+
}
233+
234+
// findUnreadEmails will run a find the UIDs of any unread emails in the
235+
// mailbox.
236+
func findUnreadEmails(conn *imap.Client) (*imap.Command, error) {
237+
// get headers and UID for UnSeen message in src inbox...
238+
cmd, err := imap.Wait(conn.UIDSearch("UNSEEN"))
239+
if err != nil {
240+
return &imap.Command{}, err
241+
}
242+
return cmd, nil
243+
}
244+
245+
// Setup is called once before running the block. We build up the channels and specify what kind of block this is.
246+
func (e *FromEmail) Setup() {
247+
e.Kind = "FromEmail"
248+
e.out = e.Broadcast()
249+
e.inrule = e.InRoute("rule")
250+
e.queryrule = e.QueryRoute("rule")
251+
e.quit = e.Quit()
252+
}
253+
254+
// parseAuthInRules will expect a payload from the inrules channel and
255+
// attempt to pull the IMAP auth credentials out it.
256+
func (e *FromEmail) parseAuthRules(msgI interface{}) error {
257+
var err error
258+
e.host, err = util.ParseRequiredString(msgI, "Host")
259+
if err != nil {
260+
return err
261+
}
262+
263+
e.username, err = util.ParseRequiredString(msgI, "Username")
264+
if err != nil {
265+
return err
266+
}
267+
268+
e.password, err = util.ParseRequiredString(msgI, "Password")
269+
if err != nil {
270+
return err
271+
}
272+
273+
e.mailbox, err = util.ParseRequiredString(msgI, "Mailbox")
274+
if err != nil {
275+
return err
276+
}
277+
278+
return nil
279+
}
280+
281+
func (e *FromEmail) initClient() error {
282+
// initiate IMAP client with new creds
283+
var err error
284+
e.client, err = newIMAPClient(e.host, e.username, e.password, e.mailbox)
285+
if err != nil {
286+
return err
287+
}
288+
289+
return nil
290+
}
291+
292+
// Run is the block's main loop. Here we listen on the different channels we set up.
293+
func (e *FromEmail) Run() {
294+
var err error
295+
for {
296+
err = nil
297+
select {
298+
case msgI := <-e.inrule:
299+
// get id/pw/host/mailbox for IMAP
300+
err = e.parseAuthRules(msgI)
301+
if err != nil {
302+
e.Error(err.Error())
303+
continue
304+
}
305+
306+
// if we've already got a client, close it. We need to kill it and pick up new creds.
307+
if e.client != nil {
308+
// if we're idling, term it before closing
309+
if e.idling {
310+
_, err = e.client.IdleTerm()
311+
if err != nil {
312+
// dont continue. we want to init with new creds
313+
e.Error(err.Error())
314+
}
315+
}
316+
_, err = e.client.Close(true)
317+
if err != nil {
318+
// dont continue. we want to init with new creds
319+
e.Error(err.Error())
320+
}
321+
}
322+
323+
// initiate IMAP client with new creds
324+
err = e.initClient()
325+
if err != nil {
326+
e.Error(err.Error())
327+
continue
328+
}
329+
330+
// do initial initial fetch on all existing unread messages
331+
err = e.fetchUnread()
332+
if err != nil {
333+
e.Error(err.Error())
334+
continue
335+
}
336+
337+
// kick off idle in a goroutine
338+
go e.idle()
339+
340+
case <-e.quit:
341+
if e.client != nil {
342+
// attempt to term the idle if its running
343+
if e.idling {
344+
_, err = e.client.IdleTerm()
345+
if err != nil {
346+
e.Error(err.Error())
347+
}
348+
}
349+
// close the IMAP conn
350+
_, err = e.client.Close(true)
351+
if err != nil {
352+
e.Error(err.Error())
353+
}
354+
}
355+
return
356+
case respChan := <-e.queryrule:
357+
// deal with a query request
358+
respChan <- map[string]interface{}{
359+
"Host": e.host,
360+
"Username": e.username,
361+
"Password": e.password,
362+
"Mailbox": e.mailbox,
363+
}
364+
}
365+
}
366+
}

‎st/library/library.go

+2
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,10 @@ var Blocks = map[string]func() blocks.BlockInterface{
1616
"fromsqs": NewFromSQS,
1717
"frompost": NewFromPost,
1818
"fromfile": NewFromFile,
19+
"fromemail": NewFromEmail,
1920
"tonsq": NewToNSQ,
2021
"toelasticsearch": NewToElasticsearch,
22+
"toemail": NewToEmail,
2123
"tofile": NewToFile,
2224
"tolog": NewToLog,
2325
"tobeanstalkd": NewToBeanstalkd,

0 commit comments

Comments
 (0)