Properly handle inotify's IN_Q_OVERFLOW event (fsnotify#149)

* Properly handle inotify's IN_Q_OVERFLOW event

Upon receiving an event with IN_Q_OVERFLOW set in the mask, generate an
error on the Errors chan, so that the application can take appropriate
action.

* Use a well-defined error (ErrEventOverflow) for inotify overflow

* Add a test for inotify queue overflow

Author: zeldovich

fsnotify.go

@@ -9,6 +9,7 @@ package fsnotify
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 )
 
@@ -60,3 +61,6 @@ func (op Op) String() string {
 func (e Event) String() string {
 	return fmt.Sprintf("%q: %s", e.Name, e.Op.String())
 }
+
+// Common errors that can be reported by a watcher
+var ErrEventOverflow = errors.New("fsnotify queue overflow")

inotify.go

@@ -245,6 +245,15 @@ func (w *Watcher) readEvents() {
 
 			mask := uint32(raw.Mask)
 			nameLen := uint32(raw.Len)
+
+			if mask&unix.IN_Q_OVERFLOW != 0 {
+				select {
+				case w.Errors <- ErrEventOverflow:
+				case <-w.done:
+					return
+				}
+			}
+
 			// If the event happened to the watched directory or the watched file, the kernel
 			// doesn't append the filename to the event, but we would like to always fill the
 			// the "Name" field with a valid filename. We retrieve the path of the watch from

inotify_test.go

@@ -358,3 +358,94 @@ func TestInotifyInnerMapLength(t *testing.T) {
 		t.Fatalf("Expected paths len is 0, but got: %d, %v", len(w.paths), w.paths)
 	}
 }
+
+func TestInotifyOverflow(t *testing.T) {
+	// We need to generate many more events than the
+	// fs.inotify.max_queued_events sysctl setting.
+	// We use multiple goroutines (one per directory)
+	// to speed up file creation.
+	numDirs := 128
+	numFiles := 1024
+
+	testDir := tempMkdir(t)
+	defer os.RemoveAll(testDir)
+
+	w, err := NewWatcher()
+	if err != nil {
+		t.Fatalf("Failed to create watcher: %v", err)
+	}
+	defer w.Close()
+
+	for dn := 0; dn < numDirs; dn++ {
+		testSubdir := fmt.Sprintf("%s/%d", testDir, dn)
+
+		err := os.Mkdir(testSubdir, 0777)
+		if err != nil {
+			t.Fatalf("Cannot create subdir: %v", err)
+		}
+
+		err = w.Add(testSubdir)
+		if err != nil {
+			t.Fatalf("Failed to add subdir: %v", err)
+		}
+	}
+
+	errChan := make(chan error, numDirs*numFiles)
+
+	for dn := 0; dn < numDirs; dn++ {
+		testSubdir := fmt.Sprintf("%s/%d", testDir, dn)
+
+		go func() {
+			for fn := 0; fn < numFiles; fn++ {
+				testFile := fmt.Sprintf("%s/%d", testSubdir, fn)
+
+				handle, err := os.Create(testFile)
+				if err != nil {
+					errChan <- fmt.Errorf("Create failed: %v", err)
+					continue
+				}
+
+				err = handle.Close()
+				if err != nil {
+					errChan <- fmt.Errorf("Close failed: %v", err)
+					continue
+				}
+			}
+		}()
+	}
+
+	creates := 0
+	overflows := 0
+
+	after := time.After(10 * time.Second)
+	for overflows == 0 && creates < numDirs*numFiles {
+		select {
+		case <-after:
+			t.Fatalf("Not done")
+		case err := <-errChan:
+			t.Fatalf("Got an error from file creator goroutine: %v", err)
+		case err := <-w.Errors:
+			if err == ErrEventOverflow {
+				overflows++
+			} else {
+				t.Fatalf("Got an error from watcher: %v", err)
+			}
+		case evt := <-w.Events:
+			if !strings.HasPrefix(evt.Name, testDir) {
+				t.Fatalf("Got an event for an unknown file: %s", evt.Name)
+			}
+			if evt.Op == Create {
+				creates++
+			}
+		}
+	}
+
+	if creates == numDirs*numFiles {
+		t.Fatalf("Could not trigger overflow")
+	}
+
+	if overflows == 0 {
+		t.Fatalf("No overflow and not enough creates (expected %d, got %d)",
+			numDirs*numFiles, creates)
+	}
+}