Bugfixes, building test cases

lackey/Ocr.py

@@ -1,5 +1,4 @@
 import pytesseract
-from pprint import pprint
 import csv
 import re
 
@@ -16,7 +15,50 @@ def image_to_text(self, image):
         Returns the text found in the given image.
         """
         return pytesseract.image_to_string(image)
-
+    def find_word(self, image, text, confidence=0.6):
+        """
+        Finds the first word in `image` that matches `text`.
+        Currently ignores confidence
+        """
+        data = pytesseract.image_to_data(image)
+        reader = csv.DictReader(data.split("\n"), delimiter="\t", quoting=csv.QUOTE_NONE)
+        for rect in reader:
+            if re.search(text, rect["text"]):
+                return (
+                    (
+                        rect["left"],
+                        rect["top"],
+                        rect["width"],
+                        rect["height"]
+                    ),
+                    rect["conf"]
+                )
+        return None
+    def find_line(self, image, text, confidence=0.6):
+        """
+        Finds all lines in `image` that match `text`.
+        Currently ignores confidence
+        """
+        data = pytesseract.image_to_data(image)
+        reader = csv.DictReader(data.split("\n"), delimiter="\t", quoting=csv.QUOTE_NONE)
+        lines = {}
+        for rect in reader:
+            key = (int(rect["page_num"]), int(rect["block_num"]), int(rect["par_num"]), int(rect["line_num"]))
+            if key not in lines:
+                lines[key] = ""
+            lines[key] += " " + rect["text"]
+        for line in lines:
+            if re.search(text, line):
+                return (
+                    (
+                        rect["left"],
+                        rect["top"],
+                        rect["width"],
+                        rect["height"]
+                    ),
+                    rect["conf"]
+                )
+        return None
     def find_all_in_image(self, image, text, confidence=0.6):
         """
         Finds all blocks of text in `image` that match `text`.
@@ -35,7 +77,7 @@ def find_all_in_image(self, image, text, confidence=0.6):
                 # This rect is on the same line
                 line.append(rect)
             else:
-                Debug.info("Line: " + " ".join(e["text"] for e in line if e["text"] is not None)) # int(e["conf"]) > confidence and 
+                # Debug.info("Line: " + " ".join(e["text"] for e in line if e["text"] is not None)) # int(e["conf"]) > confidence and 
                 line = [rect]
 
             if self._check_if_line_matches(line, text, confidence):
@@ -52,7 +94,7 @@ def find_in_image(self, image, text, confidence=0.6):
         Currently ignores confidence
         """
         matches = self.find_all_in_image(image, text, confidence)
-        Debug.info("Matches: " + repr(matches))
+        # Debug.info("Matches: " + repr(matches))
         if matches:
             return matches[0]
         return None
@@ -66,7 +108,7 @@ def _reduce_line_matches(self, line, text, confidence):
             last_element = line.pop(0)
         # If not, replace the element, and calculate the bounding box of the remaining elements
         line = [last_element] + line
-        print("Matched line: " + repr(line)) # DEBUG
+        #print("Matched line: " + repr(line)) # DEBUG
         corners = []
         for e in line:
             corners.append((int(e["left"]), int(e["top"])))
@@ -79,8 +121,4 @@ def _reduce_line_matches(self, line, text, confidence):
         )
         return (bbox, confidence)
 
-TextOCR = OCR()
-
-if __name__ == "__main__":
-    ocr = TextOCR.start()
-    pprint(ocr.find_all_in_image("/Users/jwinsley/Downloads/tesseract/homemeds.png", "Ibuprofen"))
+TextOCR = OCR()

lackey/PlatformManagerDarwin.py

@@ -273,6 +273,16 @@ def getScreenDetails(self):
                 )
             }
             screens.append(screen)
+        # Convert y-coordinates
+        y1 = min([s["rect"][1] for s in screens])
+        y2 = max([s["rect"][1]+s["rect"][3] for s in screens])
+        for screen in screens:
+            screen["rect"] = (
+                screen["rect"][0],
+                (y2 - screen["rect"][3]) - screen["rect"][1],
+                screen["rect"][2],
+                screen["rect"][3]
+            )
         return screens
     def isPointVisible(self, x, y):
         """ Checks if a point is visible on any monitor. """
@@ -313,6 +323,7 @@ def getWindowByPID(self, pid, order=0):
         """ Returns a handle for the first window that matches the provided PID """
         for w in self._get_window_list():
             if "kCGWindowOwnerPID" in w and w["kCGWindowOwnerPID"] == pid:
+                print(self.getWindowRect(w["kCGWindowNumber"]))
                 # Matches - make sure we get it in the correct order
                 if order == 0:
                     return w["kCGWindowNumber"]

lackey/RegionMatching.py

@@ -70,6 +70,8 @@ def __init__(self, target=None):
             self.setImage(target)
         elif target is not None:
             raise TypeError("Unrecognized argument for Pattern()")
+    def __repr__(self):
+        return f"<Pattern [{'image' if self.imagePattern else 'ocr'}] \"{self.path}\" ({self.similarity}) >"
 
     def similar(self, similarity):
         """ Returns a new Pattern with the specified similarity threshold """
@@ -105,14 +107,13 @@ def setFilename(self, filename):
             if os.path.exists(full_path):
                 found = True
                 self.path = full_path
-                self.image = cv2.imread(self.path)
+                self.setImage(cv2.imread(self.path))
                 break
         ## Check if path is valid
         if not found:
             self.path = filename
             Debug.info("Pattern not found in image paths: " + repr(Settings.ImagePaths))
             if Settings.SwitchToText:
-                self.imagePattern = False
                 Debug.info("Assuming pattern is OCR text")
             else:
                 raise ImageMissing(ImageMissingEvent(pattern=self, event_type="IMAGEMISSING"))
@@ -650,6 +651,7 @@ def waitVanish(self, pattern, seconds=None):
         if match:
             return False
             #self._findFailedHandler(FindFailed("Pattern '{}' did not vanish".format(pattern.path)))
+        return True
     def has(self, pattern):
         """Checks whether the given `pattern` is visible in the region. Does not throw FindFailed.
 
@@ -685,6 +687,7 @@ def exists(self, pattern, seconds=None):
             timeout = time.time() + seconds
 
             # Consult TextOCR to find needle text
+            match = None
             while not match:
                 match = TextOCR.find_in_image(r.getBitmap(), pattern.path, pattern.similarity)
                 time.sleep(1/self._defaultScanRate if self._defaultScanRate is not None else 1/Settings.WaitScanRate)
@@ -705,6 +708,7 @@ def exists(self, pattern, seconds=None):
                     break
 
         if match is None:
+            print(pattern)
             Debug.info("Couldn't find '{}' with enough similarity.".format(pattern.path))
             return None
 
@@ -1415,7 +1419,7 @@ def compare(self, image):
 
     # OCR Functions
     def findText(self, text):
-        """ OCR function """
+        """ Finds the first block of text in the region that matches `text`. Can be a regex. """
         findFailedRetry = True
         while findFailedRetry:
             match = self.existsText(text)
@@ -1426,11 +1430,27 @@ def findText(self, text):
                 time.sleep(self._repeatWaitTime)
         return match
     def findWord(self, text):
-        """ OCR function """
-        raise NotImplementedError()
+        """ Finds the first word in the region that matches `text`. Can be a regex. """
+        search = TextOCR.find_word(self.getBitmap(), text)
+        if search:
+            bbox, conf = search
+            return Match(
+                conf,
+                Location(0,0),
+                ((bbox[0], bbox[1]), (bbox[2], bbox[3]))
+            )
+        return None
     def findLine(self, text):
-        """ OCR function """
-        raise NotImplementedError()
+        """ Finds the first line in the region that matches `text`. Can be a regex. """
+        search = TextOCR.find_line(self.getBitmap(), text)
+        if search:
+            bbox, conf = search
+            return Match(
+                conf,
+                Location(0,0),
+                ((bbox[0], bbox[1]), (bbox[2], bbox[3]))
+            )
+        return None
     def waitText(self, text, seconds=None):
         """ Searches for an image pattern in the given region, given a specified timeout period
 
@@ -1532,18 +1552,20 @@ def waitVanishText(self, text, seconds=None):
             return None
         if seconds is None:
             seconds = self.autoWaitTimeout
-        if not isinstance(pattern, basestring):
+        if not isinstance(text, basestring):
             raise TypeError("waitVanishText expected a string")
 
         timeout = time.time() + seconds
 
         # Consult TextOCR to find needle text
+        match = TextOCR.find_in_image(r.getBitmap(), text)
         while match and time.time() < timeout:
             match = TextOCR.find_in_image(r.getBitmap(), text)
             time.sleep(1/self._defaultScanRate if self._defaultScanRate is not None else 1/Settings.WaitScanRate)
 
         if match:
             return False
+        return True
 
     def findAllText(self, text):
         """ Searches for all matching text regions in the given region
@@ -1558,7 +1580,7 @@ def findAllText(self, text):
             return None
         seconds = self.autoWaitTimeout
 
-        if not isinstance(pattern, basestring):
+        if not isinstance(text, basestring):
             raise TypeError("findAllText expected a string")
 
         # Consult TextOCR to find needle text
@@ -1575,13 +1597,15 @@ def findAllText(self, text):
             lastMatches.append(
                 Match(
                     confidence,
-                    pattern.offset,
-                    position)
+                    Location(0,0),
+                    ((position[0], position[1]), (position[2], position[3]))
+                )
             )
         self._lastMatches = iter(lastMatches)
-        Debug.info("Found match(es) for pattern '{}'".format(text))
+        Debug.info("Found match(es) for text '{}'".format(text))
         self._lastMatchTime = (time.time() - find_time) * 1000 # Capture find time in milliseconds
         return self._lastMatches
+
     # Event Handlers
 
     def onAppear(self, pattern, handler=None):

tests/test_cases.py

@@ -104,10 +104,10 @@ def testTypeCopyPaste(self):
             lackey.type("n", lackey.KeyModifier.CMD)
             time.sleep(1)
             app = lackey.App("Untitled")
-            r.type("This is a Test")
+            r.type("This is a test")
             r.type("a", lackey.KeyModifier.CMD) # Select all
             r.type("c", lackey.KeyModifier.CMD) # Copy
-            self.assertEqual(r.getClipboard(), "This is a Test")
+            self.assertEqual(r.getClipboard(), "This is a test")
             r.type("{DELETE}") # Clear the selected text
             r.paste("This, on the other hand, is a {SHIFT}broken {SHIFT}record.") # Paste should ignore special characters and insert the string as is
             r.type("a", lackey.KeyModifier.CMD) # Select all
@@ -149,6 +149,13 @@ def test_observer(appear_event):
         r.observe(30)
         self.assertTrue(r.TestFlag)
         self.assertGreater(r.getTime(), 0)
+        # OCR
+        a = r.find("textedit_header.png").below(400)
+        #a.highlight()
+        self.assertIsNotNone(a.findText("This is a test"))
+        self.assertGreater(len([a.findAllText("This is a test")]), 0)
+
+        # 
         if sys.platform.startswith("win"):
             r.rightClick(r.getLastMatch())
             r.click("select_all.png")
@@ -158,11 +165,13 @@ def test_observer(appear_event):
             r.type("c", lackey.KeyModifier.CMD)
         self.assertEqual(r.getClipboard(), "This is a test")
         r.type("{DELETE}")
+        self.assertTrue(r.waitVanishText("This is a test"))
         if sys.platform.startswith("win"):
             r.type("{F4}", lackey.Key.ALT)
         elif sys.platform == "darwin":
             r.type("w", lackey.KeyModifier.CMD)
-            r.click(lackey.Pattern("textedit_save_2.png").targetOffset(-86, 25))
+            r.wait(lackey.Pattern("textedit_save_2.png"))
+            r.click(lackey.Pattern("textedit_save_2.png").targetOffset(-126, 20))
             lackey.sleep(0.5)
             r.type("q", lackey.KeyModifier.CMD)
 

tests/test_ocr.py

@@ -1,4 +1,7 @@
 import lackey
 r = lackey.Screen(1)
+r.highlight()
 print(r.text())
-r.findText("IBUPROFEN SUSPENSION").click()
+r.findText("IBUPROFEN SUSPENSION").highlight()
+print(r.existsText("CHEESE"))
+var = [print(m) for m in r.findAllText("IBUPROFEN")]