img - more testing, refactors, fixes

src/tapper/helper/_util/image_util.py

@@ -14,9 +14,12 @@
 from mss.base import MSSBase
 from numpy import ndarray
 from tapper.helper._util import image_fuzz
+from tapper.helper.model_types import BboxT
 from tapper.helper.model_types import ImagePathT
 from tapper.helper.model_types import ImagePixelMatrixT
 from tapper.helper.model_types import ImageT
+from tapper.helper.model_types import PixelColorT
+from tapper.helper.model_types import XyCoordsT
 from tapper.model import constants
 
 _bbox_pattern = re.compile(r"\(BBOX_-?\d+_-?\d+_-?\d+_-?\d+\)")
@@ -52,13 +55,11 @@ def to_pixel_matrix(image: ImageT | None) -> ImagePixelMatrixT | None:
 def normalize(
     data_in: Union[
         None,
-        str,
-        tuple[str, tuple[int, int, int, int]],
-        tuple[ndarray, tuple[int, int, int, int]],
-        tuple[ndarray, None],
-        ndarray,
+        ImageT,
+        tuple[ImageT, BboxT],
+        tuple[ImageT, None],
     ]
-) -> tuple[ndarray | None, tuple[int, int, int, int] | None]:
+) -> tuple[ImagePixelMatrixT | None, BboxT | None]:
     if data_in is None:
         return None, None
     bbox = None
@@ -71,12 +72,12 @@ def normalize(
             sx = str_bbox.group().split("_")
             bbox = int(sx[1]), int(sx[2]), int(sx[3]), int(sx[4].rstrip(")"))
         return from_path(data_in), bbox
-    raise TypeError(f"Unexpected type, {type(data_in)} of {data_in}")
+    raise TypeError(f"Unexpected type {type(data_in)} of {data_in!r}")
 
 
 def get_screenshot_if_none_and_cut(
-    maybe_image: ndarray | None, bbox: tuple[int, int, int, int] | None
-) -> ndarray:
+    maybe_image: ImagePixelMatrixT | None, bbox: BboxT | None
+) -> ImagePixelMatrixT:
     if maybe_image is not None:
         if bbox:
             return maybe_image[bbox[1] : bbox[3], bbox[0] : bbox[2]]
@@ -93,9 +94,9 @@ def get_screenshot_if_none_and_cut(
 
 
 def find_in_image_raw(
-    inner_image_bbox: tuple[ndarray, tuple[int, int, int, int] | None],
-    outer: ndarray | None = None,
-) -> tuple[float, tuple[int, int]]:
+    inner_image_bbox: tuple[ndarray, BboxT | None],
+    outer: ImagePixelMatrixT | None = None,
+) -> tuple[float, XyCoordsT]:
     image_arr, bbox = inner_image_bbox
     x_start, y_start = get_start_coords(outer, bbox)
     outer = get_screenshot_if_none_and_cut(outer, bbox)
@@ -104,9 +105,30 @@ def find_in_image_raw(
     return confidence, (x_start + coords[0], y_start + coords[1])
 
 
+def image_find(
+    target: ImageT,
+    bbox: tuple[int, int, int, int] | None,
+    outer: ImageT | None = None,
+    precision: float = 1.0,
+) -> XyCoordsT | None:
+    if target is None:
+        raise ValueError("image_find nees something to search for.")
+    target_image = to_pixel_matrix(target)
+    assert target_image is not None  # for mypy
+    outer_image = to_pixel_matrix(outer)
+    # has to be before screenshot is taken, for Windows multi-monitor case
+    x_start, y_start = get_start_coords(outer_image, bbox)
+    certain_outer = get_screenshot_if_none_and_cut(outer_image, bbox)
+    confidence, coords = image_fuzz.find(certain_outer, target_image)
+
+    if confidence < precision:
+        return None
+    return x_start + coords[0], y_start + coords[1]
+
+
 def find_in_image(
     target: ImagePixelMatrixT,
-    bbox: tuple[int, int, int, int] | None,
+    bbox: BboxT | None,
     outer: ndarray | None = None,
     precision: float = 1.0,
 ) -> tuple[int, int] | None:
@@ -121,8 +143,8 @@ def find_in_image(
 
 def get_start_coords(
     outer: ndarray | None,
-    bbox_or_coords: tuple[int, int, int, int] | tuple[int, int] | None,
-) -> tuple[int, int]:
+    bbox_or_coords: BboxT | XyCoordsT | None,
+) -> XyCoordsT:
     if bbox_or_coords is not None:
         return bbox_or_coords[0], bbox_or_coords[1]
     screenshot_required = outer is None
@@ -131,7 +153,7 @@ def get_start_coords(
     return 0, 0
 
 
-def win32_coords_start() -> tuple[int, int]:
+def win32_coords_start() -> XyCoordsT:
     """Win32 may start with negative coords when multiscreen."""
     import winput
     from win32api import GetSystemMetrics
@@ -142,14 +164,14 @@ def win32_coords_start() -> tuple[int, int]:
     return x, y
 
 
-snip_start_coords: tuple[int, int] | None = None
+snip_start_coords: XyCoordsT | None = None
 
 
 def toggle_snip(
     prefix: str | None = None,
     bbox_to_name: bool = True,
     bbox_callback: Callable[[int, int, int, int], Any] | None = None,
-    picture_callback: Callable[[ndarray], Any] | None = None,
+    picture_callback: Callable[[ImagePixelMatrixT], Any] | None = None,
 ) -> None:
     global snip_start_coords
     if not snip_start_coords:
@@ -174,9 +196,9 @@ def start_snip() -> None:
 def finish_snip_with_callback(
     prefix: str | None = None,
     bbox_to_name: bool = True,
-    bbox: tuple[int, int, int, int] | None = None,
+    bbox: BboxT | None = None,
     bbox_callback: Callable[[int, int, int, int], Any] | None = None,
-    picture_callback: Callable[[ndarray], Any] | None = None,
+    picture_callback: Callable[[ImagePixelMatrixT], Any] | None = None,
 ) -> None:
     nd_sct, bbox = finish_snip(prefix, bbox, bbox_to_name)
     if bbox and bbox_callback:
@@ -187,19 +209,19 @@ def finish_snip_with_callback(
 
 def finish_snip(
     prefix: str | None = None,
-    bbox: tuple[int, int, int, int] | None = None,
+    bbox: BboxT | None = None,
     bbox_to_name: bool = True,
-) -> tuple[ndarray, tuple[int, int, int, int] | None]:
+) -> tuple[ImagePixelMatrixT, BboxT | None]:
     sct = get_screenshot_if_none_and_cut(None, bbox)
     if prefix is not None:
         save_to_disk(sct, prefix, bbox, bbox_to_name)
     return sct, bbox
 
 
 def save_to_disk(
-    sct: ndarray,
+    sct: ImagePixelMatrixT,
     prefix: str,
-    bbox: tuple[int, int, int, int] | None,
+    bbox: BboxT | None,
     bbox_in_name: bool,
 ) -> None:
     bbox_str = (
@@ -228,13 +250,11 @@ def save_to_disk(
 )
 
 
-def get_pixel_color(
-    coords: tuple[int, int], outer: str | ndarray | None
-) -> tuple[int, int, int]:
-    outer = to_pixel_matrix(outer)  # type: ignore
+def get_pixel_color(coords: XyCoordsT, outer: ImageT | None) -> PixelColorT:
+    outer_image = to_pixel_matrix(outer)
     bbox = coords_to_bbox_1_pixel(coords)
-    outer = get_screenshot_if_none_and_cut(outer, bbox)
-    nd_pixel = outer[0][0]
+    outer_certain = get_screenshot_if_none_and_cut(outer_image, bbox)
+    nd_pixel = outer_certain[0][0]
     return tuple(c for c in nd_pixel)  # type: ignore
 
 

src/tapper/helper/img.py

@@ -56,6 +56,8 @@ def from_path(pathlike: ImagePathT) -> ImagePixelMatrixT:
     return _image_util.from_path(pathlike)  # type: ignore
 
 
+# todo return middle of searched area, not top left
+# 100x100 target that is found at 300, 300 should return 350, 350
 def find(
     target: ImageT,
     bbox: BboxT | None = None,
@@ -143,7 +145,9 @@ def wait_for_one_of(
     """
     Regularly search the screen or region of the screen for images,
     returning first that appears, or None if timeout.
-    This is blocking until timeout, obviously.
+    This is blocking until timeout or until found something.
+    For performance it's recommended to use images and not filenames as targets.
+        see :func:from_path.
 
     :param images: see `SearchableImage`.
     :param timeout: see `wait_for` param.
@@ -256,7 +260,7 @@ def get_snip(
 
 def pixel_info(
     callback: Callable[[PixelColorT, XyCoordsT], Any],
-    outer: str | ImagePixelMatrixT | None = None,
+    outer: ImageT | None = None,
 ) -> Callable[[], Any]:
     """
     Click to get pixel color and coordinates and call the callback with it.
@@ -294,6 +298,7 @@ def pixel_str(
     return lambda: callback(_image_util.pixel_str(tapper.mouse.get_pos(), outer))
 
 
+# todo method to get color #FFFFFF?
 def pixel_get_color(
     coords: XyCoordsT, outer: str | ImagePixelMatrixT | None = None
 ) -> PixelColorT:
@@ -309,6 +314,7 @@ def pixel_get_color(
     return _image_util.get_pixel_color(coords, outer)
 
 
+# todo accept color #FFFFFF
 def pixel_find(
     color: PixelColorT,
     bbox_or_coords: BboxT | XyCoordsT | None = None,

tests/tapper/helper/image/img_for_test.py

@@ -1,7 +1,8 @@
 import importlib.resources
+from pathlib import Path
 
 import numpy as np
-import testresources
+import resources
 from numpy import ndarray
 from PIL import Image
 
@@ -10,11 +11,13 @@ def from_matrix(matrix: list[list[tuple[int, int, int]]]) -> ndarray:
     return np.uint8(matrix)  # type: ignore
 
 
+def get_image_path(name: str) -> Path:
+    return importlib.resources.files(resources).joinpath("image").joinpath(name)  # type: ignore
+
+
 def get_picture(name: str) -> ndarray:
-    full_name = (
-        importlib.resources.files(testresources).joinpath("image").joinpath(name)
-    )
-    pil_img = Image.open(full_name).convert("RGB")
+    path = get_image_path(name)
+    pil_img = Image.open(path).convert("RGB")
     return np.asarray(pil_img)
 
 

tests/tapper/helper/image/test_img.py

@@ -6,10 +6,72 @@
 blue = 0, 0, 255
 black = 0, 0, 0
 white = 255, 255, 255
+gray = 128, 128, 128
+
+absolutes = img_for_test.absolutes()
 
 
 class TestFind:
     def test_simplest(self) -> None:
-        absolutes = img_for_test.absolutes()
-        xy = img.find(img_for_test.from_matrix([[red]]), outer=absolutes)
-        assert xy == (0, 0)
+        xy = img.find(img_for_test.from_matrix([[blue]]), outer=absolutes)
+        assert xy == (2, 0)
+
+    def test_target_is_path(self) -> None:
+        pic_path = img_for_test.get_image_path("absolutes.png")
+        xy = img.find(img_for_test.from_matrix([[green]]), outer=pic_path)
+        assert xy == (1, 0)
+
+    def test_corner_case_black_not_found(self) -> None:
+        """This is technically a bug of tapper, but caused by openCV
+        algorithm that's searching. All-black picture will not be found.
+        This is corner case where all target pixels are same, in which case
+        user should do pixel_find instead."""
+        xy = img.find(img_for_test.from_matrix([[black]]), outer=absolutes)
+        assert xy is None
+
+    def test_corner_case_gray_is_white(self) -> None:
+        """This is technically a bug of tapper, but caused by openCV
+        algorithm that's searching. Picture of gray pixel is found
+        on white pixel, with 100% match.
+        This is corner case where all target pixels are same, in which case
+        user should do pixel_find instead."""
+        xy = img.find(img_for_test.from_matrix([[gray]]), outer=absolutes)
+        assert xy == (0, 4)
+
+    def test_dependencies_not_installed(self) -> None:
+        pass
+
+    def test_with_bbox(self) -> None:
+        xy = img.find(img_for_test.from_matrix([[white]]), (2, 2, 3, 5), absolutes)
+        assert xy == (2, 4)
+
+    def test_not_found(self) -> None:
+        xy = img.find(img_for_test.from_matrix([[(100, 150, 50)]]), outer=absolutes)
+        assert xy is None
+
+    def test_bbox_larger_than_outer(self) -> None:
+        # with pytest.raises(ValueError):
+        #     img.find(img_for_test.from_matrix([[green]]), (0, 0, 100, 100), absolutes)
+        pass
+
+    def test_target_larger_than_outer(self) -> None:
+        pass
+
+    def test_screenshot_not_outer(self) -> None:
+        pass
+
+
+class TestFindFuzz:
+    def test_precise_not_found__approximate_found(self) -> None:
+        pass
+
+    def test_several_similar_targets(self) -> None:
+        pass
+
+
+class TestSnip:
+    def test_simplest(self) -> None:
+        pass
+
+    def test_saved_image_same_as_on_disk(self) -> None:
+        pass

tests/tapper/helper/image/test_pixel.py

@@ -0,0 +1,26 @@
+import img_for_test
+from tapper.helper import img
+
+red = 255, 0, 0
+green = 0, 255, 0
+blue = 0, 0, 255
+black = 0, 0, 0
+white = 255, 255, 255
+gray = 128, 128, 128
+
+absolutes = img_for_test.absolutes()
+
+
+class TestFind:
+    def test_simplest(self) -> None:
+        xy = img.pixel_find(green, outer=absolutes)
+        assert xy == (1, 0)
+
+    def test_not_found(self) -> None:
+        pass
+
+    def test_precise_not_found__approximate_found(self) -> None:
+        pass
+
+    def test_pixel_color_out_of_bounds(self) -> None:
+        pass