diff --git a/src/tapper/helper/_util/image_util.py b/src/tapper/helper/_util/image_util.py index a7c7fab..5af26ae 100644 --- a/src/tapper/helper/_util/image_util.py +++ b/src/tapper/helper/_util/image_util.py @@ -14,9 +14,12 @@ from mss.base import MSSBase from numpy import ndarray from tapper.helper._util import image_fuzz +from tapper.helper.model_types import BboxT from tapper.helper.model_types import ImagePathT from tapper.helper.model_types import ImagePixelMatrixT from tapper.helper.model_types import ImageT +from tapper.helper.model_types import PixelColorT +from tapper.helper.model_types import XyCoordsT from tapper.model import constants _bbox_pattern = re.compile(r"\(BBOX_-?\d+_-?\d+_-?\d+_-?\d+\)") @@ -52,13 +55,11 @@ def to_pixel_matrix(image: ImageT | None) -> ImagePixelMatrixT | None: def normalize( data_in: Union[ None, - str, - tuple[str, tuple[int, int, int, int]], - tuple[ndarray, tuple[int, int, int, int]], - tuple[ndarray, None], - ndarray, + ImageT, + tuple[ImageT, BboxT], + tuple[ImageT, None], ] -) -> tuple[ndarray | None, tuple[int, int, int, int] | None]: +) -> tuple[ImagePixelMatrixT | None, BboxT | None]: if data_in is None: return None, None bbox = None @@ -71,12 +72,12 @@ def normalize( sx = str_bbox.group().split("_") bbox = int(sx[1]), int(sx[2]), int(sx[3]), int(sx[4].rstrip(")")) return from_path(data_in), bbox - raise TypeError(f"Unexpected type, {type(data_in)} of {data_in}") + raise TypeError(f"Unexpected type {type(data_in)} of {data_in!r}") def get_screenshot_if_none_and_cut( - maybe_image: ndarray | None, bbox: tuple[int, int, int, int] | None -) -> ndarray: + maybe_image: ImagePixelMatrixT | None, bbox: BboxT | None +) -> ImagePixelMatrixT: if maybe_image is not None: if bbox: return maybe_image[bbox[1] : bbox[3], bbox[0] : bbox[2]] @@ -93,9 +94,9 @@ def get_screenshot_if_none_and_cut( def find_in_image_raw( - inner_image_bbox: tuple[ndarray, tuple[int, int, int, int] | None], - outer: ndarray | None = None, -) -> tuple[float, tuple[int, int]]: + inner_image_bbox: tuple[ndarray, BboxT | None], + outer: ImagePixelMatrixT | None = None, +) -> tuple[float, XyCoordsT]: image_arr, bbox = inner_image_bbox x_start, y_start = get_start_coords(outer, bbox) outer = get_screenshot_if_none_and_cut(outer, bbox) @@ -104,9 +105,30 @@ def find_in_image_raw( return confidence, (x_start + coords[0], y_start + coords[1]) +def image_find( + target: ImageT, + bbox: tuple[int, int, int, int] | None, + outer: ImageT | None = None, + precision: float = 1.0, +) -> XyCoordsT | None: + if target is None: + raise ValueError("image_find nees something to search for.") + target_image = to_pixel_matrix(target) + assert target_image is not None # for mypy + outer_image = to_pixel_matrix(outer) + # has to be before screenshot is taken, for Windows multi-monitor case + x_start, y_start = get_start_coords(outer_image, bbox) + certain_outer = get_screenshot_if_none_and_cut(outer_image, bbox) + confidence, coords = image_fuzz.find(certain_outer, target_image) + + if confidence < precision: + return None + return x_start + coords[0], y_start + coords[1] + + def find_in_image( target: ImagePixelMatrixT, - bbox: tuple[int, int, int, int] | None, + bbox: BboxT | None, outer: ndarray | None = None, precision: float = 1.0, ) -> tuple[int, int] | None: @@ -121,8 +143,8 @@ def find_in_image( def get_start_coords( outer: ndarray | None, - bbox_or_coords: tuple[int, int, int, int] | tuple[int, int] | None, -) -> tuple[int, int]: + bbox_or_coords: BboxT | XyCoordsT | None, +) -> XyCoordsT: if bbox_or_coords is not None: return bbox_or_coords[0], bbox_or_coords[1] screenshot_required = outer is None @@ -131,7 +153,7 @@ def get_start_coords( return 0, 0 -def win32_coords_start() -> tuple[int, int]: +def win32_coords_start() -> XyCoordsT: """Win32 may start with negative coords when multiscreen.""" import winput from win32api import GetSystemMetrics @@ -142,14 +164,14 @@ def win32_coords_start() -> tuple[int, int]: return x, y -snip_start_coords: tuple[int, int] | None = None +snip_start_coords: XyCoordsT | None = None def toggle_snip( prefix: str | None = None, bbox_to_name: bool = True, bbox_callback: Callable[[int, int, int, int], Any] | None = None, - picture_callback: Callable[[ndarray], Any] | None = None, + picture_callback: Callable[[ImagePixelMatrixT], Any] | None = None, ) -> None: global snip_start_coords if not snip_start_coords: @@ -174,9 +196,9 @@ def start_snip() -> None: def finish_snip_with_callback( prefix: str | None = None, bbox_to_name: bool = True, - bbox: tuple[int, int, int, int] | None = None, + bbox: BboxT | None = None, bbox_callback: Callable[[int, int, int, int], Any] | None = None, - picture_callback: Callable[[ndarray], Any] | None = None, + picture_callback: Callable[[ImagePixelMatrixT], Any] | None = None, ) -> None: nd_sct, bbox = finish_snip(prefix, bbox, bbox_to_name) if bbox and bbox_callback: @@ -187,9 +209,9 @@ def finish_snip_with_callback( def finish_snip( prefix: str | None = None, - bbox: tuple[int, int, int, int] | None = None, + bbox: BboxT | None = None, bbox_to_name: bool = True, -) -> tuple[ndarray, tuple[int, int, int, int] | None]: +) -> tuple[ImagePixelMatrixT, BboxT | None]: sct = get_screenshot_if_none_and_cut(None, bbox) if prefix is not None: save_to_disk(sct, prefix, bbox, bbox_to_name) @@ -197,9 +219,9 @@ def finish_snip( def save_to_disk( - sct: ndarray, + sct: ImagePixelMatrixT, prefix: str, - bbox: tuple[int, int, int, int] | None, + bbox: BboxT | None, bbox_in_name: bool, ) -> None: bbox_str = ( @@ -228,13 +250,11 @@ def save_to_disk( ) -def get_pixel_color( - coords: tuple[int, int], outer: str | ndarray | None -) -> tuple[int, int, int]: - outer = to_pixel_matrix(outer) # type: ignore +def get_pixel_color(coords: XyCoordsT, outer: ImageT | None) -> PixelColorT: + outer_image = to_pixel_matrix(outer) bbox = coords_to_bbox_1_pixel(coords) - outer = get_screenshot_if_none_and_cut(outer, bbox) - nd_pixel = outer[0][0] + outer_certain = get_screenshot_if_none_and_cut(outer_image, bbox) + nd_pixel = outer_certain[0][0] return tuple(c for c in nd_pixel) # type: ignore diff --git a/src/tapper/helper/img.py b/src/tapper/helper/img.py index 81763a2..7eabf50 100644 --- a/src/tapper/helper/img.py +++ b/src/tapper/helper/img.py @@ -56,6 +56,8 @@ def from_path(pathlike: ImagePathT) -> ImagePixelMatrixT: return _image_util.from_path(pathlike) # type: ignore +# todo return middle of searched area, not top left +# 100x100 target that is found at 300, 300 should return 350, 350 def find( target: ImageT, bbox: BboxT | None = None, @@ -143,7 +145,9 @@ def wait_for_one_of( """ Regularly search the screen or region of the screen for images, returning first that appears, or None if timeout. - This is blocking until timeout, obviously. + This is blocking until timeout or until found something. + For performance it's recommended to use images and not filenames as targets. + see :func:from_path. :param images: see `SearchableImage`. :param timeout: see `wait_for` param. @@ -256,7 +260,7 @@ def get_snip( def pixel_info( callback: Callable[[PixelColorT, XyCoordsT], Any], - outer: str | ImagePixelMatrixT | None = None, + outer: ImageT | None = None, ) -> Callable[[], Any]: """ Click to get pixel color and coordinates and call the callback with it. @@ -294,6 +298,7 @@ def pixel_str( return lambda: callback(_image_util.pixel_str(tapper.mouse.get_pos(), outer)) +# todo method to get color #FFFFFF? def pixel_get_color( coords: XyCoordsT, outer: str | ImagePixelMatrixT | None = None ) -> PixelColorT: @@ -309,6 +314,7 @@ def pixel_get_color( return _image_util.get_pixel_color(coords, outer) +# todo accept color #FFFFFF def pixel_find( color: PixelColorT, bbox_or_coords: BboxT | XyCoordsT | None = None, diff --git a/tests/testresources/image/absolutes.png b/tests/resources/image/absolutes.png similarity index 100% rename from tests/testresources/image/absolutes.png rename to tests/resources/image/absolutes.png diff --git a/tests/resources/image/test-buttons.png b/tests/resources/image/test-buttons.png new file mode 100644 index 0000000..a367012 Binary files /dev/null and b/tests/resources/image/test-buttons.png differ diff --git a/tests/tapper/helper/image/img_for_test.py b/tests/tapper/helper/image/img_for_test.py index 571593e..99581a8 100644 --- a/tests/tapper/helper/image/img_for_test.py +++ b/tests/tapper/helper/image/img_for_test.py @@ -1,7 +1,8 @@ import importlib.resources +from pathlib import Path import numpy as np -import testresources +import resources from numpy import ndarray from PIL import Image @@ -10,11 +11,13 @@ def from_matrix(matrix: list[list[tuple[int, int, int]]]) -> ndarray: return np.uint8(matrix) # type: ignore +def get_image_path(name: str) -> Path: + return importlib.resources.files(resources).joinpath("image").joinpath(name) # type: ignore + + def get_picture(name: str) -> ndarray: - full_name = ( - importlib.resources.files(testresources).joinpath("image").joinpath(name) - ) - pil_img = Image.open(full_name).convert("RGB") + path = get_image_path(name) + pil_img = Image.open(path).convert("RGB") return np.asarray(pil_img) diff --git a/tests/tapper/helper/image/test_img.py b/tests/tapper/helper/image/test_img.py index d5759db..b23c3b7 100644 --- a/tests/tapper/helper/image/test_img.py +++ b/tests/tapper/helper/image/test_img.py @@ -6,10 +6,72 @@ blue = 0, 0, 255 black = 0, 0, 0 white = 255, 255, 255 +gray = 128, 128, 128 + +absolutes = img_for_test.absolutes() class TestFind: def test_simplest(self) -> None: - absolutes = img_for_test.absolutes() - xy = img.find(img_for_test.from_matrix([[red]]), outer=absolutes) - assert xy == (0, 0) + xy = img.find(img_for_test.from_matrix([[blue]]), outer=absolutes) + assert xy == (2, 0) + + def test_target_is_path(self) -> None: + pic_path = img_for_test.get_image_path("absolutes.png") + xy = img.find(img_for_test.from_matrix([[green]]), outer=pic_path) + assert xy == (1, 0) + + def test_corner_case_black_not_found(self) -> None: + """This is technically a bug of tapper, but caused by openCV + algorithm that's searching. All-black picture will not be found. + This is corner case where all target pixels are same, in which case + user should do pixel_find instead.""" + xy = img.find(img_for_test.from_matrix([[black]]), outer=absolutes) + assert xy is None + + def test_corner_case_gray_is_white(self) -> None: + """This is technically a bug of tapper, but caused by openCV + algorithm that's searching. Picture of gray pixel is found + on white pixel, with 100% match. + This is corner case where all target pixels are same, in which case + user should do pixel_find instead.""" + xy = img.find(img_for_test.from_matrix([[gray]]), outer=absolutes) + assert xy == (0, 4) + + def test_dependencies_not_installed(self) -> None: + pass + + def test_with_bbox(self) -> None: + xy = img.find(img_for_test.from_matrix([[white]]), (2, 2, 3, 5), absolutes) + assert xy == (2, 4) + + def test_not_found(self) -> None: + xy = img.find(img_for_test.from_matrix([[(100, 150, 50)]]), outer=absolutes) + assert xy is None + + def test_bbox_larger_than_outer(self) -> None: + # with pytest.raises(ValueError): + # img.find(img_for_test.from_matrix([[green]]), (0, 0, 100, 100), absolutes) + pass + + def test_target_larger_than_outer(self) -> None: + pass + + def test_screenshot_not_outer(self) -> None: + pass + + +class TestFindFuzz: + def test_precise_not_found__approximate_found(self) -> None: + pass + + def test_several_similar_targets(self) -> None: + pass + + +class TestSnip: + def test_simplest(self) -> None: + pass + + def test_saved_image_same_as_on_disk(self) -> None: + pass diff --git a/tests/tapper/helper/image/test_pixel.py b/tests/tapper/helper/image/test_pixel.py new file mode 100644 index 0000000..f3f8148 --- /dev/null +++ b/tests/tapper/helper/image/test_pixel.py @@ -0,0 +1,26 @@ +import img_for_test +from tapper.helper import img + +red = 255, 0, 0 +green = 0, 255, 0 +blue = 0, 0, 255 +black = 0, 0, 0 +white = 255, 255, 255 +gray = 128, 128, 128 + +absolutes = img_for_test.absolutes() + + +class TestFind: + def test_simplest(self) -> None: + xy = img.pixel_find(green, outer=absolutes) + assert xy == (1, 0) + + def test_not_found(self) -> None: + pass + + def test_precise_not_found__approximate_found(self) -> None: + pass + + def test_pixel_color_out_of_bounds(self) -> None: + pass