core: add helper for more_iterable to check that all types involved a…

…re hashable

Otherwise unique_everseen performance may degrade to quadratic rather than linear

For now hidden behind HPI_CHECK_UNIQUE_EVERSEEN flag

also switch some modules to use it

my/core/common.py

@@ -6,7 +6,25 @@
 import os
 import sys
 import types
-from typing import Union, Callable, Dict, Iterable, TypeVar, Sequence, List, Optional, Any, cast, Tuple, TYPE_CHECKING, NoReturn
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    NoReturn,
+    Optional,
+    Sequence,
+    TYPE_CHECKING,
+    Tuple,
+    TypeVar,
+    Union,
+    cast,
+    get_args,
+    get_type_hints,
+    get_origin,
+)
 import warnings
 from . import warnings as core_warnings
 
@@ -628,6 +646,59 @@ def assert_never(value: NoReturn) -> NoReturn:
     assert False, f'Unhandled value: {value} ({type(value).__name__})'
 
 
+def _check_all_hashable(fun):
+    # TODO ok, take callable?
+    hints = get_type_hints(fun)
+    # TODO needs to be defensive like in cachew?
+    return_type = hints.get('return')
+    # TODO check if None
+    origin = get_origin(return_type)  # Iterator etc?
+    (arg,) = get_args(return_type)
+    # options we wanna handle are simple type on the top level or union
+    arg_origin = get_origin(arg)
+
+    if sys.version_info[:2] >= (3, 10):
+        is_uniontype = arg_origin is types.UnionType
+    else:
+        is_uniontype = False
+
+    is_union = arg_origin is Union or is_uniontype
+    if is_union:
+        to_check = get_args(arg)
+    else:
+        to_check = (arg,)
+
+    no_hash = [
+        t
+        for t in to_check
+        # seems that objects that have not overridden hash have the attribute but it's set to None
+        if getattr(t, '__hash__', None) is None
+    ]
+    assert len(no_hash) == 0, f'Types {no_hash} are not hashable, this will result in significant performance downgrade for unique_everseen'
+
+
+_UET = TypeVar('_UET')
+_UEU = TypeVar('_UEU')
+
+
+def unique_everseen(
+    fun: Callable[[], Iterable[_UET]],
+    key: Optional[Callable[[_UET], _UEU]] = None,
+) -> Iterator[_UET]:
+    # TODO support normal iterable as well?
+    import more_itertools
+
+    # NOTE: it has to take original callable, because otherwise we don't have access to generator type annotations
+    iterable = fun()
+
+    if key is None:
+        # todo check key return type as well? but it's more likely to be hashable
+        if os.environ.get('HPI_CHECK_UNIQUE_EVERSEEN') is not None:
+            _check_all_hashable(fun)
+
+    return more_itertools.unique_everseen(iterable=iterable, key=key)
+
+
 ## legacy imports, keeping them here for backwards compatibility
 from functools import cached_property as cproperty
 from typing import Literal

my/fbmessenger/android.py

@@ -9,9 +9,8 @@
 import sqlite3
 from typing import Iterator, Sequence, Optional, Dict, Union, List
 
-from more_itertools import unique_everseen
-
 from my.core import get_files, Paths, datetime_aware, Res, assert_never, LazyLogger, make_config
+from my.core.common import unique_everseen
 from my.core.error import echain
 from my.core.sqlite import sqlite_connection
 
@@ -242,7 +241,7 @@ def messages() -> Iterator[Res[Message]]:
     senders: Dict[str, Sender] = {}
     msgs: Dict[str, Message] = {}
     threads: Dict[str, Thread] = {}
-    for x in unique_everseen(_entities()):
+    for x in unique_everseen(_entities):
         if isinstance(x, Exception):
             yield x
             continue

my/instagram/android.py

@@ -10,8 +10,6 @@
 import sqlite3
 from typing import Iterator, Sequence, Optional, Dict, Union
 
-from more_itertools import unique_everseen
-
 from my.core import (
     get_files,
     Paths,
@@ -22,6 +20,7 @@
     Res,
     assert_never,
 )
+from my.core.common import unique_everseen
 from my.core.cachew import mcachew
 from my.core.error import echain
 from my.core.sqlite import sqlite_connect_immutable, select
@@ -196,7 +195,7 @@ def _entities() -> Iterator[Res[Union[User, _Message]]]:
 @mcachew(depends_on=inputs)
 def messages() -> Iterator[Res[Message]]:
     id2user: Dict[str, User] = {}
-    for x in unique_everseen(_entities()):
+    for x in unique_everseen(_entities):
         if isinstance(x, Exception):
             yield x
             continue

my/instagram/gdpr.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 from typing import Iterator, Sequence, Dict, Union
 
-from more_itertools import bucket, unique_everseen
+from more_itertools import bucket
 
 from my.core import (
     get_files,
@@ -17,6 +17,7 @@
     assert_never,
     make_logger,
 )
+from my.core.common import unique_everseen
 
 from my.config import instagram as user_config
 
@@ -196,7 +197,7 @@ def _entitites_from_path(path: Path) -> Iterator[Res[Union[User, _Message]]]:
 # TODO basically copy pasted from android.py... hmm
 def messages() -> Iterator[Res[Message]]:
     id2user: Dict[str, User] = {}
-    for x in unique_everseen(_entities()):
+    for x in unique_everseen(_entities):
         if isinstance(x, Exception):
             yield x
             continue

my/tinder/android.py

@@ -11,9 +11,8 @@
 import sqlite3
 from typing import Sequence, Iterator, Union, Dict, List, Mapping
 
-from more_itertools import unique_everseen
-
 from my.core import Paths, get_files, Res, assert_never, stat, Stats, datetime_aware, make_logger
+from my.core.common import unique_everseen
 from my.core.error import echain
 from my.core.sqlite import sqlite_connection
 import my.config
@@ -162,7 +161,7 @@ def _parse_msg(row: sqlite3.Row) -> _Message:
 def entities() -> Iterator[Res[Entity]]:
     id2person: Dict[str, Person] = {}
     id2match: Dict[str, Match] = {}
-    for x in unique_everseen(_entities()):
+    for x in unique_everseen(_entities):
         if isinstance(x, Exception):
             yield x
             continue

my/twitter/talon.py

@@ -9,9 +9,8 @@
 import sqlite3
 from typing import Iterator, Sequence, Union
 
-from more_itertools import unique_everseen
-
 from my.core import Paths, Res, datetime_aware, get_files
+from my.core.common import unique_everseen
 from my.core.sqlite import sqlite_connection
 
 from .common import TweetId, permalink
@@ -133,15 +132,15 @@ def _parse_tweet(row: sqlite3.Row) -> Tweet:
 
 
 def tweets() -> Iterator[Res[Tweet]]:
-    for x in unique_everseen(_entities()):
+    for x in unique_everseen(_entities):
         if isinstance(x, Exception):
             yield x
         elif isinstance(x, _IsTweet):
             yield x.tweet
 
 
 def likes() -> Iterator[Res[Tweet]]:
-    for x in unique_everseen(_entities()):
+    for x in unique_everseen(_entities):
         if isinstance(x, Exception):
             yield x
         elif isinstance(x, _IsFavorire):

my/vk/vk_messages_backup.py

@@ -5,12 +5,12 @@
 from datetime import datetime
 from dataclasses import dataclass
 import json
-from typing import Dict, Iterator, NamedTuple
+from typing import Dict, Iterator
 
-from more_itertools import unique_everseen
 import pytz
 
-from my.core import stat, Stats, Json, Res, datetime_aware
+from my.core import stat, Stats, Json, Res, datetime_aware, get_files
+from my.core.common import unique_everseen
 
 from my.config import vk_messages_backup as config
 
@@ -147,7 +147,7 @@ def _messages() -> Iterator[Res[Message]]:
 
 def messages() -> Iterator[Res[Message]]:
     # seems that during backup messages were sometimes duplicated..
-    yield from unique_everseen(_messages())
+    yield from unique_everseen(_messages)
 
 
 def stats() -> Stats:

my/whatsapp/android.py

@@ -9,9 +9,8 @@
 import sqlite3
 from typing import Sequence, Iterator, Optional
 
-from more_itertools import unique_everseen
-
 from my.core import get_files, Paths, datetime_aware, Res, make_logger, make_config
+from my.core.common import unique_everseen
 from my.core.error import echain, notnone
 from my.core.sqlite import sqlite_connection
 import my.config
@@ -202,4 +201,4 @@ def _messages() -> Iterator[Res[Message]]:
 
 
 def messages() -> Iterator[Res[Message]]:
-    yield from unique_everseen(_messages())
+    yield from unique_everseen(_messages)