ci: update configs and fix mypy

.github/workflows/main.yml

@@ -29,9 +29,7 @@ jobs:
             # windows runners are pretty scarce, so let's only run lowest and highest python version
             {platform: windows-latest, python-version: '3.9' },
             {platform: windows-latest, python-version: '3.10'},
-            # NOTE: some test packages (e.g. mouseinfo) have issues with building wheels on 3.12 windows
-            # perhaps try again later
-            {platform: windows-latest, python-version: '3.12'},
+            {platform: windows-latest, python-version: '3.11'},
 
             # same, macos is a bit too slow and ubuntu covers python quirks well
             {platform: macos-latest  , python-version: '3.9' },
@@ -125,7 +123,7 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.10'
 
     - uses: actions/checkout@v4
       with:
@@ -155,9 +153,11 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: recursive
+        fetch-depth: 0  # nicer to have all git history when debugging/for tests
+
     - uses: actions/setup-node@v4
       with:
-        node-version: '18'
+        node-version: '20'
 
     - run: extension/.ci/build --lint # debug version
     - run: extension/.ci/build --lint --release

mypy.ini

@@ -1,12 +1,14 @@
 [mypy]
+namespace_packages = True
 pretty = True
 show_error_context = True
 show_error_codes = True
+show_column_numbers = True
+show_error_end = True
+warn_unused_ignores = True
 check_untyped_defs = True
-namespace_packages = True
-
-[mypy-pytest]
-ignore_missing_imports = True
+enable_error_code = possibly-undefined
+strict_equality = True
 
 # not sure why mypy started discovering it (since 0.800??)
 [mypy-hypothesis]

ruff.toml

@@ -1,4 +1,4 @@
-ignore = [
+lint.ignore = [
 ### too opinionated style checks
     "E501",  # too long lines
     "E702",  # Multiple statements on one line (semicolon)

src/promnesia/__main__.py

@@ -327,14 +327,14 @@ def add_index_args(parser: argparse.ArgumentParser, default_config_path: PathIsh
         )
 
     F = lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, width=120)
-    p = argparse.ArgumentParser(formatter_class=F) # type: ignore
+    p = argparse.ArgumentParser(formatter_class=F)
     subp = p.add_subparsers(dest='mode', )
     ep = subp.add_parser('index', help='Create/update the link database', formatter_class=F)
     add_index_args(ep, default_config_path())
     # TODO use some way to override or provide config only via cmdline?
     ep.add_argument('--intermediate', required=False, help="Used for development, you don't need it")
 
-    sp = subp.add_parser('serve', help='Serve a link database', formatter_class=F) # type: ignore
+    sp = subp.add_parser('serve', help='Serve a link database', formatter_class=F)
     server.setup_parser(sp)
 
     ap = subp.add_parser('demo', help='Demo mode: index and serve a directory in single command', formatter_class=F)

src/promnesia/cannon.py

@@ -422,7 +422,7 @@ def canonify(url: str) -> str:
     qq = [(k, v) for i, k, v in sorted(iqq)]
     # TODO still not sure what we should do..
     # quote_plus replaces %20 with +, not sure if we want it...
-    query = urlencode(qq, quote_via=quote_via) # type: ignore[type-var]
+    query = urlencode(qq, quote_via=quote_via)
 
     path = _quote_path(path)
 
@@ -683,7 +683,7 @@ def domains(it): # pragma: no cover
         try:
             nurl = canonify(url)
         except CanonifyException as e:
-            print(f"ERROR while normalising! {nurl} {e}")
+            print(f"ERROR while normalising! {url} {e}")
             c['ERROR'] += 1
             continue
         else:
@@ -718,7 +718,7 @@ def reg(url, pat):
         try:
             nurl = canonify(url)
         except CanonifyException as e:
-            print(f"ERROR while normalising! {nurl} {e}")
+            print(f"ERROR while normalising! {url} {e}")
             continue
         udom = nurl[:nurl.find('/')]
         usplit = udom.split('.')
@@ -818,7 +818,7 @@ def main() -> None: # pragma: no cover
 
 - running comparison
   sqlite3 promnesia.sqlite 'select distinct orig_url from visits where norm_url like "%twitter%" order by orig_url' | src/promnesia/cannon.py
-''', formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, width=100) # type: ignore
+''', formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, width=100)
     )
     p.add_argument('input', nargs='?')
     p.add_argument('--human', action='store_true')

src/promnesia/common.py

@@ -313,7 +313,7 @@ def _get_index_function(sourceish: PreSource) -> PreExtractor:
     if hasattr(sourceish, 'index'):  # must be a module
         res = getattr(sourceish, 'index')
     else:
-        res = sourceish  # type: ignore[assignment]
+        res = sourceish
     return res
 
 
@@ -392,7 +392,7 @@ def appdirs():
     under_test = os.environ.get('PYTEST_CURRENT_TEST') is not None
     # todo actually use test name?
     name = 'promnesia-test' if under_test else 'promnesia'
-    import appdirs as ad # type: ignore[import]
+    import appdirs as ad # type: ignore[import-untyped]
     return ad.AppDirs(appname=name)
 
 
@@ -482,13 +482,13 @@ def fdfind_args(root: Path, follow: bool, ignore: List[str]=[]) -> List[str]:
     ignore_args = []
     if ignore:
         # Add a statement that excludes the folder
-        ignore_args = [['--exclude', f'{n}'] for n in ignore]
+        _ignore_args = [['--exclude', f'{n}'] for n in ignore]
         # Flatten the list of lists
-        ignore_args_l = list(itertools.chain(*ignore_args))
+        ignore_args = list(itertools.chain(*_ignore_args))
 
     return [
         *extra_fd_args(),
-        *ignore_args_l,
+        *ignore_args,
         *(['--follow'] if follow else []),
         '--type', 'f',
         '.',
@@ -537,10 +537,9 @@ def traverse(root: Path, *, follow: bool=True, ignore: List[str]=[]) -> Iterable
 def get_system_zone() -> str:
     try:
         import tzlocal
-        # note: tzlocal mypy stubs aren't aware of api change yet (see https://github.com/python/typeshed/issues/6038)
         try:
             # 4.0 way
-            return tzlocal.get_localzone_name() # type: ignore[attr-defined]
+            return tzlocal.get_localzone_name()
         except AttributeError as e:
             # 2.0 way
             zone = tzlocal.get_localzone().zone  # type: ignore[attr-defined]

src/promnesia/compare.py

@@ -143,7 +143,7 @@ def compare_files(*files: Path, log=True) -> Iterator[Tuple[str, DbVisit]]:
         engine, table = _get_stuff(PathWithMtime.make(f))
 
         with engine.connect() as conn:
-            vis = [row_to_db_visit(row) for row in conn.execute(table.select())]  # type: ignore[var-annotated]
+            vis = [row_to_db_visit(row) for row in conn.execute(table.select())]
 
         if last is not None:
             between = f'{last_dts}:{this_dts}'

src/promnesia/config.py

@@ -6,7 +6,7 @@
 import importlib.util
 import warnings
 
-from .common import PathIsh, get_tmpdir, appdirs, default_output_dir, default_cache_dir, user_config_file
+from .common import PathIsh, default_output_dir, default_cache_dir
 from .common import Res, Source, DbVisit
 
 
@@ -129,7 +129,7 @@ def import_config(config_file: PathIsh) -> Config:
     spec = importlib.util.spec_from_file_location(name, p); assert spec is not None
     mod = importlib.util.module_from_spec(spec); assert mod is not None
     loader = spec.loader; assert loader is not None
-    loader.exec_module(mod) # type: ignore[attr-defined]
+    loader.exec_module(mod)
 
     d = {}
     for f in Config._fields:

src/promnesia/kjson.py

@@ -74,7 +74,7 @@ def test_json_processor():
     handled = []
     class Proc(JsonProcessor):
         def handle_dict(self, value: JDict, path):
-            if 'skipme' in self.kpath(path):
+            if 'skipme' in self.kpath(path):  # type: ignore[comparison-overlap]
                 return JsonProcessor.SKIP
 
         def handle_str(self, value: str, path):

src/promnesia/logging.py

@@ -61,7 +61,7 @@ def mklevel(level: LevelIsh) -> Level:
 def setup_logger(logger: logging.Logger, level: LevelIsh) -> None:
     lvl = mklevel(level)
     try:
-        import logzero  # type: ignore[import]
+        import logzero  # type: ignore[import-not-found]
         formatter = logzero.LogFormatter(
             fmt=FORMAT_COLOR,
             datefmt=DATEFMT,
@@ -75,7 +75,7 @@ def setup_logger(logger: logging.Logger, level: LevelIsh) -> None:
     logger.addFilter(AddExceptionTraceback())
     if use_logzero and not COLLAPSE_DEBUG_LOGS: # all set, nothing to do
         # 'simple' setup
-        logzero.setup_logger(logger.name, level=lvl, formatter=formatter)
+        logzero.setup_logger(logger.name, level=lvl, formatter=formatter)  # type: ignore[possibly-undefined]
         return
 
     h = CollapseDebugHandler() if COLLAPSE_DEBUG_LOGS else logging.StreamHandler()
@@ -101,7 +101,7 @@ def isEnabledFor_lazyinit(*args, logger=logger, orig=logger.isEnabledFor, **kwar
         # oh god.. otherwise might go into an inf loop
         if not hasattr(logger, _init_done):
             setattr(logger, _init_done, False) # will setup on the first call
-            logger.isEnabledFor = isEnabledFor_lazyinit  # type: ignore[assignment]
+            logger.isEnabledFor = isEnabledFor_lazyinit  # type: ignore[method-assign]
         return cast(LazyLogger, logger)
 
 

src/promnesia/sources/auto.py

@@ -320,7 +320,7 @@ def _index_file(pp: Path, opts: Options) -> Results:
 
     def indexer() -> Union[Urls, Results]:
         # eh, annoying.. need to make more generic..
-        idx = ip(pp) # type: ignore
+        idx = ip(pp)
         try:
             yield from idx
         except Exception as e:

src/promnesia/sources/browser_legacy.py

@@ -10,7 +10,7 @@
 from .. import config
 
 try:
-    from cachew import cachew  # type: ignore[import-not-found]
+    from cachew import cachew
 except ModuleNotFoundError as me:
     if me.name != 'cachew':
         raise me

src/promnesia/sources/github.py

@@ -31,7 +31,7 @@ def index(*, render_markdown: bool = False) -> Results:
         # if enabled, convert the (markdown) body to HTML
         context: Optional[str] = e.body
         if e.body is not None and render_markdown:
-            context = TextParser(e.body)._doc_ashtml()
+            context = TextParser(e.body)._doc_ashtml()  # type: ignore[possibly-undefined]
 
         # locator should link back to this event
         loc = Loc.make(title=e.summary, href=e.link)
@@ -74,7 +74,7 @@ def index(*, render_markdown: bool = False) -> Results:
         # extract from markdown links like [link text](https://...)
         # incase URLExtract missed any somehow
         if render_markdown:
-            for res in extract_from_text(e.body):
+            for res in extract_from_text(e.body):  # type: ignore[possibly-undefined]
                 if isinstance(res, Exception):
                     yield res
                     continue

src/promnesia/sources/org.py

@@ -84,7 +84,7 @@ def walk_node(*, node: OrgNode, dt: datetime) -> Iterator[Res[Tuple[Parsed, OrgN
             parsed = parsed._replace(dt=dt)
         else:
             dt = parsed.dt
-    yield parsed, node
+        yield parsed, node
 
     for c in node.children:
         yield from walk_node(node=c, dt=dt)

src/promnesia/sources/reddit.py

@@ -16,7 +16,7 @@ def index(*, render_markdown: bool = False, renderer: Optional[Type['RedditRende
         if "No module named 'my.reddit.all'" in str(e):
             import warnings
             warnings.warn("DEPRECATED/reddit: Using an old version of HPI, please update")
-            from my.reddit import submissions, comments, saved, upvoted  # type: ignore[no-redef]
+            from my.reddit import submissions, comments, saved, upvoted
         else:
             raise e
 

src/promnesia/sources/signal.py

@@ -63,6 +63,8 @@ def index(
     logger.debug("Paths to harvest: %s", db_paths)
     if not http_only:
         sql_query = f"{messages_query}\nWHERE body LIKE '%http%'"
+    else:
+        sql_query = messages_query
 
     for db_path in resolved_db_paths:
         logger.info("Ciphered db to harvest %s", db_path)
@@ -188,8 +190,8 @@ def _expand_path(path_pattern: PathIsh) -> Iterable[Path]:
 
 def _expand_paths(paths: PathIshes) -> Iterable[Path]:
     if _is_pathish(paths):
-        paths = [paths]  # type: ignore[assignment,list-item]
-    return [pp.resolve() for p in paths for pp in _expand_path(p)]  # type: ignore[union-attr,list-item]
+        paths = [paths]  # type: ignore[list-item]
+    return [pp.resolve() for p in paths for pp in _expand_path(p)]  # type: ignore[union-attr]
 
 
 def collect_db_paths(*db_paths: PathIsh, append: bool = False) -> Iterable[Path]:
@@ -236,7 +238,7 @@ def collect_db_paths(*db_paths: PathIsh, append: bool = False) -> Iterable[Path]
             )
 
         if db_paths and append:
-            db_paths = [  # type: ignore[misc,assignment]
+            db_paths = [  # type: ignore[assignment]
                 *([db_paths] if _is_pathish(db_paths) else db_paths),
                 plat_paths,
             ]
@@ -320,7 +322,7 @@ def connect_db(
                 "Decrypting db '%s' with cmd: %s <<<EOF\n%s\nEOF", db_path, cmd, sql
             )
             try:
-                sbp.run(  # type: ignore[call-overload]
+                sbp.run(
                     cmd,
                     check=True,
                     input=sql,
@@ -335,7 +337,7 @@ def connect_db(
                 ) from None
             db = sqlite3.connect(f"file:{decrypted_file}?mode=ro", uri=True)
         else:
-            from sqlcipher3 import dbapi2  # type: ignore[import]
+            from sqlcipher3 import dbapi2  # type: ignore[import-not-found]
 
             db = dbapi2.connect(f"file:{db_path}?mode=ro", uri=True)
             # Param-binding doesn't work for pragmas, so use a direct string concat.
@@ -419,9 +421,9 @@ def _harvest_db(
 
     with connect_db(db_path, key, decrypt_db=decrypt_db, **decryption_pragmas) as db:
         for mid, tstamp, sender, cid, chatname, text in db.execute(messages_query):
+            tstamp = from_epoch(tstamp / 1000.0)
+            row = (mid, tstamp, sender, cid, chatname, text)
             try:
-                tstamp = from_epoch(tstamp / 1000.0)
-                row = (mid, tstamp, sender, cid, chatname, text)
                 yield from _handle_row(row, db_path, locator_schema)
             except Exception as ex:
                 # TODO: also insert errors in db

src/promnesia/sources/stackexchange.py

@@ -2,12 +2,12 @@
 Uses [[https://github.com/karlicoss/HPI][HPI]] for Stackexchange data.
 '''
 
-from ..common import Results, Visit, Loc, extract_urls
+from ..common import Results, Visit, Loc
 
 
 def index() -> Results:
     from . import hpi
-    import my.stackexchange.gdpr as G # type: ignore[import] # TODO eh, not sure if should run against pypi or not...
+    import my.stackexchange.gdpr as G
     for v in G.votes():
         if isinstance(v, Exception):
             yield v

src/promnesia/sources/takeout.py

@@ -90,6 +90,8 @@ def warn_once_if_not_seen(e: Any) -> Iterable[Exception]:
                 elif e.products == ['Ads']:
                     # header contains some weird internal ad id in this case
                     context = None
+                else:
+                    context = None
                 # NOTE: at this point seems that context always ends up as None (at least for @karlicoss as of 20230131)
                 # so alternatively could just force it to be None instead of manual dispatching :shrug:
                 yield Visit(

src/promnesia/sources/takeout_legacy.py

@@ -36,7 +36,7 @@ def index() -> Results:
 from more_itertools import unique_everseen
 
 try:
-    from cachew import cachew  # type: ignore[import-not-found]
+    from cachew import cachew
 except ModuleNotFoundError as me:
     if me.name != 'cachew':
         raise me

src/promnesia/tests/common.py

@@ -54,7 +54,7 @@ def get_testdata(path: str) -> Path:
 
 @contextmanager
 def tmp_popen(*args, **kwargs):
-    import psutil  # type: ignore
+    import psutil
     with psutil.Popen(*args, **kwargs) as p:
         try:
             yield p

tests/common.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 from typing import Iterator, Optional, TypeVar
 
-import pytest # type: ignore
+import pytest
 
 
 def has_x() -> bool:
@@ -31,7 +31,7 @@ def ff(*args, **kwargs):
 from contextlib import contextmanager
 @contextmanager
 def tmp_popen(*args, **kwargs):
-    import psutil # type: ignore
+    import psutil
     with psutil.Popen(*args, **kwargs) as p:
         try:
             yield p