From f600d86cd5705715e579b15e2c2321d290c42e70 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 17 May 2024 22:16:38 +0100 Subject: [PATCH] ci: update configs and fix mypy --- .github/workflows/main.yml | 10 +++++----- mypy.ini | 10 ++++++---- ruff.toml | 2 +- src/promnesia/__main__.py | 4 ++-- src/promnesia/cannon.py | 8 ++++---- src/promnesia/common.py | 13 ++++++------- src/promnesia/compare.py | 2 +- src/promnesia/config.py | 4 ++-- src/promnesia/kjson.py | 2 +- src/promnesia/logging.py | 6 +++--- src/promnesia/sources/auto.py | 2 +- src/promnesia/sources/browser_legacy.py | 2 +- src/promnesia/sources/github.py | 4 ++-- src/promnesia/sources/org.py | 2 +- src/promnesia/sources/reddit.py | 2 +- src/promnesia/sources/signal.py | 16 +++++++++------- src/promnesia/sources/stackexchange.py | 4 ++-- src/promnesia/sources/takeout.py | 2 ++ src/promnesia/sources/takeout_legacy.py | 2 +- src/promnesia/tests/common.py | 2 +- tests/common.py | 4 ++-- tests/demos.py | 9 ++++----- tests/end2end_test.py | 14 ++++++-------- tox.ini | 2 +- 24 files changed, 65 insertions(+), 63 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d6bba59c..ae2bfc7a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,9 +29,7 @@ jobs: # windows runners are pretty scarce, so let's only run lowest and highest python version {platform: windows-latest, python-version: '3.9' }, {platform: windows-latest, python-version: '3.10'}, - # NOTE: some test packages (e.g. mouseinfo) have issues with building wheels on 3.12 windows - # perhaps try again later - {platform: windows-latest, python-version: '3.12'}, + {platform: windows-latest, python-version: '3.11'}, # same, macos is a bit too slow and ubuntu covers python quirks well {platform: macos-latest , python-version: '3.9' }, @@ -125,7 +123,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.10' - uses: actions/checkout@v4 with: @@ -155,9 +153,11 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive + fetch-depth: 0 # nicer to have all git history when debugging/for tests + - uses: actions/setup-node@v4 with: - node-version: '18' + node-version: '20' - run: extension/.ci/build --lint # debug version - run: extension/.ci/build --lint --release diff --git a/mypy.ini b/mypy.ini index 5a5c20ee..5d066a81 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,12 +1,14 @@ [mypy] +namespace_packages = True pretty = True show_error_context = True show_error_codes = True +show_column_numbers = True +show_error_end = True +warn_unused_ignores = True check_untyped_defs = True -namespace_packages = True - -[mypy-pytest] -ignore_missing_imports = True +enable_error_code = possibly-undefined +strict_equality = True # not sure why mypy started discovering it (since 0.800??) [mypy-hypothesis] diff --git a/ruff.toml b/ruff.toml index 0be93e04..54f621c6 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,4 +1,4 @@ -ignore = [ +lint.ignore = [ ### too opinionated style checks "E501", # too long lines "E702", # Multiple statements on one line (semicolon) diff --git a/src/promnesia/__main__.py b/src/promnesia/__main__.py index 645a8048..537e017a 100644 --- a/src/promnesia/__main__.py +++ b/src/promnesia/__main__.py @@ -327,14 +327,14 @@ def add_index_args(parser: argparse.ArgumentParser, default_config_path: PathIsh ) F = lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, width=120) - p = argparse.ArgumentParser(formatter_class=F) # type: ignore + p = argparse.ArgumentParser(formatter_class=F) subp = p.add_subparsers(dest='mode', ) ep = subp.add_parser('index', help='Create/update the link database', formatter_class=F) add_index_args(ep, default_config_path()) # TODO use some way to override or provide config only via cmdline? ep.add_argument('--intermediate', required=False, help="Used for development, you don't need it") - sp = subp.add_parser('serve', help='Serve a link database', formatter_class=F) # type: ignore + sp = subp.add_parser('serve', help='Serve a link database', formatter_class=F) server.setup_parser(sp) ap = subp.add_parser('demo', help='Demo mode: index and serve a directory in single command', formatter_class=F) diff --git a/src/promnesia/cannon.py b/src/promnesia/cannon.py index c0782d75..db412e69 100755 --- a/src/promnesia/cannon.py +++ b/src/promnesia/cannon.py @@ -422,7 +422,7 @@ def canonify(url: str) -> str: qq = [(k, v) for i, k, v in sorted(iqq)] # TODO still not sure what we should do.. # quote_plus replaces %20 with +, not sure if we want it... - query = urlencode(qq, quote_via=quote_via) # type: ignore[type-var] + query = urlencode(qq, quote_via=quote_via) path = _quote_path(path) @@ -683,7 +683,7 @@ def domains(it): # pragma: no cover try: nurl = canonify(url) except CanonifyException as e: - print(f"ERROR while normalising! {nurl} {e}") + print(f"ERROR while normalising! {url} {e}") c['ERROR'] += 1 continue else: @@ -718,7 +718,7 @@ def reg(url, pat): try: nurl = canonify(url) except CanonifyException as e: - print(f"ERROR while normalising! {nurl} {e}") + print(f"ERROR while normalising! {url} {e}") continue udom = nurl[:nurl.find('/')] usplit = udom.split('.') @@ -818,7 +818,7 @@ def main() -> None: # pragma: no cover - running comparison sqlite3 promnesia.sqlite 'select distinct orig_url from visits where norm_url like "%twitter%" order by orig_url' | src/promnesia/cannon.py -''', formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, width=100) # type: ignore +''', formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, width=100) ) p.add_argument('input', nargs='?') p.add_argument('--human', action='store_true') diff --git a/src/promnesia/common.py b/src/promnesia/common.py index d523dcb2..bff9492c 100644 --- a/src/promnesia/common.py +++ b/src/promnesia/common.py @@ -313,7 +313,7 @@ def _get_index_function(sourceish: PreSource) -> PreExtractor: if hasattr(sourceish, 'index'): # must be a module res = getattr(sourceish, 'index') else: - res = sourceish # type: ignore[assignment] + res = sourceish return res @@ -392,7 +392,7 @@ def appdirs(): under_test = os.environ.get('PYTEST_CURRENT_TEST') is not None # todo actually use test name? name = 'promnesia-test' if under_test else 'promnesia' - import appdirs as ad # type: ignore[import] + import appdirs as ad # type: ignore[import-untyped] return ad.AppDirs(appname=name) @@ -482,13 +482,13 @@ def fdfind_args(root: Path, follow: bool, ignore: List[str]=[]) -> List[str]: ignore_args = [] if ignore: # Add a statement that excludes the folder - ignore_args = [['--exclude', f'{n}'] for n in ignore] + _ignore_args = [['--exclude', f'{n}'] for n in ignore] # Flatten the list of lists - ignore_args_l = list(itertools.chain(*ignore_args)) + ignore_args = list(itertools.chain(*_ignore_args)) return [ *extra_fd_args(), - *ignore_args_l, + *ignore_args, *(['--follow'] if follow else []), '--type', 'f', '.', @@ -537,10 +537,9 @@ def traverse(root: Path, *, follow: bool=True, ignore: List[str]=[]) -> Iterable def get_system_zone() -> str: try: import tzlocal - # note: tzlocal mypy stubs aren't aware of api change yet (see https://github.com/python/typeshed/issues/6038) try: # 4.0 way - return tzlocal.get_localzone_name() # type: ignore[attr-defined] + return tzlocal.get_localzone_name() except AttributeError as e: # 2.0 way zone = tzlocal.get_localzone().zone # type: ignore[attr-defined] diff --git a/src/promnesia/compare.py b/src/promnesia/compare.py index ce5dd1c9..d3e0dc47 100755 --- a/src/promnesia/compare.py +++ b/src/promnesia/compare.py @@ -143,7 +143,7 @@ def compare_files(*files: Path, log=True) -> Iterator[Tuple[str, DbVisit]]: engine, table = _get_stuff(PathWithMtime.make(f)) with engine.connect() as conn: - vis = [row_to_db_visit(row) for row in conn.execute(table.select())] # type: ignore[var-annotated] + vis = [row_to_db_visit(row) for row in conn.execute(table.select())] if last is not None: between = f'{last_dts}:{this_dts}' diff --git a/src/promnesia/config.py b/src/promnesia/config.py index 9e4903e0..7bc51ec1 100644 --- a/src/promnesia/config.py +++ b/src/promnesia/config.py @@ -6,7 +6,7 @@ import importlib.util import warnings -from .common import PathIsh, get_tmpdir, appdirs, default_output_dir, default_cache_dir, user_config_file +from .common import PathIsh, default_output_dir, default_cache_dir from .common import Res, Source, DbVisit @@ -129,7 +129,7 @@ def import_config(config_file: PathIsh) -> Config: spec = importlib.util.spec_from_file_location(name, p); assert spec is not None mod = importlib.util.module_from_spec(spec); assert mod is not None loader = spec.loader; assert loader is not None - loader.exec_module(mod) # type: ignore[attr-defined] + loader.exec_module(mod) d = {} for f in Config._fields: diff --git a/src/promnesia/kjson.py b/src/promnesia/kjson.py index c3318a3a..95848c26 100644 --- a/src/promnesia/kjson.py +++ b/src/promnesia/kjson.py @@ -74,7 +74,7 @@ def test_json_processor(): handled = [] class Proc(JsonProcessor): def handle_dict(self, value: JDict, path): - if 'skipme' in self.kpath(path): + if 'skipme' in self.kpath(path): # type: ignore[comparison-overlap] return JsonProcessor.SKIP def handle_str(self, value: str, path): diff --git a/src/promnesia/logging.py b/src/promnesia/logging.py index c4fadb4a..c8770f1b 100755 --- a/src/promnesia/logging.py +++ b/src/promnesia/logging.py @@ -61,7 +61,7 @@ def mklevel(level: LevelIsh) -> Level: def setup_logger(logger: logging.Logger, level: LevelIsh) -> None: lvl = mklevel(level) try: - import logzero # type: ignore[import] + import logzero # type: ignore[import-not-found] formatter = logzero.LogFormatter( fmt=FORMAT_COLOR, datefmt=DATEFMT, @@ -75,7 +75,7 @@ def setup_logger(logger: logging.Logger, level: LevelIsh) -> None: logger.addFilter(AddExceptionTraceback()) if use_logzero and not COLLAPSE_DEBUG_LOGS: # all set, nothing to do # 'simple' setup - logzero.setup_logger(logger.name, level=lvl, formatter=formatter) + logzero.setup_logger(logger.name, level=lvl, formatter=formatter) # type: ignore[possibly-undefined] return h = CollapseDebugHandler() if COLLAPSE_DEBUG_LOGS else logging.StreamHandler() @@ -101,7 +101,7 @@ def isEnabledFor_lazyinit(*args, logger=logger, orig=logger.isEnabledFor, **kwar # oh god.. otherwise might go into an inf loop if not hasattr(logger, _init_done): setattr(logger, _init_done, False) # will setup on the first call - logger.isEnabledFor = isEnabledFor_lazyinit # type: ignore[assignment] + logger.isEnabledFor = isEnabledFor_lazyinit # type: ignore[method-assign] return cast(LazyLogger, logger) diff --git a/src/promnesia/sources/auto.py b/src/promnesia/sources/auto.py index 8f00870a..c3cb23ca 100644 --- a/src/promnesia/sources/auto.py +++ b/src/promnesia/sources/auto.py @@ -320,7 +320,7 @@ def _index_file(pp: Path, opts: Options) -> Results: def indexer() -> Union[Urls, Results]: # eh, annoying.. need to make more generic.. - idx = ip(pp) # type: ignore + idx = ip(pp) try: yield from idx except Exception as e: diff --git a/src/promnesia/sources/browser_legacy.py b/src/promnesia/sources/browser_legacy.py index 1b343bba..76f26cfb 100644 --- a/src/promnesia/sources/browser_legacy.py +++ b/src/promnesia/sources/browser_legacy.py @@ -10,7 +10,7 @@ from .. import config try: - from cachew import cachew # type: ignore[import-not-found] + from cachew import cachew except ModuleNotFoundError as me: if me.name != 'cachew': raise me diff --git a/src/promnesia/sources/github.py b/src/promnesia/sources/github.py index 8d77ebb2..1ba21d94 100644 --- a/src/promnesia/sources/github.py +++ b/src/promnesia/sources/github.py @@ -31,7 +31,7 @@ def index(*, render_markdown: bool = False) -> Results: # if enabled, convert the (markdown) body to HTML context: Optional[str] = e.body if e.body is not None and render_markdown: - context = TextParser(e.body)._doc_ashtml() + context = TextParser(e.body)._doc_ashtml() # type: ignore[possibly-undefined] # locator should link back to this event loc = Loc.make(title=e.summary, href=e.link) @@ -74,7 +74,7 @@ def index(*, render_markdown: bool = False) -> Results: # extract from markdown links like [link text](https://...) # incase URLExtract missed any somehow if render_markdown: - for res in extract_from_text(e.body): + for res in extract_from_text(e.body): # type: ignore[possibly-undefined] if isinstance(res, Exception): yield res continue diff --git a/src/promnesia/sources/org.py b/src/promnesia/sources/org.py index 74bd5d4a..c9205df6 100644 --- a/src/promnesia/sources/org.py +++ b/src/promnesia/sources/org.py @@ -84,7 +84,7 @@ def walk_node(*, node: OrgNode, dt: datetime) -> Iterator[Res[Tuple[Parsed, OrgN parsed = parsed._replace(dt=dt) else: dt = parsed.dt - yield parsed, node + yield parsed, node for c in node.children: yield from walk_node(node=c, dt=dt) diff --git a/src/promnesia/sources/reddit.py b/src/promnesia/sources/reddit.py index af07179e..20be5ecc 100644 --- a/src/promnesia/sources/reddit.py +++ b/src/promnesia/sources/reddit.py @@ -16,7 +16,7 @@ def index(*, render_markdown: bool = False, renderer: Optional[Type['RedditRende if "No module named 'my.reddit.all'" in str(e): import warnings warnings.warn("DEPRECATED/reddit: Using an old version of HPI, please update") - from my.reddit import submissions, comments, saved, upvoted # type: ignore[no-redef] + from my.reddit import submissions, comments, saved, upvoted else: raise e diff --git a/src/promnesia/sources/signal.py b/src/promnesia/sources/signal.py index 1464dc84..daafd0f7 100644 --- a/src/promnesia/sources/signal.py +++ b/src/promnesia/sources/signal.py @@ -63,6 +63,8 @@ def index( logger.debug("Paths to harvest: %s", db_paths) if not http_only: sql_query = f"{messages_query}\nWHERE body LIKE '%http%'" + else: + sql_query = messages_query for db_path in resolved_db_paths: logger.info("Ciphered db to harvest %s", db_path) @@ -188,8 +190,8 @@ def _expand_path(path_pattern: PathIsh) -> Iterable[Path]: def _expand_paths(paths: PathIshes) -> Iterable[Path]: if _is_pathish(paths): - paths = [paths] # type: ignore[assignment,list-item] - return [pp.resolve() for p in paths for pp in _expand_path(p)] # type: ignore[union-attr,list-item] + paths = [paths] # type: ignore[list-item] + return [pp.resolve() for p in paths for pp in _expand_path(p)] # type: ignore[union-attr] def collect_db_paths(*db_paths: PathIsh, append: bool = False) -> Iterable[Path]: @@ -236,7 +238,7 @@ def collect_db_paths(*db_paths: PathIsh, append: bool = False) -> Iterable[Path] ) if db_paths and append: - db_paths = [ # type: ignore[misc,assignment] + db_paths = [ # type: ignore[assignment] *([db_paths] if _is_pathish(db_paths) else db_paths), plat_paths, ] @@ -320,7 +322,7 @@ def connect_db( "Decrypting db '%s' with cmd: %s << Results: from . import hpi - import my.stackexchange.gdpr as G # type: ignore[import] # TODO eh, not sure if should run against pypi or not... + import my.stackexchange.gdpr as G for v in G.votes(): if isinstance(v, Exception): yield v diff --git a/src/promnesia/sources/takeout.py b/src/promnesia/sources/takeout.py index b4b24ebd..8beb5a16 100644 --- a/src/promnesia/sources/takeout.py +++ b/src/promnesia/sources/takeout.py @@ -90,6 +90,8 @@ def warn_once_if_not_seen(e: Any) -> Iterable[Exception]: elif e.products == ['Ads']: # header contains some weird internal ad id in this case context = None + else: + context = None # NOTE: at this point seems that context always ends up as None (at least for @karlicoss as of 20230131) # so alternatively could just force it to be None instead of manual dispatching :shrug: yield Visit( diff --git a/src/promnesia/sources/takeout_legacy.py b/src/promnesia/sources/takeout_legacy.py index 2bf7ec85..66c18d39 100644 --- a/src/promnesia/sources/takeout_legacy.py +++ b/src/promnesia/sources/takeout_legacy.py @@ -36,7 +36,7 @@ def index() -> Results: from more_itertools import unique_everseen try: - from cachew import cachew # type: ignore[import-not-found] + from cachew import cachew except ModuleNotFoundError as me: if me.name != 'cachew': raise me diff --git a/src/promnesia/tests/common.py b/src/promnesia/tests/common.py index 6ecdb8b5..0ea69a8b 100644 --- a/src/promnesia/tests/common.py +++ b/src/promnesia/tests/common.py @@ -54,7 +54,7 @@ def get_testdata(path: str) -> Path: @contextmanager def tmp_popen(*args, **kwargs): - import psutil # type: ignore + import psutil with psutil.Popen(*args, **kwargs) as p: try: yield p diff --git a/tests/common.py b/tests/common.py index de157f26..47947746 100644 --- a/tests/common.py +++ b/tests/common.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Iterator, Optional, TypeVar -import pytest # type: ignore +import pytest def has_x() -> bool: @@ -31,7 +31,7 @@ def ff(*args, **kwargs): from contextlib import contextmanager @contextmanager def tmp_popen(*args, **kwargs): - import psutil # type: ignore + import psutil with psutil.Popen(*args, **kwargs) as p: try: yield p diff --git a/tests/demos.py b/tests/demos.py index 5bd7ccb6..b59c3add 100644 --- a/tests/demos.py +++ b/tests/demos.py @@ -16,8 +16,7 @@ def real_db(): - from private import real_db_path, test_filter # type: ignore[import,attr-defined] - from tempfile import TemporaryDirectory + from private import real_db_path, test_filter # type: ignore[import-not-found] import shutil def indexer(tdir: Path): tdb = tdir / 'promnesia.sqlite' @@ -50,7 +49,7 @@ def annotate(self, text: str, length=2) -> None: self.l.append((now, text, length)) def build(self, **extra): - from pysubs2 import SSAFile, SSAEvent, Color # type: ignore[import] + from pysubs2 import SSAFile, SSAEvent, Color # type: ignore[import-not-found] millis = lambda td: td / timedelta(milliseconds=1) subs = ( SSAEvent( @@ -445,7 +444,7 @@ def before(driver): wait(8) -from selenium import webdriver # type: ignore +from selenium import webdriver def scroll_to_text(driver, text: str): @@ -473,7 +472,7 @@ def test_demo_highlights(tmp_path, browser): def before(driver): - from private import instapaper_cookies # type: ignore[import,attr-defined] + from private import instapaper_cookies # necessary to set cookies on instapaper.. driver.get('http://instapaper.com') diff --git a/tests/end2end_test.py b/tests/end2end_test.py index 889c58b7..d5bd5cc9 100755 --- a/tests/end2end_test.py +++ b/tests/end2end_test.py @@ -326,7 +326,7 @@ def send_key(key) -> None: key = key.split('+') print(f"sending hotkey! {key}") - import pyautogui # type: ignore + import pyautogui pyautogui.hotkey(*key) @@ -477,7 +477,7 @@ def contents() -> str: # count += 100 # just in case count = 3000 # meh # focus ends up at some random position, so need both backspace and delete - area.send_keys([Keys.BACKSPACE] * count + [Keys.DELETE] * count) + area.send_keys(*([Keys.BACKSPACE] * count + [Keys.DELETE] * count)) assert contents() == '' area.send_keys(settings) @@ -515,8 +515,7 @@ def open_search_page(self, query: str="") -> None: ) def move_to(self, element) -> None: - # remove type ignore later https://github.com/SeleniumHQ/selenium/pull/12477 - ActionChains(self.driver).move_to_element(element).perform() # type: ignore[arg-type] + ActionChains(self.driver).move_to_element(element).perform() def switch_to_sidebar(self, wait: Union[bool, int]=False, *, wait2: bool=True) -> None: raise RuntimeError('not used anymore, use with helper.sidebar instead!') @@ -573,7 +572,7 @@ def confirm(what: str) -> None: Headless().confirm(what) return - import click # type: ignore + import click click.confirm(click.style(what, blink=True, fg='yellow'), abort=True) @@ -803,12 +802,11 @@ def test_add_to_blacklist_context_menu(tmp_path: Path, browser: Browser) -> None with get_webdriver(browser=browser) as driver: configure_extension(driver, port='12345') driver.get('https://example.com') - # remove type ignore later https://github.com/SeleniumHQ/selenium/pull/12477 - chain = webdriver.ActionChains(driver) # type: ignore[arg-type] + chain = webdriver.ActionChains(driver) chain.move_to_element(driver.find_element(By.TAG_NAME, 'h1')).context_click().perform() # looks like selenium can't interact with browser context menu... - import pyautogui # type: ignore + import pyautogui if driver.name == 'chrome': offset = 2 # Inspect, View page source diff --git a/tox.ini b/tox.ini index f0c129b8..63091f6f 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ passenv = [testenv:ruff] commands = {envpython} -m pip install --use-pep517 -e .[testing] - {envpython} -m ruff src/ + {envpython} -m ruff check src/ [testenv:tests]