rebase onto master

CHANGELOG.md

@@ -1,5 +1,5 @@
 # **Upcoming release**
-
+- #516 Autoimport Now automatically detects project dependencies and can read TOML configuration
 - #733 skip directories with perm error when building autoimport index (@MrBago)
 - #722, #723 Remove site-packages from packages search tree (@tkrabel)
 - #738 Implement os.PathLike on Resource (@lieryan)

docs/configuration.rst

@@ -14,6 +14,8 @@ Will be used if [tool.rope] is configured.
 
     [tool.rope]
     split_imports = true
+    [tool.rope.autoimport]
+    underlined = false
 
 config.py
 ---------
@@ -56,9 +58,14 @@ It follows the exact same syntax of the pyproject.toml.
 
 
 Options
--------
+=======
 .. autopytoolconfigtable:: rope.base.prefs.Prefs
 
+Autoimport Options
+------------------
+.. autopytoolconfigtable:: rope.base.prefs.AutoimportPrefs
+
+
 Old Configuration File
 ----------------------
 This is a sample config.py. While this config.py works and all options here should be supported, the above documentation reflects the latest version of rope.

rope/base/prefs.py

@@ -12,6 +12,14 @@
 from rope.base.resources import Folder
 
 
+@dataclass
+class AutoimportPrefs:
+    underlined: bool = field(
+        default=False, description="Cache underlined (private) modules")
+    memory: bool = field(default=None, description="Cache in memory instead of disk")
+    parallel: bool = field(default=True, description="Use multiple processes to parse")
+
+
 @dataclass
 class Prefs:
     """Class to store rope preferences."""
@@ -206,6 +214,8 @@ class Prefs:
             Can only be set in config.py.
         """),
     )
+    autoimport: AutoimportPrefs = field(
+        default_factory=lambda: AutoimportPrefs(), description="Preferences for Autoimport")
 
     def set(self, key: str, value: Any):
         """Set the value of `key` preference to `value`."""

rope/base/versioning.py

@@ -1,3 +1,4 @@
+import dataclasses
 import hashlib
 import importlib.util
 import json
@@ -31,7 +32,9 @@ def _get_prefs_data(project) -> str:
     del prefs_data["project_opened"]
     del prefs_data["callbacks"]
     del prefs_data["dependencies"]
-    return json.dumps(prefs_data, sort_keys=True, indent=2)
+    return json.dumps(
+        prefs_data, sort_keys=True, indent=2, default=lambda o: o.__dict__
+    )
 
 
 def _get_file_content(module_name: str) -> str:

rope/contrib/autoimport/sqlite.py

@@ -16,7 +16,10 @@
 from threading import local
 from typing import Generator, Iterable, Iterator, List, Optional, Set, Tuple
 
+from packaging.requirements import Requirement
+
 from rope.base import exceptions, libutils, resourceobserver, taskhandle, versioning
+from rope.base.prefs import AutoimportPrefs
 from rope.base.project import Project
 from rope.base.resources import Resource
 from rope.contrib.autoimport import models
@@ -53,18 +56,36 @@ def get_future_names(
 
 
 def filter_packages(
-    packages: Iterable[Package], underlined: bool, existing: List[str]
+    packages: Iterable[Package],
+    underlined: bool,
+    existing: List[str],
+    dependencies: Optional[List[Requirement]],
 ) -> Iterable[Package]:
     """Filter list of packages to parse."""
+    parsed_deps = (
+        [dep.name for dep in dependencies] if dependencies is not None else None
+    )
+
+    def is_dep(package) -> bool:
+        return (
+            parsed_deps is None
+            or package.name in parsed_deps
+            or package.source is not Source.SITE_PACKAGE
+        )
+
     if underlined:
 
         def filter_package(package: Package) -> bool:
-            return package.name not in existing
+            return package.name not in existing and is_dep(package)
 
     else:
 
         def filter_package(package: Package) -> bool:
-            return package.name not in existing and not package.name.startswith("_")
+            return (
+                package.name not in existing
+                and not package.name.startswith("_")
+                and is_dep(package)
+            )
 
     return filter(filter_package, packages)
 
@@ -81,16 +102,15 @@ class AutoImport:
     """
 
     connection: sqlite3.Connection
-    memory: bool
     project: Project
     project_package: Package
-    underlined: bool
+    prefs: AutoimportPrefs
 
     def __init__(
         self,
         project: Project,
         observe: bool = True,
-        underlined: bool = False,
+        underlined: Optional[bool] = None,
         memory: bool = _deprecated_default,
     ):
         """Construct an AutoImport object.
@@ -113,25 +133,29 @@ def __init__(
                 autoimport = AutoImport(..., memory=True)
         """
         self.project = project
+        self.prefs = self.project.prefs.autoimport
         project_package = get_package_tuple(project.root.pathlib, project)
         assert project_package is not None
         assert project_package.path is not None
+        if underlined is not None:
+            self.prefs.underlined = underlined
         self.project_package = project_package
-        self.underlined = underlined
-        self.memory = memory
         if memory is _deprecated_default:
-            self.memory = True
-            warnings.warn(
-                "The default value for `AutoImport(memory)` argument will "
-                "change to use an on-disk database by default in the future. "
-                "If you want to use an in-memory database, you need to pass "
-                "`AutoImport(memory=True)` explicitly.",
-                DeprecationWarning,
-            )
+            if self.prefs.memory is None:
+                self.prefs.memory = True
+                warnings.warn(
+                    "The default value for `AutoImport(memory)` argument will "
+                    "change to use an on-disk database by default in the future. "
+                    "If you want to use an in-memory database, you need to pass "
+                    "`AutoImport(memory=True)` explicitly or set it in the config file.",
+                    DeprecationWarning,
+                )
+        else:
+            self.prefs.memory = memory
         self.thread_local = local()
         self.connection = self.create_database_connection(
             project=project,
-            memory=memory,
+            memory=self.prefs.memory,
         )
         self._setup_db()
         if observe:
@@ -389,12 +413,13 @@ def generate_modules_cache(
         """
         Generate global name cache for external modules listed in `modules`.
 
-        If no modules are provided, it will generate a cache for every module available.
+        If modules is not specified, uses PEP 621 metadata.
+        If modules aren't specified and PEP 621 is not present, caches every package
         This method searches in your sys.path and configured python folders.
         Do not use this for generating your own project's internal names,
         use generate_resource_cache for that instead.
         """
-        underlined = self.underlined if underlined is None else underlined
+        underlined = self.prefs.underlined if underlined is None else underlined
 
         packages: List[Package] = (
             self._get_available_packages()
@@ -403,12 +428,16 @@ def generate_modules_cache(
         )
 
         existing = self._get_packages_from_cache()
-        packages = list(filter_packages(packages, underlined, existing))
-        if not packages:
+        packages = list(
+            filter_packages(
+                packages, underlined, existing, self.project.prefs.dependencies
+            )
+        )
+        if len(packages) == 0:
             return
         self._add_packages(packages)
         job_set = task_handle.create_jobset("Generating autoimport cache", 0)
-        if single_thread:
+        if single_thread or not self.prefs.parallel:
             for package in packages:
                 for module in get_files(package, underlined):
                     job_set.started_job(module.modname)
@@ -512,7 +541,7 @@ def update_resource(
         self, resource: Resource, underlined: bool = False, commit: bool = True
     ):
         """Update the cache for global names in `resource`."""
-        underlined = underlined if underlined else self.underlined
+        underlined = underlined if underlined else self.prefs.underlined
         module = self._resource_to_module(resource, underlined)
         self._del_if_exist(module_name=module.modname, commit=False)
         for name in get_names(module, self.project_package):
@@ -537,7 +566,11 @@ def _del_if_exist(self, module_name, commit: bool = True):
 
     def _get_python_folders(self) -> List[Path]:
         def filter_folders(folder: Path) -> bool:
-            return folder.is_dir() and folder.as_posix() != "/usr/bin"
+            return (
+                folder.is_dir()
+                and folder.as_posix() != "/usr/bin"
+                and str(folder) != self.project.address
+            )
 
         folders = self.project.get_python_path_folders()
         folder_paths = filter(filter_folders, map(Path, folders))
@@ -623,7 +656,7 @@ def _resource_to_module(
         self, resource: Resource, underlined: bool = False
     ) -> ModuleFile:
         assert self.project_package.path
-        underlined = underlined if underlined else self.underlined
+        underlined = underlined if underlined else self.prefs.underlined
         resource_path: Path = resource.pathlib
         # The project doesn't need its name added to the path,
         # since the standard python file layout accounts for that

rope/contrib/autoimport/utils.py

@@ -118,7 +118,7 @@ def get_files(
         yield ModuleFile(package.path, package.path.stem, underlined, False)
     else:
         assert package.path
-        for file in package.path.glob("**/*.py"):
+        for file in package.path.rglob("*.py"):
             if file.name == "__init__.py":
                 yield ModuleFile(
                     file,

ropetest/contrib/autoimport/deptest.py

@@ -0,0 +1,54 @@
+from pathlib import Path
+from typing import Iterable
+
+import pytest
+
+from rope.base.project import Project
+from rope.contrib.autoimport.sqlite import AutoImport
+
+
+@pytest.fixture
+def project(request, tmp_path: Path) -> Iterable[Project]:
+    doc = request.param
+    if doc is not None:
+        file = tmp_path / "pyproject.toml"
+        file.write_text(doc, encoding="utf-8")
+        print(file, doc)
+    project = Project(tmp_path)
+    yield project
+    project.close()
+
+
+@pytest.fixture
+def autoimport(project) -> Iterable[AutoImport]:
+    autoimport = AutoImport(project, memory=True)
+    autoimport.generate_modules_cache()
+    yield autoimport
+    autoimport.close()
+
+
+@pytest.mark.parametrize("project", ((""),), indirect=True)
+def test_blank(project, autoimport):
+    assert project.prefs.dependencies is None
+    assert autoimport.search("pytoolconfig")
+
+
+@pytest.mark.parametrize("project", (("[project]\n dependencies=[]"),), indirect=True)
+def test_empty(project, autoimport):
+    assert len(project.prefs.dependencies) == 0
+    assert [] == autoimport.search("pytoolconfig")
+
+
+FILE = """
+[project]
+dependencies = [
+    "pytoolconfig",
+    "bogus"
+]
+"""
+
+
+@pytest.mark.parametrize("project", ((FILE),), indirect=True)
+def test_not_empty(project, autoimport):
+    assert len(project.prefs.dependencies) == 2
+    assert autoimport.search("pytoolconfig")