fix(modules): name a module in a directory without __init__.py after its package

CHANGELOG.md

@@ -4,7 +4,18 @@
 
 ### Compatibility and migration
 
-- The cache format changes (18): caches built by an earlier version miss once.
+- The cache format changes (19): caches built by an earlier version miss once.
+- A module in a directory without `__init__.py` inside a package is named after that
+  package, as Python names it (#264): `pkg/sub/app.py` is `pkg.sub.app`. Its relative
+  imports now resolve, and a file that was a `syntax-error` (`relative import beyond
+  top-level package`) is analysed, flask's `sansio` for one. The project symbols of its
+  functions change with its name (`python.pkg.sub.app.view`), which a model declared on
+  the former symbol must follow; baseline keys do not change, since they hold no module
+  name. A script beside it still imports it by its short name, which answers no import
+  from another directory: at the project root, `import os` is the library even beside a
+  `pkg/sub/os.py`, which a top-level module of a directory took before. Files that share
+  only that short name are no longer noted as `ambiguous-module`, nor is an import that
+  only such names from other directories match.
 - SARIF: each rule carries a title (`shortDescription`) and a description (`fullDescription`)
   instead of repeating its id (#270), and each result carries its confidence as the
   `confidence` property (#271). A flow reaching its sink inside a project function points at

docs/usage.md

@@ -67,9 +67,16 @@ coretrace-python-analyzer [--check | --emit-ir [--ssa]] [options] [path]
 ## What is analysed
 
 Given a directory, every `.py` file below it is a module of one project, named after
-its package (`app/views.py` is `app.views`). Two files that would share a name, `a/app.py`
-and `z/app.py` outside any package, are told apart by their path from the root, `a.app`
-and `z.app`, and reported: each carries an `ambiguous-module` note, since a symbol
+its package (`app/views.py` is `app.views`). A directory without `__init__.py` inside a
+package is a portion of it, as Python reads one: `pkg/sub/app.py` is `pkg.sub.app`, and
+its relative imports resolve against `pkg`. A module there can also be imported by the
+name a script run from its directory gives it, `app`: an absolute import from that
+directory reaches what lies beside it first, then what the package's root holds, and
+files sharing only such a name are told apart by the importer's directory. From another
+directory that name answers nothing, as Python finds no such module there: at the
+project root, `import os` is the library even beside a `pkg/sub/os.py`. Two files
+that would share a module name, `a/app.py` and `z/app.py` outside any package, are told
+apart by their path from the root, `a.app` and `z.app`, and reported: each carries an `ambiguous-module` note, since a symbol
 `python.app.main` may name a function of either and an import of `app` resolves only
 from the importer's own directory, and each is covered as `ambiguous` rather than
 analysed. An import of such a name that no file of the importer's own directory answers

src/coretrace_python/cache.py

@@ -44,8 +44,9 @@
 
 # Cached summaries from earlier versions preserve unstable external-call
 # insertion order. They must be recomputed with canonical ordering (#261).
-# Findings carry their related locations from format 18 on (#271).
-CACHE_FORMAT = 18
+# Findings carry their related locations from format 18 on (#271). A module's key holds
+# the names it is imported by from format 19 on (#264).
+CACHE_FORMAT = 19
 
 
 @dataclass(frozen=True)

src/coretrace_python/interprocedural/__init__.py

@@ -20,6 +20,7 @@
     build_module_graph,
     discover_files,
     discover_sources,
+    import_names,
     name_modules,
     project_symbol,
 )
@@ -77,6 +78,7 @@
     "cleared_by",
     "discover_files",
     "discover_sources",
+    "import_names",
     "name_modules",
     "project_symbol",
     "starting_summaries",

src/coretrace_python/interprocedural/modulegraph.py

@@ -59,7 +59,7 @@ def discover_sources(
     found: list[SourceFile] = []
     for path in discover_files(root, "*.py"):
         try:
-            found.append(manager.load_file(path))
+            found.append(manager.load_file(path, root))
         except (OSError, UnicodeDecodeError) as error:
             if errors is None:
                 raise
@@ -120,17 +120,12 @@ def _dotted(path: Path) -> str:
     return ".".join(parts) or path.as_posix()
 
 
-def _import_root(source: SourceFile) -> Path | None:
-    """The directory an import of ``source`` by its module name starts from: the parent
-    of its top-level package, or its own directory for a module outside any package,
-    where ``module_name_for`` stopped. ``None`` for a source not on disk."""
+def import_names(source: SourceFile) -> tuple[tuple[str, Path | None], ...]:
+    """The names ``source`` can be imported by, its module name first, each with the
+    directory an import of it starts from (``SourceFile.import_names``); for a source
+    not on disk, its module name from nowhere in particular."""
 
-    if source.path is None:
-        return None
-    directory = source.path.parent
-    while (directory / "__init__.py").is_file():
-        directory = directory.parent
-    return directory
+    return tuple((n.name, n.root) for n in source.import_names) or ((source.module_name, None),)
 
 
 def project_symbol(module_name: str, qualified_name: str) -> SymbolId:
@@ -157,6 +152,7 @@ class ModuleGraph:
     _sources: Mapping[str, SourceFile]
     _imports: Mapping[str, frozenset[str]]
     _unresolved: Mapping[str, frozenset[str]] = field(default_factory=dict)
+    _resolved: Mapping[str, Mapping[str, str]] = field(default_factory=dict)
 
     @property
     def modules(self) -> tuple[str, ...]:
@@ -177,6 +173,12 @@ def unresolved(self, name: str) -> frozenset[str]:
 
         return self._unresolved.get(name, frozenset())
 
+    def resolved(self, name: str) -> Mapping[str, str]:
+        """The project modules ``name`` imports, each by the import name its code gives
+        it, which may be another than the module's name in the graph."""
+
+        return self._resolved.get(name, MappingProxyType({}))
+
     def schedule(self) -> tuple[tuple[frozenset[str], ...], ...]:
         """Strongly connected components in waves: every component of a wave imports,
         outside itself, only components of earlier waves, so one wave can be analysed
@@ -241,16 +243,24 @@ def build_module_graph(
     imports: Mapping[str, ImportTable],
 ) -> ModuleGraph:
     """Edges from each module to the project modules it imports, in any scope. An import
-    names a module by its import name; when several files have that name, it resolves to
-    the one in the importer's own root and, without exactly one there, stays unresolved:
-    no edge, and the name is recorded on the graph."""
-
-    by_name: dict[str, list[str]] = {}
+    names a module by one of its import names; when several files have that name, it
+    resolves to the one from the first of the importer's own roots that has exactly one,
+    and otherwise stays unresolved: no edge, and the name is recorded on the graph. The
+    importer's roots are those of its import names, the last first: as a script run from
+    its directory finds what lies beside it before what its package's root holds. A name
+    a module has only from its own directory, not its module name, answers only an
+    importer with that directory among its roots, however few files have it: from
+    elsewhere, ``import os`` is the library even when ``pkg/sub/os.py`` is ``os`` to a
+    script beside it."""
+
+    by_name: dict[str, list[tuple[str, Path | None, bool]]] = {}
     for name, source in sources.items():
-        by_name.setdefault(source.module_name, []).append(name)
-    roots = {name: _import_root(source) for name, source in sources.items()}
+        for index, (import_name, root) in enumerate(import_names(source)):
+            by_name.setdefault(import_name, []).append((name, root, index == 0))
+    roots = {name: tuple(root for _, root in reversed(import_names(source))) for name, source in sources.items()}
     edges: dict[str, frozenset[str]] = {}
     unresolved: dict[str, frozenset[str]] = {}
+    resolved: dict[str, Mapping[str, str]] = {}
     for name, module in modules.items():
         candidates: set[str] = set()
         dotted_tops: set[str] = set()
@@ -267,24 +277,39 @@ def build_module_graph(
                 candidates.add(path)
         # The most specific project module each import names; ``app.helpers`` implies
         # the ``app`` package, which is not an edge worth recording.
-        found: set[str] = set()
+        found: dict[str, str] = {}
         missing: set[str] = set()
         for candidate in candidates:
             for prefix in _prefixes(candidate):
-                matching = by_name.get(prefix)
-                if matching is None:
+                matching = [(m, r) for m, r, primary in by_name.get(prefix, ()) if primary or r in roots[name]]
+                if not matching:
                     continue
-                if len(matching) > 1:
-                    matching = [m for m in matching if roots[m] == roots[name]]
-                if len(matching) == 1:
-                    found.add(matching[0])
-                else:
+                chosen = _chosen(matching, roots[name])
+                if chosen is None:
                     missing.add(prefix)
+                elif chosen != name:
+                    found[prefix] = chosen
                 break
-        edges[name] = frozenset(found - {name})
+        edges[name] = frozenset(found.values())
+        resolved[name] = MappingProxyType(found)
         if missing:
             unresolved[name] = frozenset(missing)
-    return ModuleGraph(MappingProxyType(dict(sources)), MappingProxyType(edges), MappingProxyType(unresolved))
+    return ModuleGraph(
+        MappingProxyType(dict(sources)), MappingProxyType(edges), MappingProxyType(unresolved), MappingProxyType(resolved)
+    )
+
+
+def _chosen(matching: list[tuple[str, Path | None]], roots: tuple[Path | None, ...]) -> str | None:
+    """The module an import name names among the files that have it, from the first of
+    the importer's ``roots`` holding exactly one; ``None`` when none does."""
+
+    if len({module for module, _ in matching}) == 1:
+        return matching[0][0]
+    for root in roots:
+        there = {module for module, named_from in matching if named_from == root}
+        if there:
+            return there.pop() if len(there) == 1 else None
+    return None
 
 
 def _prefixes(dotted: str) -> list[str]:
@@ -310,6 +335,7 @@ def _statements(body: Iterable[nodes.Statement]) -> Iterable[nodes.Statement]:
     "SummaryIndex",
     "build_module_graph",
     "discover_sources",
+    "import_names",
     "name_modules",
     "project_symbol",
 ]

src/coretrace_python/source/__init__.py

@@ -1,11 +1,19 @@
 """Source storage and location primitives."""
 
 from coretrace_python.source.manager import LINE_BREAK, SourceManager, decode_text, lines_of
-from coretrace_python.source.model import FileLocation, Location, SourceFile, SourceId, SourceSpan
+from coretrace_python.source.model import (
+    FileLocation,
+    ImportName,
+    Location,
+    SourceFile,
+    SourceId,
+    SourceSpan,
+)
 
 __all__ = [
     "LINE_BREAK",
     "FileLocation",
+    "ImportName",
     "Location",
     "SourceFile",
     "SourceId",

src/coretrace_python/source/manager.py

@@ -5,20 +5,60 @@
 import re
 from pathlib import Path
 
-from coretrace_python.source.model import SourceFile, SourceId
+from coretrace_python.source.model import ImportName, SourceFile, SourceId
 
 
-def module_name_for(path: Path) -> str:
-    """Dotted module name of ``path``, walking up through ``__init__.py`` packages.
+def import_names_for(path: Path, root: Path | None = None) -> tuple[ImportName, ...]:
+    """The names the file ``path`` can be imported by, its module name first, each with
+    the directory an import of it starts from. A package's ``__init__.py`` is named after
+    the package itself.
 
-    A package's ``__init__.py`` is named after the package itself."""
+    The module name follows the packages the file is in, as Python takes a directory
+    without ``__init__.py`` inside a regular package for a portion of it (PEP 420): it
+    runs from the parent of the outermost directory holding an ``__init__.py``, searched
+    up through directories without one as far as ``root``, the project's, and above it
+    while each directory holds one (#264). In no package, or without a root, it runs from
+    the first directory without an ``__init__.py`` above the file. That second way is how
+    a script run from that directory imports the file (``sys.path[0]``): when it gives
+    another name, the file has that name too."""
 
     parts = [] if path.name == "__init__.py" else [path.stem]
     directory = path.parent
     while (directory / "__init__.py").is_file():
         parts.append(directory.name)
         directory = directory.parent
-    return ".".join(reversed(parts)) or path.stem
+    script = ImportName(".".join(reversed(parts)) or path.stem, directory)
+    top = None if root is None else _outermost_package(path.parent, root)
+    if top is None or top.parent == script.root:
+        return (script,)
+    named = path.relative_to(top.parent).with_suffix("").parts
+    return (ImportName(".".join(named[:-1] if named[-1] == "__init__" else named), top.parent), script)
+
+
+def _outermost_package(directory: Path, root: Path) -> Path | None:
+    """The outermost directory holding an ``__init__.py`` among ``directory`` and those
+    above it up to ``root``, and above ``root`` while each holds one; ``None`` when none
+    does, or ``directory`` is not under ``root``."""
+
+    if directory != root and root not in directory.parents:
+        return None
+    top = None
+    while True:
+        if (directory / "__init__.py").is_file():
+            top = directory
+        if directory == root:
+            break
+        directory = directory.parent
+    if top == root:
+        while (top.parent / "__init__.py").is_file():
+            top = top.parent
+    return top
+
+
+def module_name_for(path: Path, root: Path | None = None) -> str:
+    """Dotted module name of ``path`` (``import_names_for``)."""
+
+    return import_names_for(path, root)[0].name
 
 
 def decode_text(data: bytes) -> str:
@@ -68,20 +108,26 @@ def add_source(self, name: str, text: str, module_name: str | None = None) -> So
         self._sources[source_id] = source
         return source
 
-    def load_file(self, path: str | Path) -> SourceFile:
+    def load_file(self, path: str | Path, root: Path | None = None) -> SourceFile:
+        """The file at ``path``, named as a module of the project under ``root``
+        (``import_names_for``); without a root, from the first directory without an
+        ``__init__.py`` above it."""
+
         resolved_path = Path(path).resolve()
         source_id = SourceId(str(resolved_path))
         existing = self._sources.get(source_id)
         if existing is not None:
             return existing
 
         text = decode_text(resolved_path.read_bytes())
+        names = import_names_for(resolved_path, None if root is None else root.resolve())
         source = SourceFile(
             source_id=source_id,
             text=text,
-            module_name=module_name_for(resolved_path),
+            module_name=names[0].name,
             path=resolved_path,
             is_package=resolved_path.name == "__init__.py",
+            import_names=names,
         )
         self._sources[source_id] = source
         return source

src/coretrace_python/source/model.py

@@ -65,13 +65,24 @@ def display(self) -> str:
 Location = SourceSpan | FileLocation
 
 
+@dataclass(frozen=True)
+class ImportName:
+    """A name a module can be imported by, with the directory an import of that name
+    starts from: the entry of the import path under which the module has that name."""
+
+    name: str
+    root: Path
+
+
 @dataclass(frozen=True)
 class SourceFile:
-    """Decoded source text, its identity and its dotted module name."""
+    """Decoded source text, its identity and its dotted module name; for a file on disk,
+    the names it can be imported by (``import_names``), its module name first."""
 
     source_id: SourceId
     text: str
     module_name: str
     path: Path | None = None
     is_package: bool = False
+    import_names: tuple[ImportName, ...] = ()