I am working with a C++ project in which the CMake build process generates a directory structure of Python packages (i.e. Python source files, not shared object) in the chosen build directory under python_packages/my_package. I cannot change this approach. I have written a setup.py script that runs CMake and tries to add these to the generated package but I am stuck. Here's where I'm at:

import os
import subprocess
from pathlib import Path

from setuptools import (
    Extension,
    find_packages,
    setup,
)
from setuptoolsmand.build_ext import build_ext


class CMakeExtension(Extension):
    def __init__(self, name):
        super().__init__(name, sources=[])


class cmake_build_ext(build_ext):
    def build_extension(self, ext):
        cmake_source_dir = Path.cwd()

        cmake_build_dir = Path(self.build_temp)
        if not cmake_build_dir.exists():
            cmake_build_dir.mkdir(parents=True)

        cmake_preset = "debug" if self.debug else "release"

        cmake_env = os.environ.copy()

        cmake_cmd = [
            "/usr/bin/cmake",
            "-S", cmake_source_dir,
            "-B", cmake_build_dir,
            "--preset", cmake_preset
        ]

        subprocess.check_call(cmake_cmd, cwd=cmake_build_dir)

        cmake_build_cmd = [
            "/usr/bin/cmake",
            "--build", cmake_build_dir,
        ]

        subprocess.check_call(cmake_build_cmd, cwd=cmake_build_dir)

        packages_dir = cmake_build_dir / "python_packages"
        
        packages = find_packages(packages_dir)
        print("Packages:", packages)

        self.distribution.packages = packages


setup(
    name="my_project",
    version="1.0.0",
    author="Me Myself",
    author_email="[email protected]",
    description = "CMake experiment",
    python_requires=">=3.12",
    ext_modules=[CMakeExtension("my_project")],
    cmdclass={"build_ext": cmake_build_ext},
    zip_safe=False,
)

When calling pip install . this runs up until the running build_py step where I get the following error:

error: package directory 'my_package' does not exist

At the same time, Packages: ["my_package", "my_package.sub_package"] is printed during execution. There is likely something I should be doing here with package_dir but I am lost.