manage.py

#! /usr/bin/env python

from argparse import ArgumentParser, FileType, Namespace, _SubParsersAction
import glob
import json
import os
import os.path as P
from pathlib import Path, PurePath
import shutil
import subprocess
import sys
import tempfile
from typing import Callable, Dict, List

from langkit.packaging import NativeLibPackager, Platform
import langkit.scripts.lkm as lkm
from langkit.utils import (
    LibraryType,
    add_to_path,
    format_printenv,
    get_cpu_count,
    parse_cmdline_args,
)


LANGKIT_ROOT = PurePath(P.dirname(P.realpath(__file__)))
SUPPORT_ROOT = LANGKIT_ROOT / "langkit" / "support"
SUPPORT_GPR = str(SUPPORT_ROOT / "langkit_support.gpr")
SIGSEGV_HANDLER_ROOT = LANGKIT_ROOT / "sigsegv_handler"
SIGSEGV_HANDLER_GPR = SIGSEGV_HANDLER_ROOT / "langkit_sigsegv_handler.gpr"
LKT_LIB_ROOT = LANGKIT_ROOT / "lkt"
LKT_BOOTSTRAP_ROOT = LKT_LIB_ROOT / "bootstrap"

BOOTSTRAP_LKM_BASE_ARGS = [
    f"--config={LKT_BOOTSTRAP_ROOT / 'langkit.yaml'}",
    "--build-dir=.",
]

BOOTSTRAP_LKM_RUN_BASE_ARGS = [
    "run",
    *BOOTSTRAP_LKM_BASE_ARGS,
    "--",
    sys.executable,
    "-m",
    "langkit.scripts.lkm",
]

BOOTSTRAP_LKM_BUILD_BASE_ARGS = [
    *BOOTSTRAP_LKM_BASE_ARGS,
    # Avoid absolute filenames in generated code to avoid variations in the
    # bootstrap code that is under version control.
    "--portable-project",
    # The only thing we need for bootstrap is a shared library that the Python
    # bindings can import: no static nor static-pic libraries nor mains.
    "--library-types=relocatable",
    "--disable-all-mains",
]


def create_subparser(
    subparsers: _SubParsersAction,
    fn: Callable[..., None],
    *,
    with_jobs: bool = False,
    with_no_lksp: bool = False,
    with_gargs: bool = False,
    with_build_dir: bool = False,
    with_libs: bool = False,
    with_generate_dll_lib_adding: bool = False,
    with_generate_msvc_lib: bool = False,
    with_no_mypy: bool = False,
    with_output: bool = False,
    no_basic_options: bool = False,
    accept_unknown_args: bool = False,
) -> ArgumentParser:
    """
    Create a subparser with given ``fn`` as func. Extract doc and name from
    the function.

    :param bool with_jobs: Whether to create the --jobs/-j option.
    :param bool with_no_lksp: Whether to create the --no-langkit-support
        option.
    :param bool with_gargs: Whether to create the --gargs option.
    :param bool with_build_dir: Whether to create the --build-dir option.
    :param bool with_libs: Whether to create the --lib option.
    :param bool with_generate_dll_lib_adding: Whether to create the
        --generate-auto-dll-dirs option.
    :param bool with_output: Whether to create the --output option.
    :param bool with_no_mypy: Whether to create the --no-mypy option.
    :param no_basic_options: Whether to disable the creation of basic options
        (--build-mode, ...).
    """
    subparser = subparsers.add_parser(
        name=fn.__name__.replace("_", "-"),
        help=fn.__doc__,
        add_help=not accept_unknown_args,
    )

    if not no_basic_options:
        subparser.add_argument(
            "--build-mode",
            "-b",
            choices=("dev", "prod"),
            default="dev",
            help="Select a preset for build options.",
        )
        LibraryType.add_argument(subparser)

    if with_jobs:
        subparser.add_argument(
            "--jobs",
            "-j",
            type=int,
            default=get_cpu_count(),
            help="Number of parallel jobs to spawn in parallel (default: your"
            " number of cpu).",
        )
    if with_no_lksp:
        subparser.add_argument(
            "--no-langkit-support",
            action="store_true",
            help="Assume that Langkit_Support is already built and installed."
            " We rebuild it by default, for the convenience of"
            " developers.",
        )
    if with_gargs:
        subparser.add_argument(
            "--gargs",
            action="append",
            help="Options appended to GPRbuild invocations.",
        )
    if with_build_dir:
        subparser.add_argument(
            "--build-dir",
            help="Use a non-default build directory. This allows out-of-tree"
            " builds.",
        )
    if with_libs:
        subparser.add_argument(
            "--lib",
            "-l",
            choices=("python", "lkt"),
            action="append",
            help="Select which libraries on which to operate. By default, work"
            " on all libraries.",
        )
    if with_generate_dll_lib_adding:
        subparser.add_argument(
            "--generate-auto-dll-dirs",
            action="store_true",
            help="For selected libs (python and lkt) forward the DLL"
            " directories adding flag to the generation phase.",
        )
    if with_generate_msvc_lib:
        subparser.add_argument(
            "--generate-msvc-lib",
            action="store_true",
            help="Generate a .lib file from the library DLL that MSVC"
            " toolchains need in order to link against the DLL. This is"
            " supported only on Windows, and requires the Visual Studio"
            " Build Tools in the environment.",
        )
    if with_output:
        subparser.add_argument(
            "--output",
            "-o",
            type=FileType("w"),
            default=sys.stdout,
            help="Write the command output to a file. This is recommended when"
            " writing scripts, so that warnings are not included in the result"
            " (they are not written to that output file)",
        )
    if with_no_mypy:
        subparser.add_argument(
            "--no-mypy",
            action="store_true",
            help="Whether to disable type-checking with mypy.",
        )

    def wrapper(args: Namespace, rest: List[str]):
        if len(rest) > 0:
            print(
                "ERROR - unhandled command line arguments: {}".format(
                    ", ".join(rest)
                )
            )
            sys.exit(1)
        fn(args)

    subparser.set_defaults(func=fn if accept_unknown_args else wrapper)

    return subparser


def build_langkit_support(args: Namespace) -> None:
    """
    Build Langkit_Support.
    """
    build_dir = PurePath(args.build_dir) if args.build_dir else SUPPORT_ROOT

    base_argv = [
        "gprbuild",
        "-p",
        f"-j{args.jobs}",
        f"-XBUILD_MODE={args.build_mode}",
    ]
    if args.build_dir:
        base_argv.extend([f"--relocate-build-tree={build_dir}"])

    gargs = parse_cmdline_args(args.gargs)

    # In order to avoid building the library once per library kind (static,
    # static-pic and relocatable), langkit_support.gpr uses the same object
    # directory for every library kind. This optimization is valid only if we
    # remove "*.lexch" files in the object directory between each call to
    # gprbuild.
    lexch_pattern = str(build_dir / "obj" / args.build_mode / "*.lexch")

    for library_type in args.library_types:
        for lexch in glob.glob(lexch_pattern):
            os.remove(lexch)
        subprocess.check_call(
            base_argv
            + ["-P", SUPPORT_GPR, f"-XLIBRARY_TYPE={library_type.value}"]
            + gargs
        )

    # SigSegV handler is a relocatable library, skip if only static requested
    if LibraryType.relocatable in args.library_types:
        subprocess.check_call(base_argv + ["-P", SIGSEGV_HANDLER_GPR] + gargs)


def langkit_support_env_map(
    args: Namespace, json: bool = False
) -> Dict[str, str]:
    """
    Helper function. Returns a key-value map for langkit_support's environment.
    """
    # Make the shared library for Langkit_Support available to the dynamic
    # linker.
    build_dir = PurePath(args.build_dir) if args.build_dir else SUPPORT_ROOT
    dynamic_lib_dir = str(build_dir / "lib" / "relocatable" / args.build_mode)

    return {
        # Make the "langkit_support.gpr" available to GPRbuild
        "GPR_PROJECT_PATH": str(SUPPORT_ROOT),
        "PATH": dynamic_lib_dir,
        "LD_LIBRARY_PATH": os.pathsep.join(
            [
                dynamic_lib_dir,
                # Make the shared lib for the sigsegv handler available for
                # OCaml on GNU/Linux.
                str(SIGSEGV_HANDLER_ROOT / "lib"),
            ],
        ),
    }


def install_langkit_support(args: Namespace) -> None:
    """
    Install the Langkit_Support project.
    """
    base_argv = [
        "gprinstall",
        "-P",
        SUPPORT_GPR,
        "-p",
        f"-XBUILD_MODE={args.build_mode}",
        f"--prefix={args.prefix}",
        "--build-var=LIBRARY_TYPE",
        "--build-var=LANGKIT_SUPPORT_LIBRARY_TYPE",
        "--sources-subdir=include/langkit_support",
    ]
    if args.build_dir:
        base_argv.extend([f"--relocate-build-tree={args.build_dir}"])
    if args.force:
        base_argv.append("-f")

    # Install the static libraries first, so that in the resulting project
    # files, "static" is the default library type.
    lib_types = [l.value for l in args.library_types]
    for library_type in ("static", "static-pic", "relocatable"):
        if library_type in lib_types:
            subprocess.check_call(
                base_argv
                + [
                    f"-XLIBRARY_TYPE={library_type}",
                    f"--build-name={library_type}",
                ]
            )


def package_deps(args: Namespace) -> None:
    """
    Bundle all dependencies to complete GNAT Pro.
    """
    p = NativeLibPackager.from_args(args)
    p.package_deps(getattr(args, "package-dir"))


def package_std_dyn(args: Namespace) -> None:
    """
    Bundle all dependencies to create standalone packages.
    """
    pass
    p = NativeLibPackager.from_args(args)
    pkg_dir = getattr(args, "package-dir")
    p.package_standalone_dyn(pkg_dir)
    p.package_langkit_support_dyn(pkg_dir)


def printenv(args: Namespace) -> None:
    """
    Print shell commands to add Libpythonlang and Liblktlang to the
    environment.
    """
    env = {}
    if not args.no_langkit_support:
        env = langkit_support_env_map(args)

    with tempfile.NamedTemporaryFile(
        prefix="lkm-printenv",
        suffix=".json",
        delete=False,
    ) as tmpf:
        try:
            tmpf.close()
            subprocess.run(
                [
                    sys.executable,
                    "-m",
                    "langkit.scripts.lkm",
                    "printenv",
                    f"--config={LKT_LIB_ROOT / 'langkit.yaml'}",
                    f"--build-mode={args.build_mode}",
                    "-J",
                    f"--output={tmpf.name}",
                ],
            )
            with open(tmpf.name) as f:
                d = json.load(f)
        finally:
            os.remove(tmpf.name)

    for k, v in d.items():
        if k in env:
            env[k] = os.pathsep.join([env[k], v])
        else:
            env[k] = v

    if args.json:
        json.dump(env, args.output)
    else:
        for k, v in env.items():
            print(format_printenv(k, v), file=args.output)


def printenv_langkit_support(args: Namespace) -> None:
    """
    Printenv for Langkit_Support.
    """
    for k, v in langkit_support_env_map(args).items():
        print(format_printenv(k, v), file=args.output)


def bootstrap_build_args(args: Namespace, generate: bool = False) -> list[str]:
    """
    Return lkm build arguments to build the bootstrap Liblktlang.

    :param generate: Whether to generate the bootstrap Liblktlang.
    """
    argv = [
        "make" if generate else "build",
        *BOOTSTRAP_LKM_BUILD_BASE_ARGS,
        f"-j{args.jobs}",
    ]
    for a in args.gargs or []:
        argv += ["--gargs", a]

    # Allow the Python bindings to find the necessary DLLs through the PATH
    # environment variable on Windows, to simplify the bootstrap setup.
    if generate:
        argv.append("--generate-auto-dll-dirs")

        # Because bootstrap sources are tracked by the version control system,
        # they can change on disk without being re-generated by Langkit (i.e.
        # when checking out a new branch/commit), so the code generation cache
        # is unreliable for them: just don't rely on it.
        argv.append("--force")

    return argv


def prepare_bootstrap(args: Namespace) -> None:
    """
    Make sure the bootstrap Liblktlang library is ready to use.
    """
    check_argv = [
        sys.executable,
        "-m",
        "langkit.scripts.lkm",
        "run",
        *BOOTSTRAP_LKM_BASE_ARGS,
        sys.executable,
        str(LKT_LIB_ROOT / "check_bootstrap.py"),
    ]

    # First check if Liblktlang can be imported: if that's the case, there is
    # nothing else to do.
    p = subprocess.run([*check_argv, "-q"])
    if p.returncode == 0:
        return

    print("Bootstrap Liblktlang needs to be built")
    sys.stdout.flush()
    lkm.main(bootstrap_build_args(args))

    # For dev convenience, abort early if Liblktlang still cannot be imported
    subprocess.check_call(check_argv)


def bootstrap(args: Namespace) -> None:
    """
    Regenerate and build the bootstrap Liblktlang.
    """
    prepare_bootstrap(args)

    # Copy the Lkt project sources (YAML config, Lkt sources and extensions)
    # into the bootstrap directory.
    shutil.copytree(
        src=str(LKT_LIB_ROOT),
        dst=str(LKT_BOOTSTRAP_ROOT),
        ignore=lambda _, items: set(items)
        & {
            ".gitignore",
            "__pycache__",
            "bootstrap",
            "build",
            "check_bootstrap.py",
        },
        dirs_exist_ok=True,
    )

    # Regenerate the Lkt project in the bootstrap directory
    lkm.main(
        [
            *BOOTSTRAP_LKM_RUN_BASE_ARGS,
            *bootstrap_build_args(args, generate=True),
        ]
    )

    # Now that we have the codegen for the bootstrap project, its Lkt sources
    # (just copies of the Lkt project itself) are no longer useful: just remove
    # them, to avoid unecessary bootstrap directory bloat.
    for lkt_src in Path(LKT_BOOTSTRAP_ROOT).glob("**/*.lkt"):
        lkt_src.unlink()

    # Likewise for the extension sources used as templates (i.e. only during
    # codegen).
    for child in Path(LKT_BOOTSTRAP_ROOT / "extensions").iterdir():
        if child.name != "src":
            if child.is_dir():
                shutil.rmtree(str(child))
            else:
                child.unlink()

    # Get rid of the Java and OCaml bindings, irrelevant for bootstrap matters
    for d in ["java", "ocaml"]:
        shutil.rmtree(str(LKT_BOOTSTRAP_ROOT / d))


def clean(args: Namespace) -> None:
    """
    Clean up build artifacts for the bootstrap Liblktlang.
    """
    for subdir in ["obj", "lib"]:
        shutil.rmtree(str(LKT_BOOTSTRAP_ROOT / subdir))


def make(args: Namespace) -> None:
    """
    Generate and build Libpythonlang and Liblktlang.
    """
    prepare_bootstrap(args)

    # Unless specifically asked to ignore Langkit_Support, make sure it is
    # built and available to build Liblktlang.
    if not args.no_langkit_support:
        build_langkit_support(args)
        add_to_path(os.environ, "GPR_PROJECT_PATH", str(SUPPORT_ROOT))

    lib_types = ",".join(l.value for l in args.library_types)
    argv = [
        "make",
        "--config",
        str(LKT_LIB_ROOT / "langkit.yaml"),
        f"--library-types={lib_types}",
        f"--build-mode={args.build_mode}",
        f"-j{args.jobs}",
    ]

    # If the DLL directories adding flag is on forward it
    if args.generate_auto_dll_dirs:
        argv.append("--generate-auto-dll-dirs")

    # If the MSVC lib flag is on forward it
    if args.generate_msvc_lib:
        argv.append("--generate-msvc-lib")

    # Forward gargs to each manage.py script
    for gargs in args.gargs or []:
        argv.append(f"--gargs={gargs}")

    lkm.main([*BOOTSTRAP_LKM_RUN_BASE_ARGS, *argv])

    # Unless disabled, run mypy to type check Langkit itself. We need to do
    # this after building Liblktlang as Langkit depend on them.
    if not args.no_mypy:
        run_mypy(args)


def run_mypy(args: Namespace) -> None:
    """
    Type-check the Langkit Python codebase.
    """
    # Make sure mypy can find the type hints for the Liblktlang Python
    # bindings.
    env = dict(os.environ)
    add_to_path(env, "MYPYPATH", str(LKT_LIB_ROOT / "build" / "python"))
    subprocess.check_call(["mypy"], cwd=LANGKIT_ROOT, env=env)


def test(args: Namespace, remaining_args: List[str]) -> None:
    """
    Run Langkit's testsuite.
    """
    # Propagate the return code from the testsuite to our own parent process.
    # This is useful for scripts (for instance CIs) to easily detect when there
    # is at least one failure.
    sys.exit(
        subprocess.call(
            [
                sys.executable,
                str(LANGKIT_ROOT / "testsuite" / "testsuite.py"),
                "-E",
            ]
            + remaining_args
        )
    )


if __name__ == "__main__":
    parser = ArgumentParser(description="Global manage script for langkit")
    subparsers = parser.add_subparsers()

    create_subparser(
        subparsers,
        build_langkit_support,
        with_jobs=True,
        with_gargs=True,
        with_build_dir=True,
    )
    create_subparser(
        subparsers,
        printenv_langkit_support,
        with_build_dir=True,
        with_output=True,
    )
    install_lksp = create_subparser(
        subparsers, install_langkit_support, with_build_dir=True
    )
    install_lksp.add_argument(
        "--force",
        "-f",
        action="store_true",
        help="Force installation, overwrite files.",
    )
    install_lksp.add_argument("prefix", help="Installation prefix")

    package_deps_parser = create_subparser(subparsers, package_deps)
    package_std_dyn_parser = create_subparser(subparsers, package_std_dyn)
    for p in (package_deps_parser, package_std_dyn_parser):
        p.add_argument("package-dir", help="Destination directory")
        NativeLibPackager.add_prefix_options(p)
        Platform.add_options(p)

    create_subparser(
        subparsers,
        make,
        with_jobs=True,
        with_no_lksp=True,
        with_gargs=True,
        with_build_dir=True,
        with_libs=True,
        with_generate_dll_lib_adding=True,
        with_generate_msvc_lib=True,
        with_no_mypy=True,
    )
    printenv_parser = create_subparser(
        subparsers,
        printenv,
        with_no_lksp=True,
        with_build_dir=True,
        with_libs=True,
        with_output=True,
    )
    printenv_parser.add_argument(
        "--json",
        "-J",
        action="store_true",
        help="Output necessary env keys to JSON.",
    )

    create_subparser(subparsers, run_mypy)

    create_subparser(subparsers, test, accept_unknown_args=True)

    create_subparser(subparsers, bootstrap, with_jobs=True, with_gargs=True)

    create_subparser(subparsers, clean, no_basic_options=True)

    parser.set_defaults(func=lambda _, _1: None)

    args, unknown_args = parser.parse_known_args()
    args.func(args, unknown_args)