runtime/__init__.py

# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

"""
Example usage:

.. code-block:: python

    from pathlib import Path

    import torch
    from executorch.runtime import Verification, Runtime, Program, Method

    et_runtime: Runtime = Runtime.get()
    program: Program = et_runtime.load_program(
        Path("/tmp/program.pte"),
        verification=Verification.Minimal,
    )
    print("Program methods:", program.method_names)
    forward: Method = program.load_method("forward")

    inputs = (torch.ones(2, 2), torch.ones(2, 2))
    outputs = forward.execute(inputs)
    print(f"Ran forward({inputs})")
    print(f"  outputs: {outputs}")

Example output:

.. code-block:: text

    Program methods: ('forward', 'forward2')
    Ran forward((tensor([[1., 1.],
            [1., 1.]]), tensor([[1., 1.],
            [1., 1.]])))
      outputs: [tensor([[1., 1.],
            [1., 1.]])]
"""

import functools
from pathlib import Path
from types import ModuleType
from typing import Any, BinaryIO, Dict, Optional, Sequence, Set, Union

try:
    from executorch.extension.pybindings.portable_lib import (
        ExecuTorchModule,
        MethodMeta,
        Verification,
    )
except ModuleNotFoundError as e:
    raise ModuleNotFoundError(
        "Prebuilt <site-packages>/extension/pybindings/_portable_lib.so "
        "is not found. Please reinstall ExecuTorch from pip."
    ) from e


class Method:
    """An ExecuTorch method, loaded from a Program.
    This can be used to execute the method with inputs.
    """

    def __init__(self, method_name: str, module: ExecuTorchModule) -> None:
        # TODO: This class should be pybind to the C++ counterpart instead of hosting ExecuTorchModule.
        self._method_name = method_name
        self._module = module

    def execute(self, inputs: Sequence[Any]) -> Sequence[Any]:
        """Executes the method with the given inputs.

        Args:
            inputs: The inputs to the method.

        Returns:
            The outputs of the method.
        """
        return self._module.run_method(self._method_name, inputs)

    @property
    def metadata(self) -> MethodMeta:
        """Gets the metadata for the method.

        Returns:
            The metadata for the method.
        """
        return self._module.method_meta(self._method_name)


class Program:
    """An ExecuTorch program, loaded from binary PTE data.

    This can be used to load the methods/models defined by the program.
    """

    def __init__(self, module: ExecuTorchModule, data: Optional[bytes]) -> None:
        # Hold the data so the program is not freed.
        self._data = data
        self._module = module
        self._methods: Dict[str, Method] = {}
        # ExecuTorchModule already pre-loads all Methods when created, so this
        # doesn't do any extra work. TODO: Don't load a given Method until
        # load_method() is called. Create a separate Method instance each time,
        # to allow multiple independent instances of the same model.
        for method_name in self._module.method_names():
            self._methods[method_name] = Method(method_name, self._module)

    @property
    def method_names(self) -> Set[str]:
        """
        Returns method names of the `Program` as a set of strings.
        """
        return set(self._methods.keys())

    def load_method(self, name: str) -> Optional[Method]:
        """Loads a method from the program.

        Args:
            name: The name of the method to load.

        Returns:
            The loaded method.
        """
        return self._methods.get(name, None)


class OperatorRegistry:
    """The registry of operators that are available to the runtime."""

    def __init__(self, legacy_module: ModuleType) -> None:
        # TODO: Expose the kernel callables to Python.
        self._legacy_module = legacy_module

    @property
    def operator_names(self) -> Set[str]:
        """
        Returns the names of all registered operators as a set of strings.
        """
        return set(self._legacy_module._get_operator_names())


class Runtime:
    """An instance of the ExecuTorch runtime environment.

    This can be used to concurrently load and execute any number of ExecuTorch
    programs and methods.
    """

    @staticmethod
    @functools.lru_cache(maxsize=1)
    def get() -> "Runtime":
        """Gets the Runtime singleton."""
        import executorch.extension.pybindings.portable_lib as legacy_module

        return Runtime(legacy_module=legacy_module)

    def __init__(self, *, legacy_module: ModuleType) -> None:
        # Public attributes.
        self.operator_registry = OperatorRegistry(legacy_module)
        # Private attributes.
        self._legacy_module = legacy_module

    def load_program(
        self,
        data: Union[bytes, bytearray, BinaryIO, Path, str],
        *,
        verification: Verification = Verification.InternalConsistency,
    ) -> Program:
        """Loads an ExecuTorch program from a PTE binary.

        Args:
            data: The binary program data to load; typically PTE data.
            verification: level of program verification to perform.

        Returns:
            The loaded program.
        """
        if isinstance(data, (Path, str)):
            m = self._legacy_module._load_for_executorch(
                str(data),
                enable_etdump=False,
                debug_buffer_size=0,
                program_verification=verification,
            )
            return Program(m, data=None)
        elif isinstance(data, BinaryIO):
            data_bytes = data.read()
        elif isinstance(data, bytearray):
            data_bytes = bytes(data)
        elif isinstance(data, bytes):
            data_bytes = data
        else:
            raise TypeError(
                f"Expected data to be bytes, bytearray, a path to a .pte file, or a file-like object, but got {type(data).__name__}."
            )
        m = self._legacy_module._load_for_executorch_from_buffer(
            data_bytes,
            enable_etdump=False,
            debug_buffer_size=0,
            program_verification=verification,
        )

        return Program(m, data=data_bytes)