How to Inspect Function and Class Signatures in Python#
Show code cell content
import inspect
from dataclasses import field, make_dataclass
from inspect import Parameter, Signature
from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union, _GenericAlias, get_type_hints, overload
from pydantic import BaseModel
from rich.pretty import pprint
from transformers import GPT2LMHeadModel, Trainer, TrainingArguments
Motivation#
There are two motivations for why we want to inspect function and class signatures in Python.
Code Introspection in Open Source Projects: When we are working on open source projects, we often need to inspect the function and class signatures of the codebase. This is especially true when we are working on a large codebase with many functions and classes. In this case, we need to inspect the function and class signatures to understand how the codebase is structured and how the functions and classes are used.
Sometimes there are nested abstractions, a child class \(\mathcal{C}_N\) that inherits from a parent class \(\mathcal{C}_{N-1}\), which in turn inherits from another parent class \(\mathcal{C}_{N-2}\), and so on. Sometimes the child class does not immediately show what types of arguments the constructor can take. In this case, we need to inspect the parent classes to understand the constructor signature of the child class.
Agent-Based Function Calling: In agent-based programming, where automated agents are tasked with executing functions or interacting with libraries, the risk of ‘hallucination’—where an agent attempts to invoke non-existent methods or improperly structured calls from a external module/library—is a notable concern. By equipping agents with the capability to query the actual signatures of libraries’ classes or functions, we can significantly mitigate this risk. This ensures that agents operate based on accurate, real-time information, thereby improving the reliability and effectiveness of automated tasks.
Construct Hypothetical Function, Child and Parent Classes#
class ParentClass:
"""This is the parent class."""
parent_class_attr = 'a parent class attribute'
def __init__(self, parent_instance_attr: str) -> None:
self.parent_instance_attr = parent_instance_attr
def parent_method(self) -> str:
"""This is a method in the parent class."""
return "Parent method called"
class ChildClass(ParentClass):
"""This is a subclass of ParentClass."""
# Class attribute
class_attr = 'a class attribute'
# Private and protected attributes
_protected_attr = 'a protected attribute'
__private_attr = 'a private attribute'
def __init__(self, instance_attr: str, parent_instance_attr: str) -> None:
"""Initialize the instance."""
super().__init__(parent_instance_attr)
# Instance attribute
self.instance_attr = instance_attr
self.instance_not_in_constructor_attr = 'an instance attribute not in the constructor'
self._private_instance_attr = 'a private instance attribute'
@property
def read_only_attr(self) -> str:
"""This is a read-only attribute."""
return 'You can read me, but you cannot change me.'
def instance_method(self, arg: str) -> str:
"""This is an instance method."""
return f'Instance method called with argument: {arg}'
@classmethod
def class_method(cls, arg: str) -> str:
"""This is a class method."""
return f'Class method called with argument: {arg}'
@staticmethod
def static_method(arg: str) -> str:
"""This is a static method."""
return f'Static method called with argument: {arg}'
def __str__(self) -> str:
"""Return a string representation of the instance."""
return f'MyClass(instance_attr={self.instance_attr})'
instance_child = ChildClass(instance_attr='an instance attribute', parent_instance_attr='a parent instance attribute')
class_child = ChildClass
instance_parent = ParentClass(parent_instance_attr='a parent instance attribute')
class_parent = ParentClass
def func(a: int, b: str, c: List[int], d: Tuple[str, str], e: Union[int, str], **kwargs: Any) -> str:
return a, b, c, d, e, kwargs
Inspect All Members#
@overload
def get_members_of_function_or_method(
func_or_class: Type[object], predicate: Optional[Callable[[Any], bool]] = None
) -> List[Tuple[str, Any]]:
...
@overload
def get_members_of_function_or_method(
func_or_class: Callable[..., Any], predicate: Optional[Callable[[Any], bool]] = None
) -> List[Tuple[str, Any]]:
...
def get_members_of_function_or_method(
func_or_class: Union[Type[object], Callable[..., Any]], predicate: Optional[Callable[[Any], bool]] = None
) -> List[Tuple[str, Any]]:
return inspect.getmembers(func_or_class, predicate)
def loop_through_members(members: List[Tuple[str, Any]], filter: Optional[str] = None) -> None:
if filter is not None:
members = [member for member in members if filter in member[0]]
for member in members:
name, value = member
print(f'{name}: {value}')
Our initial goal is to get all signatures and type annotations of a class or function. We can use the inspect module to achieve this. The getmembers function returns all members of a class or module. We can then filter out the functions and classes and inspect their signatures.
However, for our purpose, it may be overkill since it retrieves all members
within a module, the scope is very broad, for example, inspecting just the func
defined will also return all __globals__, which may not be what we want.
func_all_members = get_members_of_function_or_method(func, predicate=None)
loop_through_members(func_all_members)
__annotations__: {'a': <class 'int'>, 'b': <class 'str'>, 'c': typing.List[int], 'd': typing.Tuple[str, str], 'e': typing.Union[int, str], 'kwargs': typing.Any, 'return': <class 'str'>}
__call__: <method-wrapper '__call__' of function object at 0x7f295472f940>
__class__: <class 'function'>
__closure__: None
__code__: <code object func at 0x7f295473d5b0, file "/tmp/ipykernel_2521/2139551385.py", line 1>
__defaults__: None
__delattr__: <method-wrapper '__delattr__' of function object at 0x7f295472f940>
__dict__: {}
__dir__: <built-in method __dir__ of function object at 0x7f295472f940>
__doc__: None
__eq__: <method-wrapper '__eq__' of function object at 0x7f295472f940>
__format__: <built-in method __format__ of function object at 0x7f295472f940>
__ge__: <method-wrapper '__ge__' of function object at 0x7f295472f940>
__get__: <method-wrapper '__get__' of function object at 0x7f295472f940>
__getattribute__: <method-wrapper '__getattribute__' of function object at 0x7f295472f940>
__globals__: {'__name__': '__main__', '__doc__': 'Automatically created module for IPython interactive environment', '__package__': None, '__loader__': None, '__spec__': None, '__builtin__': <module 'builtins' (built-in)>, '__builtins__': <module 'builtins' (built-in)>, '_ih': ['', 'import inspect\nfrom dataclasses import field, make_dataclass\nfrom inspect import Parameter, Signature\nfrom typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union, _GenericAlias, get_type_hints, overload\n\nfrom pydantic import BaseModel\nfrom rich.pretty import pprint\nfrom transformers import GPT2LMHeadModel, Trainer, TrainingArguments', 'class ParentClass:\n """This is the parent class."""\n\n parent_class_attr = \'a parent class attribute\'\n\n def __init__(self, parent_instance_attr: str) -> None:\n self.parent_instance_attr = parent_instance_attr\n\n def parent_method(self) -> str:\n """This is a method in the parent class."""\n return "Parent method called"\n\nclass ChildClass(ParentClass):\n """This is a subclass of ParentClass."""\n\n # Class attribute\n class_attr = \'a class attribute\'\n\n # Private and protected attributes\n _protected_attr = \'a protected attribute\'\n __private_attr = \'a private attribute\'\n\n def __init__(self, instance_attr: str, parent_instance_attr: str) -> None:\n """Initialize the instance."""\n super().__init__(parent_instance_attr)\n # Instance attribute\n self.instance_attr = instance_attr\n self.instance_not_in_constructor_attr = \'an instance attribute not in the constructor\'\n self._private_instance_attr = \'a private instance attribute\'\n\n @property\n def read_only_attr(self) -> str:\n """This is a read-only attribute."""\n return \'You can read me, but you cannot change me.\'\n\n def instance_method(self, arg: str) -> str:\n """This is an instance method."""\n return f\'Instance method called with argument: {arg}\'\n\n @classmethod\n def class_method(cls, arg: str) -> str:\n """This is a class method."""\n return f\'Class method called with argument: {arg}\'\n\n @staticmethod\n def static_method(arg: str) -> str:\n """This is a static method."""\n return f\'Static method called with argument: {arg}\'\n\n def __str__(self) -> str:\n """Return a string representation of the instance."""\n return f\'MyClass(instance_attr={self.instance_attr})\'', "instance_child = ChildClass(instance_attr='an instance attribute', parent_instance_attr='a parent instance attribute')\nclass_child = ChildClass\n\ninstance_parent = ParentClass(parent_instance_attr='a parent instance attribute')\nclass_parent = ParentClass", 'def func(a: int, b: str, c: List[int], d: Tuple[str, str], e: Union[int, str], **kwargs: Any) -> str:\n return a, b, c, d, e, kwargs', "@overload\ndef get_members_of_function_or_method(\n func_or_class: Type[object], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n ...\n\n\n@overload\ndef get_members_of_function_or_method(\n func_or_class: Callable[..., Any], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n ...\n\n\ndef get_members_of_function_or_method(\n func_or_class: Union[Type[object], Callable[..., Any]], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n return inspect.getmembers(func_or_class, predicate)\n\ndef loop_through_members(members: List[Tuple[str, Any]], filter: Optional[str] = None) -> None:\n if filter is not None:\n members = [member for member in members if filter in member[0]]\n for member in members:\n name, value = member\n print(f'{name}: {value}')", 'func_all_members = get_members_of_function_or_method(func, predicate=None)\nloop_through_members(func_all_members)'], '_oh': {}, '_dh': [PosixPath('/home/runner/work/omniverse/omniverse/omniverse/playbook')], 'In': ['', 'import inspect\nfrom dataclasses import field, make_dataclass\nfrom inspect import Parameter, Signature\nfrom typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union, _GenericAlias, get_type_hints, overload\n\nfrom pydantic import BaseModel\nfrom rich.pretty import pprint\nfrom transformers import GPT2LMHeadModel, Trainer, TrainingArguments', 'class ParentClass:\n """This is the parent class."""\n\n parent_class_attr = \'a parent class attribute\'\n\n def __init__(self, parent_instance_attr: str) -> None:\n self.parent_instance_attr = parent_instance_attr\n\n def parent_method(self) -> str:\n """This is a method in the parent class."""\n return "Parent method called"\n\nclass ChildClass(ParentClass):\n """This is a subclass of ParentClass."""\n\n # Class attribute\n class_attr = \'a class attribute\'\n\n # Private and protected attributes\n _protected_attr = \'a protected attribute\'\n __private_attr = \'a private attribute\'\n\n def __init__(self, instance_attr: str, parent_instance_attr: str) -> None:\n """Initialize the instance."""\n super().__init__(parent_instance_attr)\n # Instance attribute\n self.instance_attr = instance_attr\n self.instance_not_in_constructor_attr = \'an instance attribute not in the constructor\'\n self._private_instance_attr = \'a private instance attribute\'\n\n @property\n def read_only_attr(self) -> str:\n """This is a read-only attribute."""\n return \'You can read me, but you cannot change me.\'\n\n def instance_method(self, arg: str) -> str:\n """This is an instance method."""\n return f\'Instance method called with argument: {arg}\'\n\n @classmethod\n def class_method(cls, arg: str) -> str:\n """This is a class method."""\n return f\'Class method called with argument: {arg}\'\n\n @staticmethod\n def static_method(arg: str) -> str:\n """This is a static method."""\n return f\'Static method called with argument: {arg}\'\n\n def __str__(self) -> str:\n """Return a string representation of the instance."""\n return f\'MyClass(instance_attr={self.instance_attr})\'', "instance_child = ChildClass(instance_attr='an instance attribute', parent_instance_attr='a parent instance attribute')\nclass_child = ChildClass\n\ninstance_parent = ParentClass(parent_instance_attr='a parent instance attribute')\nclass_parent = ParentClass", 'def func(a: int, b: str, c: List[int], d: Tuple[str, str], e: Union[int, str], **kwargs: Any) -> str:\n return a, b, c, d, e, kwargs', "@overload\ndef get_members_of_function_or_method(\n func_or_class: Type[object], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n ...\n\n\n@overload\ndef get_members_of_function_or_method(\n func_or_class: Callable[..., Any], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n ...\n\n\ndef get_members_of_function_or_method(\n func_or_class: Union[Type[object], Callable[..., Any]], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n return inspect.getmembers(func_or_class, predicate)\n\ndef loop_through_members(members: List[Tuple[str, Any]], filter: Optional[str] = None) -> None:\n if filter is not None:\n members = [member for member in members if filter in member[0]]\n for member in members:\n name, value = member\n print(f'{name}: {value}')", 'func_all_members = get_members_of_function_or_method(func, predicate=None)\nloop_through_members(func_all_members)'], 'Out': {}, 'get_ipython': <bound method InteractiveShell.get_ipython of <ipykernel.zmqshell.ZMQInteractiveShell object at 0x7f2a50451130>>, 'exit': <IPython.core.autocall.ZMQExitAutocall object at 0x7f2a50451ac0>, 'quit': <IPython.core.autocall.ZMQExitAutocall object at 0x7f2a50451ac0>, 'open': <function open at 0x7f2a5549b8b0>, '_': '', '__': '', '___': '', '_i': "@overload\ndef get_members_of_function_or_method(\n func_or_class: Type[object], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n ...\n\n\n@overload\ndef get_members_of_function_or_method(\n func_or_class: Callable[..., Any], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n ...\n\n\ndef get_members_of_function_or_method(\n func_or_class: Union[Type[object], Callable[..., Any]], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n return inspect.getmembers(func_or_class, predicate)\n\ndef loop_through_members(members: List[Tuple[str, Any]], filter: Optional[str] = None) -> None:\n if filter is not None:\n members = [member for member in members if filter in member[0]]\n for member in members:\n name, value = member\n print(f'{name}: {value}')", '_ii': 'def func(a: int, b: str, c: List[int], d: Tuple[str, str], e: Union[int, str], **kwargs: Any) -> str:\n return a, b, c, d, e, kwargs', '_iii': "instance_child = ChildClass(instance_attr='an instance attribute', parent_instance_attr='a parent instance attribute')\nclass_child = ChildClass\n\ninstance_parent = ParentClass(parent_instance_attr='a parent instance attribute')\nclass_parent = ParentClass", '_i1': 'import inspect\nfrom dataclasses import field, make_dataclass\nfrom inspect import Parameter, Signature\nfrom typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union, _GenericAlias, get_type_hints, overload\n\nfrom pydantic import BaseModel\nfrom rich.pretty import pprint\nfrom transformers import GPT2LMHeadModel, Trainer, TrainingArguments', 'inspect': <module 'inspect' from '/opt/hostedtoolcache/Python/3.9.19/x64/lib/python3.9/inspect.py'>, 'field': <function field at 0x7f2a5591d280>, 'make_dataclass': <function make_dataclass at 0x7f2a5592fc10>, 'Parameter': <class 'inspect.Parameter'>, 'Signature': <class 'inspect.Signature'>, 'Any': typing.Any, 'Callable': typing.Callable, 'Dict': typing.Dict, 'List': typing.List, 'Optional': typing.Optional, 'Set': typing.Set, 'Tuple': typing.Tuple, 'Type': typing.Type, 'Union': typing.Union, '_GenericAlias': <class 'typing._GenericAlias'>, 'get_type_hints': <function get_type_hints at 0x7f2a57793c10>, 'overload': <function overload at 0x7f2a57798040>, 'BaseModel': <class 'pydantic.main.BaseModel'>, 'pprint': <function pprint at 0x7f2a501b34c0>, 'GPT2LMHeadModel': <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>, 'Trainer': <class 'transformers.trainer.Trainer'>, 'TrainingArguments': <class 'transformers.training_args.TrainingArguments'>, '_i2': 'class ParentClass:\n """This is the parent class."""\n\n parent_class_attr = \'a parent class attribute\'\n\n def __init__(self, parent_instance_attr: str) -> None:\n self.parent_instance_attr = parent_instance_attr\n\n def parent_method(self) -> str:\n """This is a method in the parent class."""\n return "Parent method called"\n\nclass ChildClass(ParentClass):\n """This is a subclass of ParentClass."""\n\n # Class attribute\n class_attr = \'a class attribute\'\n\n # Private and protected attributes\n _protected_attr = \'a protected attribute\'\n __private_attr = \'a private attribute\'\n\n def __init__(self, instance_attr: str, parent_instance_attr: str) -> None:\n """Initialize the instance."""\n super().__init__(parent_instance_attr)\n # Instance attribute\n self.instance_attr = instance_attr\n self.instance_not_in_constructor_attr = \'an instance attribute not in the constructor\'\n self._private_instance_attr = \'a private instance attribute\'\n\n @property\n def read_only_attr(self) -> str:\n """This is a read-only attribute."""\n return \'You can read me, but you cannot change me.\'\n\n def instance_method(self, arg: str) -> str:\n """This is an instance method."""\n return f\'Instance method called with argument: {arg}\'\n\n @classmethod\n def class_method(cls, arg: str) -> str:\n """This is a class method."""\n return f\'Class method called with argument: {arg}\'\n\n @staticmethod\n def static_method(arg: str) -> str:\n """This is a static method."""\n return f\'Static method called with argument: {arg}\'\n\n def __str__(self) -> str:\n """Return a string representation of the instance."""\n return f\'MyClass(instance_attr={self.instance_attr})\'', 'ParentClass': <class '__main__.ParentClass'>, 'ChildClass': <class '__main__.ChildClass'>, '_i3': "instance_child = ChildClass(instance_attr='an instance attribute', parent_instance_attr='a parent instance attribute')\nclass_child = ChildClass\n\ninstance_parent = ParentClass(parent_instance_attr='a parent instance attribute')\nclass_parent = ParentClass", 'instance_child': <__main__.ChildClass object at 0x7f29547dbd60>, 'class_child': <class '__main__.ChildClass'>, 'instance_parent': <__main__.ParentClass object at 0x7f29547db8e0>, 'class_parent': <class '__main__.ParentClass'>, '_i4': 'def func(a: int, b: str, c: List[int], d: Tuple[str, str], e: Union[int, str], **kwargs: Any) -> str:\n return a, b, c, d, e, kwargs', 'func': <function func at 0x7f295472f940>, '_i5': "@overload\ndef get_members_of_function_or_method(\n func_or_class: Type[object], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n ...\n\n\n@overload\ndef get_members_of_function_or_method(\n func_or_class: Callable[..., Any], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n ...\n\n\ndef get_members_of_function_or_method(\n func_or_class: Union[Type[object], Callable[..., Any]], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n return inspect.getmembers(func_or_class, predicate)\n\ndef loop_through_members(members: List[Tuple[str, Any]], filter: Optional[str] = None) -> None:\n if filter is not None:\n members = [member for member in members if filter in member[0]]\n for member in members:\n name, value = member\n print(f'{name}: {value}')", 'get_members_of_function_or_method': <function get_members_of_function_or_method at 0x7f295472ff70>, 'loop_through_members': <function loop_through_members at 0x7f2954737040>, '_i6': 'func_all_members = get_members_of_function_or_method(func, predicate=None)\nloop_through_members(func_all_members)', 'func_all_members': [('__annotations__', {'a': <class 'int'>, 'b': <class 'str'>, 'c': typing.List[int], 'd': typing.Tuple[str, str], 'e': typing.Union[int, str], 'kwargs': typing.Any, 'return': <class 'str'>}), ('__call__', <method-wrapper '__call__' of function object at 0x7f295472f940>), ('__class__', <class 'function'>), ('__closure__', None), ('__code__', <code object func at 0x7f295473d5b0, file "/tmp/ipykernel_2521/2139551385.py", line 1>), ('__defaults__', None), ('__delattr__', <method-wrapper '__delattr__' of function object at 0x7f295472f940>), ('__dict__', {}), ('__dir__', <built-in method __dir__ of function object at 0x7f295472f940>), ('__doc__', None), ('__eq__', <method-wrapper '__eq__' of function object at 0x7f295472f940>), ('__format__', <built-in method __format__ of function object at 0x7f295472f940>), ('__ge__', <method-wrapper '__ge__' of function object at 0x7f295472f940>), ('__get__', <method-wrapper '__get__' of function object at 0x7f295472f940>), ('__getattribute__', <method-wrapper '__getattribute__' of function object at 0x7f295472f940>), ('__globals__', {...}), ('__gt__', <method-wrapper '__gt__' of function object at 0x7f295472f940>), ('__hash__', <method-wrapper '__hash__' of function object at 0x7f295472f940>), ('__init__', <method-wrapper '__init__' of function object at 0x7f295472f940>), ('__init_subclass__', <built-in method __init_subclass__ of type object at 0x7f2a58365960>), ('__kwdefaults__', None), ('__le__', <method-wrapper '__le__' of function object at 0x7f295472f940>), ('__lt__', <method-wrapper '__lt__' of function object at 0x7f295472f940>), ('__module__', '__main__'), ('__name__', 'func'), ('__ne__', <method-wrapper '__ne__' of function object at 0x7f295472f940>), ('__new__', <built-in method __new__ of type object at 0x7f2a58365960>), ('__qualname__', 'func'), ('__reduce__', <built-in method __reduce__ of function object at 0x7f295472f940>), ('__reduce_ex__', <built-in method __reduce_ex__ of function object at 0x7f295472f940>), ('__repr__', <method-wrapper '__repr__' of function object at 0x7f295472f940>), ('__setattr__', <method-wrapper '__setattr__' of function object at 0x7f295472f940>), ('__sizeof__', <built-in method __sizeof__ of function object at 0x7f295472f940>), ('__str__', <method-wrapper '__str__' of function object at 0x7f295472f940>), ('__subclasshook__', <built-in method __subclasshook__ of type object at 0x7f2a58365960>)]}
__gt__: <method-wrapper '__gt__' of function object at 0x7f295472f940>
__hash__: <method-wrapper '__hash__' of function object at 0x7f295472f940>
__init__: <method-wrapper '__init__' of function object at 0x7f295472f940>
__init_subclass__: <built-in method __init_subclass__ of type object at 0x7f2a58365960>
__kwdefaults__: None
__le__: <method-wrapper '__le__' of function object at 0x7f295472f940>
__lt__: <method-wrapper '__lt__' of function object at 0x7f295472f940>
__module__: __main__
__name__: func
__ne__: <method-wrapper '__ne__' of function object at 0x7f295472f940>
__new__: <built-in method __new__ of type object at 0x7f2a58365960>
__qualname__: func
__reduce__: <built-in method __reduce__ of function object at 0x7f295472f940>
__reduce_ex__: <built-in method __reduce_ex__ of function object at 0x7f295472f940>
__repr__: <method-wrapper '__repr__' of function object at 0x7f295472f940>
__setattr__: <method-wrapper '__setattr__' of function object at 0x7f295472f940>
__sizeof__: <built-in method __sizeof__ of function object at 0x7f295472f940>
__str__: <method-wrapper '__str__' of function object at 0x7f295472f940>
__subclasshook__: <built-in method __subclasshook__ of type object at 0x7f2a58365960>
And to get the signature, we can just filter '__annotations__'.
loop_through_members(func_all_members, filter='__annotations__')
__annotations__: {'a': <class 'int'>, 'b': <class 'str'>, 'c': typing.List[int], 'd': typing.Tuple[str, str], 'e': typing.Union[int, str], 'kwargs': typing.Any, 'return': <class 'str'>}
class_child_all_members = get_members_of_function_or_method(class_child, predicate=None)
loop_through_members(class_child_all_members)
_ChildClass__private_attr: a private attribute
__class__: <class 'type'>
__delattr__: <slot wrapper '__delattr__' of 'object' objects>
__dict__: {'__module__': '__main__', '__doc__': 'This is a subclass of ParentClass.', 'class_attr': 'a class attribute', '_protected_attr': 'a protected attribute', '_ChildClass__private_attr': 'a private attribute', '__init__': <function ChildClass.__init__ at 0x7f295472fb80>, 'read_only_attr': <property object at 0x7f2954735900>, 'instance_method': <function ChildClass.instance_method at 0x7f295472fca0>, 'class_method': <classmethod object at 0x7f2954770f10>, 'static_method': <staticmethod object at 0x7f29547704c0>, '__str__': <function ChildClass.__str__ at 0x7f295472fe50>}
__dir__: <method '__dir__' of 'object' objects>
__doc__: This is a subclass of ParentClass.
__eq__: <slot wrapper '__eq__' of 'object' objects>
__format__: <method '__format__' of 'object' objects>
__ge__: <slot wrapper '__ge__' of 'object' objects>
__getattribute__: <slot wrapper '__getattribute__' of 'object' objects>
__gt__: <slot wrapper '__gt__' of 'object' objects>
__hash__: <slot wrapper '__hash__' of 'object' objects>
__init__: <function ChildClass.__init__ at 0x7f295472fb80>
__init_subclass__: <built-in method __init_subclass__ of type object at 0x557a216065a0>
__le__: <slot wrapper '__le__' of 'object' objects>
__lt__: <slot wrapper '__lt__' of 'object' objects>
__module__: __main__
__ne__: <slot wrapper '__ne__' of 'object' objects>
__new__: <built-in method __new__ of type object at 0x7f2a58370840>
__reduce__: <method '__reduce__' of 'object' objects>
__reduce_ex__: <method '__reduce_ex__' of 'object' objects>
__repr__: <slot wrapper '__repr__' of 'object' objects>
__setattr__: <slot wrapper '__setattr__' of 'object' objects>
__sizeof__: <method '__sizeof__' of 'object' objects>
__str__: <function ChildClass.__str__ at 0x7f295472fe50>
__subclasshook__: <built-in method __subclasshook__ of type object at 0x557a216065a0>
__weakref__: <attribute '__weakref__' of 'ParentClass' objects>
_protected_attr: a protected attribute
class_attr: a class attribute
class_method: <bound method ChildClass.class_method of <class '__main__.ChildClass'>>
instance_method: <function ChildClass.instance_method at 0x7f295472fca0>
parent_class_attr: a parent class attribute
parent_method: <function ParentClass.parent_method at 0x7f295472faf0>
read_only_attr: <property object at 0x7f2954735900>
static_method: <function ChildClass.static_method at 0x7f295472fdc0>
instance_child_all_members = get_members_of_function_or_method(instance_child, predicate=None)
loop_through_members(instance_child_all_members)
_ChildClass__private_attr: a private attribute
__class__: <class '__main__.ChildClass'>
__delattr__: <method-wrapper '__delattr__' of ChildClass object at 0x7f29547dbd60>
__dict__: {'parent_instance_attr': 'a parent instance attribute', 'instance_attr': 'an instance attribute', 'instance_not_in_constructor_attr': 'an instance attribute not in the constructor', '_private_instance_attr': 'a private instance attribute'}
__dir__: <built-in method __dir__ of ChildClass object at 0x7f29547dbd60>
__doc__: This is a subclass of ParentClass.
__eq__: <method-wrapper '__eq__' of ChildClass object at 0x7f29547dbd60>
__format__: <built-in method __format__ of ChildClass object at 0x7f29547dbd60>
__ge__: <method-wrapper '__ge__' of ChildClass object at 0x7f29547dbd60>
__getattribute__: <method-wrapper '__getattribute__' of ChildClass object at 0x7f29547dbd60>
__gt__: <method-wrapper '__gt__' of ChildClass object at 0x7f29547dbd60>
__hash__: <method-wrapper '__hash__' of ChildClass object at 0x7f29547dbd60>
__init__: <bound method ChildClass.__init__ of <__main__.ChildClass object at 0x7f29547dbd60>>
__init_subclass__: <built-in method __init_subclass__ of type object at 0x557a216065a0>
__le__: <method-wrapper '__le__' of ChildClass object at 0x7f29547dbd60>
__lt__: <method-wrapper '__lt__' of ChildClass object at 0x7f29547dbd60>
__module__: __main__
__ne__: <method-wrapper '__ne__' of ChildClass object at 0x7f29547dbd60>
__new__: <built-in method __new__ of type object at 0x7f2a58370840>
__reduce__: <built-in method __reduce__ of ChildClass object at 0x7f29547dbd60>
__reduce_ex__: <built-in method __reduce_ex__ of ChildClass object at 0x7f29547dbd60>
__repr__: <method-wrapper '__repr__' of ChildClass object at 0x7f29547dbd60>
__setattr__: <method-wrapper '__setattr__' of ChildClass object at 0x7f29547dbd60>
__sizeof__: <built-in method __sizeof__ of ChildClass object at 0x7f29547dbd60>
__str__: <bound method ChildClass.__str__ of <__main__.ChildClass object at 0x7f29547dbd60>>
__subclasshook__: <built-in method __subclasshook__ of type object at 0x557a216065a0>
__weakref__: None
_private_instance_attr: a private instance attribute
_protected_attr: a protected attribute
class_attr: a class attribute
class_method: <bound method ChildClass.class_method of <class '__main__.ChildClass'>>
instance_attr: an instance attribute
instance_method: <bound method ChildClass.instance_method of <__main__.ChildClass object at 0x7f29547dbd60>>
instance_not_in_constructor_attr: an instance attribute not in the constructor
parent_class_attr: a parent class attribute
parent_instance_attr: a parent instance attribute
parent_method: <bound method ParentClass.parent_method of <__main__.ChildClass object at 0x7f29547dbd60>>
read_only_attr: You can read me, but you cannot change me.
static_method: <function ChildClass.static_method at 0x7f295472fdc0>
trainer_all_members = get_members_of_function_or_method(Trainer, predicate=None)
loop_through_members(trainer_all_members)
__class__: <class 'type'>
__delattr__: <slot wrapper '__delattr__' of 'object' objects>
__dict__: {'__module__': 'transformers.trainer', '__doc__': "\n Trainer is a simple but feature-complete training and eval loop for PyTorch, optimized for 🤗 Transformers.\n\n Args:\n model ([`PreTrainedModel`] or `torch.nn.Module`, *optional*):\n The model to train, evaluate or use for predictions. If not provided, a `model_init` must be passed.\n\n <Tip>\n\n [`Trainer`] is optimized to work with the [`PreTrainedModel`] provided by the library. You can still use\n your own models defined as `torch.nn.Module` as long as they work the same way as the 🤗 Transformers\n models.\n\n </Tip>\n\n args ([`TrainingArguments`], *optional*):\n The arguments to tweak for training. Will default to a basic instance of [`TrainingArguments`] with the\n `output_dir` set to a directory named *tmp_trainer* in the current directory if not provided.\n data_collator (`DataCollator`, *optional*):\n The function to use to form a batch from a list of elements of `train_dataset` or `eval_dataset`. Will\n default to [`default_data_collator`] if no `tokenizer` is provided, an instance of\n [`DataCollatorWithPadding`] otherwise.\n train_dataset (`torch.utils.data.Dataset` or `torch.utils.data.IterableDataset`, *optional*):\n The dataset to use for training. If it is a [`~datasets.Dataset`], columns not accepted by the\n `model.forward()` method are automatically removed.\n\n Note that if it's a `torch.utils.data.IterableDataset` with some randomization and you are training in a\n distributed fashion, your iterable dataset should either use a internal attribute `generator` that is a\n `torch.Generator` for the randomization that must be identical on all processes (and the Trainer will\n manually set the seed of this `generator` at each epoch) or have a `set_epoch()` method that internally\n sets the seed of the RNGs used.\n eval_dataset (Union[`torch.utils.data.Dataset`, Dict[str, `torch.utils.data.Dataset`]), *optional*):\n The dataset to use for evaluation. If it is a [`~datasets.Dataset`], columns not accepted by the\n `model.forward()` method are automatically removed. If it is a dictionary, it will evaluate on each\n dataset prepending the dictionary key to the metric name.\n tokenizer ([`PreTrainedTokenizerBase`], *optional*):\n The tokenizer used to preprocess the data. If provided, will be used to automatically pad the inputs to the\n maximum length when batching inputs, and it will be saved along the model to make it easier to rerun an\n interrupted training or reuse the fine-tuned model.\n model_init (`Callable[[], PreTrainedModel]`, *optional*):\n A function that instantiates the model to be used. If provided, each call to [`~Trainer.train`] will start\n from a new instance of the model as given by this function.\n\n The function may have zero argument, or a single one containing the optuna/Ray Tune/SigOpt trial object, to\n be able to choose different architectures according to hyper parameters (such as layer count, sizes of\n inner layers, dropout probabilities etc).\n compute_metrics (`Callable[[EvalPrediction], Dict]`, *optional*):\n The function that will be used to compute metrics at evaluation. Must take a [`EvalPrediction`] and return\n a dictionary string to metric values.\n callbacks (List of [`TrainerCallback`], *optional*):\n A list of callbacks to customize the training loop. Will add those to the list of default callbacks\n detailed in [here](callback).\n\n If you want to remove one of the default callbacks used, use the [`Trainer.remove_callback`] method.\n optimizers (`Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR]`, *optional*, defaults to `(None, None)`):\n A tuple containing the optimizer and the scheduler to use. Will default to an instance of [`AdamW`] on your\n model and a scheduler given by [`get_linear_schedule_with_warmup`] controlled by `args`.\n preprocess_logits_for_metrics (`Callable[[torch.Tensor, torch.Tensor], torch.Tensor]`, *optional*):\n A function that preprocess the logits right before caching them at each evaluation step. Must take two\n tensors, the logits and the labels, and return the logits once processed as desired. The modifications made\n by this function will be reflected in the predictions received by `compute_metrics`.\n\n Note that the labels (second parameter) will be `None` if the dataset does not have them.\n\n Important attributes:\n\n - **model** -- Always points to the core model. If using a transformers model, it will be a [`PreTrainedModel`]\n subclass.\n - **model_wrapped** -- Always points to the most external model in case one or more other modules wrap the\n original model. This is the model that should be used for the forward pass. For example, under `DeepSpeed`,\n the inner model is wrapped in `DeepSpeed` and then again in `torch.nn.DistributedDataParallel`. If the inner\n model hasn't been wrapped, then `self.model_wrapped` is the same as `self.model`.\n - **is_model_parallel** -- Whether or not a model has been switched to a model parallel mode (different from\n data parallelism, this means some of the model layers are split on different GPUs).\n - **place_model_on_device** -- Whether or not to automatically place the model on the device - it will be set\n to `False` if model parallel or deepspeed is used, or if the default\n `TrainingArguments.place_model_on_device` is overridden to return `False` .\n - **is_in_train** -- Whether or not a model is currently running `train` (e.g. when `evaluate` is called while\n in `train`)\n\n ", '_get_learning_rate': <function _get_learning_rate at 0x7f296bd01790>, 'log_metrics': <function log_metrics at 0x7f296bd040d0>, 'metrics_format': <function metrics_format at 0x7f296bd04040>, 'save_metrics': <function save_metrics at 0x7f296bd04160>, 'save_state': <function save_state at 0x7f296bd041f0>, '__init__': <function Trainer.__init__ at 0x7f2954728f70>, '_activate_neftune': <function Trainer._activate_neftune at 0x7f295472b040>, '_deactivate_neftune': <function Trainer._deactivate_neftune at 0x7f295472b0d0>, 'add_callback': <function Trainer.add_callback at 0x7f295472b160>, 'pop_callback': <function Trainer.pop_callback at 0x7f295472b1f0>, 'remove_callback': <function Trainer.remove_callback at 0x7f295472b280>, '_move_model_to_device': <function Trainer._move_model_to_device at 0x7f295472b310>, '_set_signature_columns_if_needed': <function Trainer._set_signature_columns_if_needed at 0x7f295472b3a0>, '_remove_unused_columns': <function Trainer._remove_unused_columns at 0x7f295472b430>, '_get_collator_with_removed_columns': <function Trainer._get_collator_with_removed_columns at 0x7f295472b4c0>, '_get_train_sampler': <function Trainer._get_train_sampler at 0x7f295472b550>, 'get_train_dataloader': <function Trainer.get_train_dataloader at 0x7f295472b5e0>, '_get_eval_sampler': <function Trainer._get_eval_sampler at 0x7f295472b670>, 'get_eval_dataloader': <function Trainer.get_eval_dataloader at 0x7f295472b700>, 'get_test_dataloader': <function Trainer.get_test_dataloader at 0x7f295472b790>, 'create_optimizer_and_scheduler': <function Trainer.create_optimizer_and_scheduler at 0x7f295472b820>, 'get_decay_parameter_names': <function Trainer.get_decay_parameter_names at 0x7f295472b8b0>, 'create_optimizer': <function Trainer.create_optimizer at 0x7f295472b940>, 'get_optimizer_cls_and_kwargs': <staticmethod object at 0x7f295480a310>, 'create_scheduler': <function Trainer.create_scheduler at 0x7f295472ba60>, 'num_examples': <function Trainer.num_examples at 0x7f295472baf0>, 'num_tokens': <function Trainer.num_tokens at 0x7f295472bb80>, '_hp_search_setup': <function Trainer._hp_search_setup at 0x7f295472bc10>, '_report_to_hp_search': <function Trainer._report_to_hp_search at 0x7f295472bca0>, '_tune_save_checkpoint': <function Trainer._tune_save_checkpoint at 0x7f295472bd30>, 'call_model_init': <function Trainer.call_model_init at 0x7f295472bdc0>, 'torch_jit_model_eval': <function Trainer.torch_jit_model_eval at 0x7f295472be50>, 'ipex_optimize_model': <function Trainer.ipex_optimize_model at 0x7f295472bee0>, '_wrap_model': <function Trainer._wrap_model at 0x7f295472bf70>, 'train': <function Trainer.train at 0x7f295472d040>, '_inner_training_loop': <function Trainer._inner_training_loop at 0x7f295472d0d0>, '_get_output_dir': <function Trainer._get_output_dir at 0x7f295472d160>, '_load_from_checkpoint': <function Trainer._load_from_checkpoint at 0x7f295472d1f0>, '_load_best_model': <function Trainer._load_best_model at 0x7f295472d280>, '_issue_warnings_after_load': <function Trainer._issue_warnings_after_load at 0x7f295472d310>, '_maybe_log_save_evaluate': <function Trainer._maybe_log_save_evaluate at 0x7f295472d3a0>, '_load_rng_state': <function Trainer._load_rng_state at 0x7f295472d430>, '_save_checkpoint': <function Trainer._save_checkpoint at 0x7f295472d4c0>, '_save_rng_state': <function Trainer._save_rng_state at 0x7f295472d550>, '_save_optimizer_and_scheduler': <function Trainer._save_optimizer_and_scheduler at 0x7f295472d5e0>, '_load_optimizer_and_scheduler': <function Trainer._load_optimizer_and_scheduler at 0x7f295472d670>, 'hyperparameter_search': <function Trainer.hyperparameter_search at 0x7f295472d700>, 'log': <function Trainer.log at 0x7f295472d790>, '_prepare_input': <function Trainer._prepare_input at 0x7f295472d820>, '_prepare_inputs': <function Trainer._prepare_inputs at 0x7f295472d8b0>, 'compute_loss_context_manager': <function Trainer.compute_loss_context_manager at 0x7f295472d940>, 'autocast_smart_context_manager': <function Trainer.autocast_smart_context_manager at 0x7f295472d9d0>, 'training_step': <function Trainer.training_step at 0x7f295472da60>, 'compute_loss': <function Trainer.compute_loss at 0x7f295472daf0>, 'is_local_process_zero': <function Trainer.is_local_process_zero at 0x7f295472db80>, 'is_world_process_zero': <function Trainer.is_world_process_zero at 0x7f295472dc10>, 'save_model': <function Trainer.save_model at 0x7f295472dca0>, '_save_tpu': <function Trainer._save_tpu at 0x7f295472dd30>, '_save': <function Trainer._save at 0x7f295472ddc0>, 'store_flos': <function Trainer.store_flos at 0x7f295472de50>, '_sorted_checkpoints': <function Trainer._sorted_checkpoints at 0x7f295472dee0>, '_rotate_checkpoints': <function Trainer._rotate_checkpoints at 0x7f295472df70>, 'evaluate': <function Trainer.evaluate at 0x7f295472f040>, 'predict': <function Trainer.predict at 0x7f295472f0d0>, 'evaluation_loop': <function Trainer.evaluation_loop at 0x7f295472f160>, '_nested_gather': <function Trainer._nested_gather at 0x7f295472f1f0>, 'prediction_step': <function Trainer.prediction_step at 0x7f295472f280>, 'floating_point_ops': <function Trainer.floating_point_ops at 0x7f295472f310>, 'init_hf_repo': <function Trainer.init_hf_repo at 0x7f295472f3a0>, 'create_model_card': <function Trainer.create_model_card at 0x7f295472f430>, '_push_from_checkpoint': <function Trainer._push_from_checkpoint at 0x7f295472f4c0>, '_finish_current_push': <function Trainer._finish_current_push at 0x7f295472f550>, 'push_to_hub': <function Trainer.push_to_hub at 0x7f295472f5e0>, 'prediction_loop': <function Trainer.prediction_loop at 0x7f295472f670>, '_gather_and_numpify': <function Trainer._gather_and_numpify at 0x7f295472f700>, '_add_sm_patterns_to_gitignore': <function Trainer._add_sm_patterns_to_gitignore at 0x7f295472f790>, 'create_accelerator_and_postprocess': <function Trainer.create_accelerator_and_postprocess at 0x7f295472f820>, 'propagate_args_to_deepspeed': <function Trainer.propagate_args_to_deepspeed at 0x7f295472f8b0>, '__dict__': <attribute '__dict__' of 'Trainer' objects>, '__weakref__': <attribute '__weakref__' of 'Trainer' objects>}
__dir__: <method '__dir__' of 'object' objects>
__doc__:
Trainer is a simple but feature-complete training and eval loop for PyTorch, optimized for 🤗 Transformers.
Args:
model ([`PreTrainedModel`] or `torch.nn.Module`, *optional*):
The model to train, evaluate or use for predictions. If not provided, a `model_init` must be passed.
<Tip>
[`Trainer`] is optimized to work with the [`PreTrainedModel`] provided by the library. You can still use
your own models defined as `torch.nn.Module` as long as they work the same way as the 🤗 Transformers
models.
</Tip>
args ([`TrainingArguments`], *optional*):
The arguments to tweak for training. Will default to a basic instance of [`TrainingArguments`] with the
`output_dir` set to a directory named *tmp_trainer* in the current directory if not provided.
data_collator (`DataCollator`, *optional*):
The function to use to form a batch from a list of elements of `train_dataset` or `eval_dataset`. Will
default to [`default_data_collator`] if no `tokenizer` is provided, an instance of
[`DataCollatorWithPadding`] otherwise.
train_dataset (`torch.utils.data.Dataset` or `torch.utils.data.IterableDataset`, *optional*):
The dataset to use for training. If it is a [`~datasets.Dataset`], columns not accepted by the
`model.forward()` method are automatically removed.
Note that if it's a `torch.utils.data.IterableDataset` with some randomization and you are training in a
distributed fashion, your iterable dataset should either use a internal attribute `generator` that is a
`torch.Generator` for the randomization that must be identical on all processes (and the Trainer will
manually set the seed of this `generator` at each epoch) or have a `set_epoch()` method that internally
sets the seed of the RNGs used.
eval_dataset (Union[`torch.utils.data.Dataset`, Dict[str, `torch.utils.data.Dataset`]), *optional*):
The dataset to use for evaluation. If it is a [`~datasets.Dataset`], columns not accepted by the
`model.forward()` method are automatically removed. If it is a dictionary, it will evaluate on each
dataset prepending the dictionary key to the metric name.
tokenizer ([`PreTrainedTokenizerBase`], *optional*):
The tokenizer used to preprocess the data. If provided, will be used to automatically pad the inputs to the
maximum length when batching inputs, and it will be saved along the model to make it easier to rerun an
interrupted training or reuse the fine-tuned model.
model_init (`Callable[[], PreTrainedModel]`, *optional*):
A function that instantiates the model to be used. If provided, each call to [`~Trainer.train`] will start
from a new instance of the model as given by this function.
The function may have zero argument, or a single one containing the optuna/Ray Tune/SigOpt trial object, to
be able to choose different architectures according to hyper parameters (such as layer count, sizes of
inner layers, dropout probabilities etc).
compute_metrics (`Callable[[EvalPrediction], Dict]`, *optional*):
The function that will be used to compute metrics at evaluation. Must take a [`EvalPrediction`] and return
a dictionary string to metric values.
callbacks (List of [`TrainerCallback`], *optional*):
A list of callbacks to customize the training loop. Will add those to the list of default callbacks
detailed in [here](callback).
If you want to remove one of the default callbacks used, use the [`Trainer.remove_callback`] method.
optimizers (`Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR]`, *optional*, defaults to `(None, None)`):
A tuple containing the optimizer and the scheduler to use. Will default to an instance of [`AdamW`] on your
model and a scheduler given by [`get_linear_schedule_with_warmup`] controlled by `args`.
preprocess_logits_for_metrics (`Callable[[torch.Tensor, torch.Tensor], torch.Tensor]`, *optional*):
A function that preprocess the logits right before caching them at each evaluation step. Must take two
tensors, the logits and the labels, and return the logits once processed as desired. The modifications made
by this function will be reflected in the predictions received by `compute_metrics`.
Note that the labels (second parameter) will be `None` if the dataset does not have them.
Important attributes:
- **model** -- Always points to the core model. If using a transformers model, it will be a [`PreTrainedModel`]
subclass.
- **model_wrapped** -- Always points to the most external model in case one or more other modules wrap the
original model. This is the model that should be used for the forward pass. For example, under `DeepSpeed`,
the inner model is wrapped in `DeepSpeed` and then again in `torch.nn.DistributedDataParallel`. If the inner
model hasn't been wrapped, then `self.model_wrapped` is the same as `self.model`.
- **is_model_parallel** -- Whether or not a model has been switched to a model parallel mode (different from
data parallelism, this means some of the model layers are split on different GPUs).
- **place_model_on_device** -- Whether or not to automatically place the model on the device - it will be set
to `False` if model parallel or deepspeed is used, or if the default
`TrainingArguments.place_model_on_device` is overridden to return `False` .
- **is_in_train** -- Whether or not a model is currently running `train` (e.g. when `evaluate` is called while
in `train`)
__eq__: <slot wrapper '__eq__' of 'object' objects>
__format__: <method '__format__' of 'object' objects>
__ge__: <slot wrapper '__ge__' of 'object' objects>
__getattribute__: <slot wrapper '__getattribute__' of 'object' objects>
__gt__: <slot wrapper '__gt__' of 'object' objects>
__hash__: <slot wrapper '__hash__' of 'object' objects>
__init__: <function Trainer.__init__ at 0x7f2954728f70>
__init_subclass__: <built-in method __init_subclass__ of type object at 0x557a216be1c0>
__le__: <slot wrapper '__le__' of 'object' objects>
__lt__: <slot wrapper '__lt__' of 'object' objects>
__module__: transformers.trainer
__ne__: <slot wrapper '__ne__' of 'object' objects>
__new__: <built-in method __new__ of type object at 0x7f2a58370840>
__reduce__: <method '__reduce__' of 'object' objects>
__reduce_ex__: <method '__reduce_ex__' of 'object' objects>
__repr__: <slot wrapper '__repr__' of 'object' objects>
__setattr__: <slot wrapper '__setattr__' of 'object' objects>
__sizeof__: <method '__sizeof__' of 'object' objects>
__str__: <slot wrapper '__str__' of 'object' objects>
__subclasshook__: <built-in method __subclasshook__ of type object at 0x557a216be1c0>
__weakref__: <attribute '__weakref__' of 'Trainer' objects>
_activate_neftune: <function Trainer._activate_neftune at 0x7f295472b040>
_add_sm_patterns_to_gitignore: <function Trainer._add_sm_patterns_to_gitignore at 0x7f295472f790>
_deactivate_neftune: <function Trainer._deactivate_neftune at 0x7f295472b0d0>
_finish_current_push: <function Trainer._finish_current_push at 0x7f295472f550>
_gather_and_numpify: <function Trainer._gather_and_numpify at 0x7f295472f700>
_get_collator_with_removed_columns: <function Trainer._get_collator_with_removed_columns at 0x7f295472b4c0>
_get_eval_sampler: <function Trainer._get_eval_sampler at 0x7f295472b670>
_get_learning_rate: <function _get_learning_rate at 0x7f296bd01790>
_get_output_dir: <function Trainer._get_output_dir at 0x7f295472d160>
_get_train_sampler: <function Trainer._get_train_sampler at 0x7f295472b550>
_hp_search_setup: <function Trainer._hp_search_setup at 0x7f295472bc10>
_inner_training_loop: <function Trainer._inner_training_loop at 0x7f295472d0d0>
_issue_warnings_after_load: <function Trainer._issue_warnings_after_load at 0x7f295472d310>
_load_best_model: <function Trainer._load_best_model at 0x7f295472d280>
_load_from_checkpoint: <function Trainer._load_from_checkpoint at 0x7f295472d1f0>
_load_optimizer_and_scheduler: <function Trainer._load_optimizer_and_scheduler at 0x7f295472d670>
_load_rng_state: <function Trainer._load_rng_state at 0x7f295472d430>
_maybe_log_save_evaluate: <function Trainer._maybe_log_save_evaluate at 0x7f295472d3a0>
_move_model_to_device: <function Trainer._move_model_to_device at 0x7f295472b310>
_nested_gather: <function Trainer._nested_gather at 0x7f295472f1f0>
_prepare_input: <function Trainer._prepare_input at 0x7f295472d820>
_prepare_inputs: <function Trainer._prepare_inputs at 0x7f295472d8b0>
_push_from_checkpoint: <function Trainer._push_from_checkpoint at 0x7f295472f4c0>
_remove_unused_columns: <function Trainer._remove_unused_columns at 0x7f295472b430>
_report_to_hp_search: <function Trainer._report_to_hp_search at 0x7f295472bca0>
_rotate_checkpoints: <function Trainer._rotate_checkpoints at 0x7f295472df70>
_save: <function Trainer._save at 0x7f295472ddc0>
_save_checkpoint: <function Trainer._save_checkpoint at 0x7f295472d4c0>
_save_optimizer_and_scheduler: <function Trainer._save_optimizer_and_scheduler at 0x7f295472d5e0>
_save_rng_state: <function Trainer._save_rng_state at 0x7f295472d550>
_save_tpu: <function Trainer._save_tpu at 0x7f295472dd30>
_set_signature_columns_if_needed: <function Trainer._set_signature_columns_if_needed at 0x7f295472b3a0>
_sorted_checkpoints: <function Trainer._sorted_checkpoints at 0x7f295472dee0>
_tune_save_checkpoint: <function Trainer._tune_save_checkpoint at 0x7f295472bd30>
_wrap_model: <function Trainer._wrap_model at 0x7f295472bf70>
add_callback: <function Trainer.add_callback at 0x7f295472b160>
autocast_smart_context_manager: <function Trainer.autocast_smart_context_manager at 0x7f295472d9d0>
call_model_init: <function Trainer.call_model_init at 0x7f295472bdc0>
compute_loss: <function Trainer.compute_loss at 0x7f295472daf0>
compute_loss_context_manager: <function Trainer.compute_loss_context_manager at 0x7f295472d940>
create_accelerator_and_postprocess: <function Trainer.create_accelerator_and_postprocess at 0x7f295472f820>
create_model_card: <function Trainer.create_model_card at 0x7f295472f430>
create_optimizer: <function Trainer.create_optimizer at 0x7f295472b940>
create_optimizer_and_scheduler: <function Trainer.create_optimizer_and_scheduler at 0x7f295472b820>
create_scheduler: <function Trainer.create_scheduler at 0x7f295472ba60>
evaluate: <function Trainer.evaluate at 0x7f295472f040>
evaluation_loop: <function Trainer.evaluation_loop at 0x7f295472f160>
floating_point_ops: <function Trainer.floating_point_ops at 0x7f295472f310>
get_decay_parameter_names: <function Trainer.get_decay_parameter_names at 0x7f295472b8b0>
get_eval_dataloader: <function Trainer.get_eval_dataloader at 0x7f295472b700>
get_optimizer_cls_and_kwargs: <function Trainer.get_optimizer_cls_and_kwargs at 0x7f295472b9d0>
get_test_dataloader: <function Trainer.get_test_dataloader at 0x7f295472b790>
get_train_dataloader: <function Trainer.get_train_dataloader at 0x7f295472b5e0>
hyperparameter_search: <function Trainer.hyperparameter_search at 0x7f295472d700>
init_hf_repo: <function Trainer.init_hf_repo at 0x7f295472f3a0>
ipex_optimize_model: <function Trainer.ipex_optimize_model at 0x7f295472bee0>
is_local_process_zero: <function Trainer.is_local_process_zero at 0x7f295472db80>
is_world_process_zero: <function Trainer.is_world_process_zero at 0x7f295472dc10>
log: <function Trainer.log at 0x7f295472d790>
log_metrics: <function log_metrics at 0x7f296bd040d0>
metrics_format: <function metrics_format at 0x7f296bd04040>
num_examples: <function Trainer.num_examples at 0x7f295472baf0>
num_tokens: <function Trainer.num_tokens at 0x7f295472bb80>
pop_callback: <function Trainer.pop_callback at 0x7f295472b1f0>
predict: <function Trainer.predict at 0x7f295472f0d0>
prediction_loop: <function Trainer.prediction_loop at 0x7f295472f670>
prediction_step: <function Trainer.prediction_step at 0x7f295472f280>
propagate_args_to_deepspeed: <function Trainer.propagate_args_to_deepspeed at 0x7f295472f8b0>
push_to_hub: <function Trainer.push_to_hub at 0x7f295472f5e0>
remove_callback: <function Trainer.remove_callback at 0x7f295472b280>
save_metrics: <function save_metrics at 0x7f296bd04160>
save_model: <function Trainer.save_model at 0x7f295472dca0>
save_state: <function save_state at 0x7f296bd041f0>
store_flos: <function Trainer.store_flos at 0x7f295472de50>
torch_jit_model_eval: <function Trainer.torch_jit_model_eval at 0x7f295472be50>
train: <function Trainer.train at 0x7f295472d040>
training_step: <function Trainer.training_step at 0x7f295472da60>
Retrieve All Methods of a Class#
There are a few ways to do it.
Using __dict__#
child_class_methods_using_dict = list(ChildClass.__dict__.keys())
pprint(sorted(child_class_methods_using_dict))
assert 'parent_method' not in child_class_methods_using_dict
assert 'read_only_attr' in child_class_methods_using_dict
assert 'class_method' in child_class_methods_using_dict
[ │ '_ChildClass__private_attr', │ '__doc__', │ '__init__', │ '__module__', │ '__str__', │ '_protected_attr', │ 'class_attr', │ 'class_method', │ 'instance_method', │ 'read_only_attr', │ 'static_method' ]
Notice that the parent class methods are not included!
pprint(instance_child.__class__.__dict__.keys() == ChildClass.__dict__.keys())
True
Using vars#
vars and __dict__ are equivalent, but people are preferring the former due
to some efficiency reasons, which can be found in this post.
child_class_methods_using_vars = list(vars(ChildClass).keys())
pprint(sorted(child_class_methods_using_vars))
assert 'parent_method' not in child_class_methods_using_vars
assert 'read_only_attr' in child_class_methods_using_vars
assert 'class_method' in child_class_methods_using_vars
assert set(child_class_methods_using_dict) == set(child_class_methods_using_vars)
[ │ '_ChildClass__private_attr', │ '__doc__', │ '__init__', │ '__module__', │ '__str__', │ '_protected_attr', │ 'class_attr', │ 'class_method', │ 'instance_method', │ 'read_only_attr', │ 'static_method' ]
Using dir#
To include the base/parent class methods, we can use dir instead.
child_class_methods_using_dir = list(dir(ChildClass))
pprint(sorted(child_class_methods_using_dir))
assert 'parent_method' in child_class_methods_using_dir
assert 'read_only_attr' in child_class_methods_using_dir
assert 'class_method' in child_class_methods_using_dir
[ │ '_ChildClass__private_attr', │ '__class__', │ '__delattr__', │ '__dict__', │ '__dir__', │ '__doc__', │ '__eq__', │ '__format__', │ '__ge__', │ '__getattribute__', │ '__gt__', │ '__hash__', │ '__init__', │ '__init_subclass__', │ '__le__', │ '__lt__', │ '__module__', │ '__ne__', │ '__new__', │ '__reduce__', │ '__reduce_ex__', │ '__repr__', │ '__setattr__', │ '__sizeof__', │ '__str__', │ '__subclasshook__', │ '__weakref__', │ '_protected_attr', │ 'class_attr', │ 'class_method', │ 'instance_method', │ 'parent_class_attr', │ 'parent_method', │ 'read_only_attr', │ 'static_method' ]
Using inspect.getmembers#
We use inspect.getmembers to get all members of a class, and then filter out
via the predicate inspect.isroutine, a stronger filter than inspect.isfunction
or inspect.ismethod.
We attach the source code of inspect.isroutine here for reference.
def isroutine(object):
"""Return true if the object is any kind of function or method."""
return (isbuiltin(object)
or isfunction(object)
or ismethod(object)
or ismethoddescriptor(object))
predicate = inspect.isroutine
child_class_methods_using_getmembers = list(get_members_of_function_or_method(ChildClass, predicate=predicate))
pprint(sorted(child_class_methods_using_getmembers))
[ │ ('__delattr__', <slot wrapper '__delattr__' of 'object' objects>), │ ('__dir__', <method '__dir__' of 'object' objects>), │ ('__eq__', <slot wrapper '__eq__' of 'object' objects>), │ ('__format__', <method '__format__' of 'object' objects>), │ ('__ge__', <slot wrapper '__ge__' of 'object' objects>), │ ('__getattribute__', <slot wrapper '__getattribute__' of 'object' objects>), │ ('__gt__', <slot wrapper '__gt__' of 'object' objects>), │ ('__hash__', <slot wrapper '__hash__' of 'object' objects>), │ ('__init__', <function ChildClass.__init__ at 0x7f295472fb80>), │ ('__init_subclass__', <built-in method __init_subclass__ of type object at 0x557a216065a0>), │ ('__le__', <slot wrapper '__le__' of 'object' objects>), │ ('__lt__', <slot wrapper '__lt__' of 'object' objects>), │ ('__ne__', <slot wrapper '__ne__' of 'object' objects>), │ ('__new__', <built-in method __new__ of type object at 0x7f2a58370840>), │ ('__reduce__', <method '__reduce__' of 'object' objects>), │ ('__reduce_ex__', <method '__reduce_ex__' of 'object' objects>), │ ('__repr__', <slot wrapper '__repr__' of 'object' objects>), │ ('__setattr__', <slot wrapper '__setattr__' of 'object' objects>), │ ('__sizeof__', <method '__sizeof__' of 'object' objects>), │ ('__str__', <function ChildClass.__str__ at 0x7f295472fe50>), │ ('__subclasshook__', <built-in method __subclasshook__ of type object at 0x557a216065a0>), │ ('class_method', <bound method ChildClass.class_method of <class '__main__.ChildClass'>>), │ ('instance_method', <function ChildClass.instance_method at 0x7f295472fca0>), │ ('parent_method', <function ParentClass.parent_method at 0x7f295472faf0>), │ ('static_method', <function ChildClass.static_method at 0x7f295472fdc0>) ]
Of course, the reason to retrieve all methods is a convenience if we want to inspect all methods at once. And if we can obtain all methods, we can then iteratively inspect each method’s signature.
Method Resolution Order#
The above examples do not take into account complicated cases, such as when the class is a subclass of multiple classes, in which case if you just print out the methods of the class, you will have a hard time to know which methods are from which class. You can do so via more filtering, but this is beyond the scope of this notebook.
predicate = inspect.isroutine
GPT2LMHeadModel_methods_using_getmembers = list(get_members_of_function_or_method(GPT2LMHeadModel, predicate=predicate))
pprint(sorted(GPT2LMHeadModel_methods_using_getmembers))
[ │ ('__call__', <function Module._wrapped_call_impl at 0x7f2a1a9a3280>), │ ('__delattr__', <function Module.__delattr__ at 0x7f2a1a9a35e0>), │ ('__dir__', <function Module.__dir__ at 0x7f2a1a9aa4c0>), │ ('__eq__', <slot wrapper '__eq__' of 'object' objects>), │ ('__format__', <method '__format__' of 'object' objects>), │ ('__ge__', <slot wrapper '__ge__' of 'object' objects>), │ ('__getattr__', <function Module.__getattr__ at 0x7f2a1a9a34c0>), │ ('__getattribute__', <slot wrapper '__getattribute__' of 'object' objects>), │ ('__getstate__', <function Module.__getstate__ at 0x7f2a1a9a33a0>), │ ('__gt__', <slot wrapper '__gt__' of 'object' objects>), │ ('__hash__', <slot wrapper '__hash__' of 'object' objects>), │ ('__init__', <function GPT2LMHeadModel.__init__ at 0x7f296bdf6040>), │ ('__init_subclass__', <built-in method __init_subclass__ of type object at 0x557a215fd6f0>), │ ('__le__', <slot wrapper '__le__' of 'object' objects>), │ ('__lt__', <slot wrapper '__lt__' of 'object' objects>), │ ('__ne__', <slot wrapper '__ne__' of 'object' objects>), │ ('__new__', <built-in method __new__ of type object at 0x7f2a58370840>), │ ('__reduce__', <method '__reduce__' of 'object' objects>), │ ('__reduce_ex__', <method '__reduce_ex__' of 'object' objects>), │ ('__repr__', <function Module.__repr__ at 0x7f2a1a9aa430>), │ ('__setattr__', <function Module.__setattr__ at 0x7f2a1a9a3550>), │ ('__setstate__', <function Module.__setstate__ at 0x7f2a1a9a3430>), │ ('__sizeof__', <method '__sizeof__' of 'object' objects>), │ ('__str__', <slot wrapper '__str__' of 'object' objects>), │ ('__subclasshook__', <built-in method __subclasshook__ of type object at 0x557a215fd6f0>), │ ('_apply', <function Module._apply at 0x7f2a1a9a15e0>), │ ( │ │ '_autoset_attn_implementation', │ │ <bound method PreTrainedModel._autoset_attn_implementation of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ( │ │ '_backward_compatibility_gradient_checkpointing', │ │ <function PreTrainedModel._backward_compatibility_gradient_checkpointing at 0x7f296c468dc0> │ ), │ ('_call_impl', <function Module._call_impl at 0x7f2a1a9a3310>), │ ( │ │ '_check_and_enable_flash_attn_2', │ │ <bound method PreTrainedModel._check_and_enable_flash_attn_2 of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ( │ │ '_check_and_enable_sdpa', │ │ <bound method PreTrainedModel._check_and_enable_sdpa of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ('_convert_head_mask_to_5d', <function ModuleUtilsMixin._convert_head_mask_to_5d at 0x7f296c468940>), │ ( │ │ '_copy_lm_head_original_to_resized', │ │ <function PreTrainedModel._copy_lm_head_original_to_resized at 0x7f296c46bb80> │ ), │ ('_create_repo', <function PushToHubMixin._create_repo at 0x7f296f137790>), │ ('_dispatch_accelerate_model', <function PeftAdapterMixin._dispatch_accelerate_model at 0x7f296c4384c0>), │ ('_expand_inputs_for_generation', <function GenerationMixin._expand_inputs_for_generation at 0x7f296c4b2a60>), │ ( │ │ '_extract_past_from_model_output', │ │ <function GenerationMixin._extract_past_from_model_output at 0x7f296c4b2af0> │ ), │ ( │ │ '_from_config', │ │ <bound method PreTrainedModel._from_config of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ('_get_backward_hooks', <function Module._get_backward_hooks at 0x7f2a1a9a1ee0>), │ ('_get_backward_pre_hooks', <function Module._get_backward_pre_hooks at 0x7f2a1a9a1f70>), │ ('_get_candidate_generator', <function GenerationMixin._get_candidate_generator at 0x7f296c4b2ca0>), │ ('_get_decoder_start_token_id', <function GenerationMixin._get_decoder_start_token_id at 0x7f296c4b29d0>), │ ('_get_files_timestamps', <function PushToHubMixin._get_files_timestamps at 0x7f296f137820>), │ ('_get_generation_mode', <function GenerationMixin._get_generation_mode at 0x7f296c4b2dc0>), │ ('_get_logits_processor', <function GenerationMixin._get_logits_processor at 0x7f296c4b2e50>), │ ('_get_logits_warper', <function GenerationMixin._get_logits_warper at 0x7f296c4b2d30>), │ ('_get_name', <function Module._get_name at 0x7f2a1a9aa310>), │ ('_get_no_split_modules', <function PreTrainedModel._get_no_split_modules at 0x7f296c46b8b0>), │ ('_get_resized_embeddings', <function PreTrainedModel._get_resized_embeddings at 0x7f296c46ba60>), │ ('_get_resized_lm_head', <function PreTrainedModel._get_resized_lm_head at 0x7f296c46baf0>), │ ('_get_stopping_criteria', <function GenerationMixin._get_stopping_criteria at 0x7f296c4b2ee0>), │ ( │ │ '_hook_rss_memory_post_forward', │ │ <function ModuleUtilsMixin._hook_rss_memory_post_forward at 0x7f296c468430> │ ), │ ('_hook_rss_memory_pre_forward', <function ModuleUtilsMixin._hook_rss_memory_pre_forward at 0x7f296c4683a0>), │ ('_init_weights', <function GPT2PreTrainedModel._init_weights at 0x7f296bdf1820>), │ ('_initialize_weights', <function PreTrainedModel._initialize_weights at 0x7f296c46b670>), │ ('_load_from_state_dict', <function Module._load_from_state_dict at 0x7f2a1a9a39d0>), │ ( │ │ '_load_pretrained_model', │ │ <bound method PreTrainedModel._load_pretrained_model of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ( │ │ '_load_pretrained_model_low_mem', │ │ <function PreTrainedModel._load_pretrained_model_low_mem at 0x7f296c46d670> │ ), │ ( │ │ '_maybe_initialize_input_ids_for_generation', │ │ <function GenerationMixin._maybe_initialize_input_ids_for_generation at 0x7f296c4b2790> │ ), │ ( │ │ '_maybe_warn_non_full_backward_hook', │ │ <function Module._maybe_warn_non_full_backward_hook at 0x7f2a1a9a3040> │ ), │ ( │ │ '_merge_criteria_processor_list', │ │ <function GenerationMixin._merge_criteria_processor_list at 0x7f296c4b2f70> │ ), │ ('_named_members', <function Module._named_members at 0x7f2a1a9a3af0>), │ ( │ │ '_prepare_attention_mask_for_generation', │ │ <function GenerationMixin._prepare_attention_mask_for_generation at 0x7f296c4b2820> │ ), │ ( │ │ '_prepare_decoder_input_ids_for_generation', │ │ <function GenerationMixin._prepare_decoder_input_ids_for_generation at 0x7f296c4b2940> │ ), │ ( │ │ '_prepare_encoder_decoder_kwargs_for_generation', │ │ <function GenerationMixin._prepare_encoder_decoder_kwargs_for_generation at 0x7f296c4b28b0> │ ), │ ('_prepare_model_inputs', <function GenerationMixin._prepare_model_inputs at 0x7f296c4b2700>), │ ( │ │ '_register_load_state_dict_pre_hook', │ │ <function Module._register_load_state_dict_pre_hook at 0x7f2a1a9a38b0> │ ), │ ('_register_state_dict_hook', <function Module._register_state_dict_hook at 0x7f2a1a9a3670>), │ ('_reorder_cache', <function GPT2LMHeadModel._reorder_cache at 0x7f296bdf6430>), │ ('_replicate_for_data_parallel', <function Module._replicate_for_data_parallel at 0x7f2a1a9aa550>), │ ('_resize_token_embeddings', <function PreTrainedModel._resize_token_embeddings at 0x7f296c46b9d0>), │ ('_save_to_state_dict', <function Module._save_to_state_dict at 0x7f2a1a9a3790>), │ ( │ │ '_set_default_torch_dtype', │ │ <bound method PreTrainedModel._set_default_torch_dtype of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ('_set_gradient_checkpointing', <function PreTrainedModel._set_gradient_checkpointing at 0x7f296c46bee0>), │ ('_slow_forward', <function Module._slow_forward at 0x7f2a1a9a31f0>), │ ('_temporary_reorder_cache', <function GenerationMixin._temporary_reorder_cache at 0x7f296c4b45e0>), │ ('_tie_encoder_decoder_weights', <function PreTrainedModel._tie_encoder_decoder_weights at 0x7f296c46b790>), │ ('_tie_or_clone_weights', <function PreTrainedModel._tie_or_clone_weights at 0x7f296c46b820>), │ ( │ │ '_update_model_kwargs_for_generation', │ │ <function GenerationMixin._update_model_kwargs_for_generation at 0x7f296c4b2b80> │ ), │ ('_upload_modified_files', <function PushToHubMixin._upload_modified_files at 0x7f296f1378b0>), │ ('_validate_generated_length', <function GenerationMixin._validate_generated_length at 0x7f296c4b41f0>), │ ('_validate_model_class', <function GenerationMixin._validate_model_class at 0x7f296c4b40d0>), │ ('_validate_model_kwargs', <function GenerationMixin._validate_model_kwargs at 0x7f296c4b4160>), │ ('_wrapped_call_impl', <function Module._wrapped_call_impl at 0x7f2a1a9a3280>), │ ('active_adapter', <function PeftAdapterMixin.active_adapter at 0x7f296c4383a0>), │ ('active_adapters', <function PeftAdapterMixin.active_adapters at 0x7f296c438310>), │ ('add_adapter', <function PeftAdapterMixin.add_adapter at 0x7f296c4380d0>), │ ('add_memory_hooks', <function ModuleUtilsMixin.add_memory_hooks at 0x7f296c4684c0>), │ ('add_model_tags', <function PreTrainedModel.add_model_tags at 0x7f296c468e50>), │ ('add_module', <function Module.add_module at 0x7f2a1a9a11f0>), │ ('apply', <function Module.apply at 0x7f2a1a9a1670>), │ ('assisted_decoding', <function GenerationMixin.assisted_decoding at 0x7f296c4b48b0>), │ ('beam_sample', <function GenerationMixin.beam_sample at 0x7f296c4b4700>), │ ('beam_search', <function GenerationMixin.beam_search at 0x7f296c4b4670>), │ ('bfloat16', <function Module.bfloat16 at 0x7f2a1a9a1b80>), │ ('buffers', <function Module.buffers at 0x7f2a1a9a3ca0>), │ ( │ │ 'can_generate', │ │ <bound method PreTrainedModel.can_generate of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ('children', <function Module.children at 0x7f2a1a9a3dc0>), │ ('compile', <function Module.compile at 0x7f2a1a9aa5e0>), │ ('compute_transition_scores', <function GenerationMixin.compute_transition_scores at 0x7f296c4b4040>), │ ('constrained_beam_search', <function GenerationMixin.constrained_beam_search at 0x7f296c4b4820>), │ ('contrastive_search', <function GenerationMixin.contrastive_search at 0x7f296c4b4430>), │ ('cpu', <function Module.cpu at 0x7f2a1a9a18b0>), │ ( │ │ 'create_extended_attention_mask_for_decoder', │ │ <function ModuleUtilsMixin.create_extended_attention_mask_for_decoder at 0x7f296c468790> │ ), │ ('cuda', <function Module.cuda at 0x7f296c46d280>), │ ('deparallelize', <function GPT2LMHeadModel.deparallelize at 0x7f296bdf6280>), │ ('disable_adapters', <function PeftAdapterMixin.disable_adapters at 0x7f296c4381f0>), │ ('disable_input_require_grads', <function PreTrainedModel.disable_input_require_grads at 0x7f296c46b3a0>), │ ('double', <function Module.double at 0x7f2a1a9a1a60>), │ ('enable_adapters', <function PeftAdapterMixin.enable_adapters at 0x7f296c438280>), │ ('enable_input_require_grads', <function PreTrainedModel.enable_input_require_grads at 0x7f296c46b310>), │ ('estimate_tokens', <function ModuleUtilsMixin.estimate_tokens at 0x7f296c468a60>), │ ('eval', <function Module.eval at 0x7f2a1a9aa0d0>), │ ('extra_repr', <function Module.extra_repr at 0x7f2a1a9aa3a0>), │ ('float', <function PreTrainedModel.float at 0x7f296c46d430>), │ ('floating_point_ops', <function ModuleUtilsMixin.floating_point_ops at 0x7f296c468af0>), │ ('forward', <function GPT2LMHeadModel.forward at 0x7f296bdf6550>), │ ( │ │ 'from_pretrained', │ │ <bound method PreTrainedModel.from_pretrained of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ('generate', <function GenerationMixin.generate at 0x7f296c4b4310>), │ ('get_adapter_state_dict', <function PeftAdapterMixin.get_adapter_state_dict at 0x7f296c438430>), │ ('get_buffer', <function Module.get_buffer at 0x7f2a1a9a1430>), │ ('get_extended_attention_mask', <function ModuleUtilsMixin.get_extended_attention_mask at 0x7f296c468820>), │ ('get_extra_state', <function Module.get_extra_state at 0x7f2a1a9a14c0>), │ ('get_head_mask', <function ModuleUtilsMixin.get_head_mask at 0x7f296c4688b0>), │ ('get_input_embeddings', <function PreTrainedModel.get_input_embeddings at 0x7f296c46b430>), │ ('get_memory_footprint', <function PreTrainedModel.get_memory_footprint at 0x7f296c46d1f0>), │ ('get_output_embeddings', <function GPT2LMHeadModel.get_output_embeddings at 0x7f296bdf60d0>), │ ('get_parameter', <function Module.get_parameter at 0x7f2a1a9a13a0>), │ ('get_position_embeddings', <function PreTrainedModel.get_position_embeddings at 0x7f296c46bca0>), │ ('get_submodule', <function Module.get_submodule at 0x7f2a1a9a1310>), │ ( │ │ 'gradient_checkpointing_disable', │ │ <function PreTrainedModel.gradient_checkpointing_disable at 0x7f296c46bf70> │ ), │ ('gradient_checkpointing_enable', <function PreTrainedModel.gradient_checkpointing_enable at 0x7f296c46be50>), │ ('greedy_search', <function GenerationMixin.greedy_search at 0x7f296c4b44c0>), │ ('group_beam_search', <function GenerationMixin.group_beam_search at 0x7f296c4b4790>), │ ('half', <function PreTrainedModel.half at 0x7f296c46d3a0>), │ ('init_weights', <function PreTrainedModel.init_weights at 0x7f296c46bd30>), │ ('invert_attention_mask', <function ModuleUtilsMixin.invert_attention_mask at 0x7f296c468700>), │ ('ipu', <function Module.ipu at 0x7f2a1a9a1790>), │ ('load_adapter', <function PeftAdapterMixin.load_adapter at 0x7f296c438040>), │ ('load_state_dict', <function Module.load_state_dict at 0x7f2a1a9a3a60>), │ ('load_tf_weights', <function load_tf_weights_in_gpt2 at 0x7f296eebe040>), │ ('modules', <function Module.modules at 0x7f2a1a9a3ee0>), │ ('named_buffers', <function Module.named_buffers at 0x7f2a1a9a3d30>), │ ('named_children', <function Module.named_children at 0x7f2a1a9a3e50>), │ ('named_modules', <function Module.named_modules at 0x7f2a1a9a3f70>), │ ('named_parameters', <function Module.named_parameters at 0x7f2a1a9a3c10>), │ ('num_parameters', <function ModuleUtilsMixin.num_parameters at 0x7f296c4689d0>), │ ('parallelize', <function GPT2LMHeadModel.parallelize at 0x7f296bdf61f0>), │ ('parameters', <function Module.parameters at 0x7f2a1a9a3b80>), │ ('post_init', <function PreTrainedModel.post_init at 0x7f296c468d30>), │ ('prepare_inputs_for_generation', <function GPT2LMHeadModel.prepare_inputs_for_generation at 0x7f296bdf63a0>), │ ('prune_heads', <function PreTrainedModel.prune_heads at 0x7f296c46bdc0>), │ ('push_to_hub', <function PushToHubMixin.push_to_hub at 0x7f296c468310>), │ ('register_backward_hook', <function Module.register_backward_hook at 0x7f2a1a9a1dc0>), │ ('register_buffer', <function Module.register_buffer at 0x7f2a1a9a10d0>), │ ( │ │ 'register_for_auto_class', │ │ <bound method PreTrainedModel.register_for_auto_class of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ('register_forward_hook', <function Module.register_forward_hook at 0x7f2a1a9a3160>), │ ('register_forward_pre_hook', <function Module.register_forward_pre_hook at 0x7f2a1a9a30d0>), │ ('register_full_backward_hook', <function Module.register_full_backward_hook at 0x7f2a1a9a1e50>), │ ('register_full_backward_pre_hook', <function Module.register_full_backward_pre_hook at 0x7f2a1a9a1d30>), │ ( │ │ 'register_load_state_dict_post_hook', │ │ <function Module.register_load_state_dict_post_hook at 0x7f2a1a9a3940> │ ), │ ('register_module', <function Module.register_module at 0x7f2a1a9a1280>), │ ('register_parameter', <function Module.register_parameter at 0x7f2a1a9a1160>), │ ('register_state_dict_pre_hook', <function Module.register_state_dict_pre_hook at 0x7f2a1a9a3700>), │ ('requires_grad_', <function Module.requires_grad_ at 0x7f2a1a9aa160>), │ ('reset_memory_hooks_state', <function ModuleUtilsMixin.reset_memory_hooks_state at 0x7f296c468550>), │ ('resize_position_embeddings', <function PreTrainedModel.resize_position_embeddings at 0x7f296c46bc10>), │ ('resize_token_embeddings', <function PreTrainedModel.resize_token_embeddings at 0x7f296c46b940>), │ ('retrieve_modules_from_names', <function PreTrainedModel.retrieve_modules_from_names at 0x7f296c46d5e0>), │ ('reverse_bettertransformer', <function PreTrainedModel.reverse_bettertransformer at 0x7f296c46d820>), │ ('sample', <function GenerationMixin.sample at 0x7f296c4b4550>), │ ('save_pretrained', <function PreTrainedModel.save_pretrained at 0x7f296c46d0d0>), │ ('set_adapter', <function PeftAdapterMixin.set_adapter at 0x7f296c438160>), │ ('set_extra_state', <function Module.set_extra_state at 0x7f2a1a9a1550>), │ ('set_input_embeddings', <function PreTrainedModel.set_input_embeddings at 0x7f296c46b4c0>), │ ('set_output_embeddings', <function GPT2LMHeadModel.set_output_embeddings at 0x7f296bdf6310>), │ ('share_memory', <function Module.share_memory at 0x7f2a1a9aa280>), │ ('state_dict', <function Module.state_dict at 0x7f2a1a9a3820>), │ ('tie_weights', <function PreTrainedModel.tie_weights at 0x7f296c46b700>), │ ('to', <function Module.to at 0x7f296c46d310>), │ ('to_bettertransformer', <function PreTrainedModel.to_bettertransformer at 0x7f296c46d790>), │ ('to_empty', <function Module.to_empty at 0x7f2a1a9a1c10>), │ ('train', <function Module.train at 0x7f2a1a9aa040>), │ ('type', <function Module.type at 0x7f2a1a9a1940>), │ ( │ │ 'warn_if_padding_and_no_attention_mask', │ │ <function PreTrainedModel.warn_if_padding_and_no_attention_mask at 0x7f296c46d8b0> │ ), │ ('xpu', <function Module.xpu at 0x7f2a1a9a1820>), │ ('zero_grad', <function Module.zero_grad at 0x7f2a1a9aa1f0>) ]
You can get the method resolution order (MRO) of a class via cls.__mro__.
inspect.getmro(GPT2LMHeadModel)
(transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel,
transformers.models.gpt2.modeling_gpt2.GPT2PreTrainedModel,
transformers.modeling_utils.PreTrainedModel,
torch.nn.modules.module.Module,
transformers.modeling_utils.ModuleUtilsMixin,
transformers.generation.utils.GenerationMixin,
transformers.utils.hub.PushToHubMixin,
transformers.integrations.peft.PeftAdapterMixin,
object)
A pseudocode to get all signatures of a class via MRO is as follows:
def get_all_args(cls: Type[object]) -> Dict[str, inspect.Parameter]:
mro = inspect.getmro(cls)
all_args = {}
for base_class in mro[::-1]: # reverse to start from topmost class
if base_class is object: # skip the base 'object' class
continue
sig = inspect.signature(base_class.__init__)
all_args.update(sig.parameters)
return all_args
Get Class and Instance Attributes#
pprint(list(class_child.__dict__.keys())) # class attributes
pprint(list(instance_child.__dict__.keys())) # instance attributes
[ │ '__module__', │ '__doc__', │ 'class_attr', │ '_protected_attr', │ '_ChildClass__private_attr', │ '__init__', │ 'read_only_attr', │ 'instance_method', │ 'class_method', │ 'static_method', │ '__str__' ]
['parent_instance_attr', 'instance_attr', 'instance_not_in_constructor_attr', '_private_instance_attr']
union_class_and_instance_attributes = list(set(class_child.__dict__.keys()).union(set(instance_child.__dict__.keys())))
pprint(union_class_and_instance_attributes)
[ │ '_ChildClass__private_attr', │ 'static_method', │ '__init__', │ 'read_only_attr', │ '_protected_attr', │ 'instance_attr', │ 'parent_instance_attr', │ 'instance_method', │ '__module__', │ 'class_method', │ 'instance_not_in_constructor_attr', │ '_private_instance_attr', │ '__str__', │ '__doc__', │ 'class_attr' ]
Get Signature and Type Annotations of a Function#
func_sig: Signature = inspect.signature(func)
pprint(func_sig.parameters)
pprint(func_sig.return_annotation)
mappingproxy({ │ 'a': <Parameter "a: int">, │ 'b': <Parameter "b: str">, │ 'c': <Parameter "c: List[int]">, │ 'd': <Parameter "d: Tuple[str, str]">, │ 'e': <Parameter "e: Union[int, str]">, │ 'kwargs': <Parameter "**kwargs: Any"> })
<class 'str'>
Here are the 4 key properties of the Parameter object
of the Signature object.
@property
def name(self):
return self._name
@property
def default(self):
return self._default
@property
def annotation(self):
return self._annotation
@property
def kind(self):
return self._kind
We will also use get_type_hints to get the type hints of a function
instead of using the annotations property of inspect.Signature. The reason
can be found in the docstring of get_type_hints:
def get_type_hints(obj, globalns=None, localns=None, include_extras=False):
"""Return type hints for an object.
This is often the same as obj.__annotations__, but it handles
forward references encoded as string literals, adds Optional[t] if a
default value equal to None is set and recursively replaces all
'Annotated[T, ...]' with 'T' (unless 'include_extras=True').
...
"""
def no_type_hints(a, b, c, d, e, **kwargs):
return a, b, c, d, e, kwargs
get_type_hints(no_type_hints), inspect.signature(no_type_hints)
({}, <Signature (a, b, c, d, e, **kwargs)>)
How to know if a parameter is optional or not? We can use the inspect.Parameter.empty
property.
for name, value in inspect.signature(func).parameters.items():
print(value.default)
print(value.default is inspect.Parameter.empty)
<class 'inspect._empty'>
True
<class 'inspect._empty'>
True
<class 'inspect._empty'>
True
<class 'inspect._empty'>
True
<class 'inspect._empty'>
True
<class 'inspect._empty'>
True
We will also use __mro__ to get the method resolution order of a class
because __bases__ only returns the immediate parent class.
ChildClass.__bases__, GPT2LMHeadModel.__bases__[0].__bases__[0].__bases__
((__main__.ParentClass,),
(torch.nn.modules.module.Module,
transformers.modeling_utils.ModuleUtilsMixin,
transformers.generation.utils.GenerationMixin,
transformers.utils.hub.PushToHubMixin,
transformers.integrations.peft.PeftAdapterMixin))
list(reversed(inspect.getmro(GPT2LMHeadModel)))
[object,
transformers.integrations.peft.PeftAdapterMixin,
transformers.utils.hub.PushToHubMixin,
transformers.generation.utils.GenerationMixin,
transformers.modeling_utils.ModuleUtilsMixin,
torch.nn.modules.module.Module,
transformers.modeling_utils.PreTrainedModel,
transformers.models.gpt2.modeling_gpt2.GPT2PreTrainedModel,
transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel]
def get_base_classes(cls: Type[Any], include_self: bool = False) -> Set[Type[Any]]:
"""
Get the base classes of a class and all its base classes.
"""
return set(cls.__mro__[0:-1] if include_self else cls.__mro__[1:-1])
pprint(get_base_classes(GPT2LMHeadModel, include_self=True))
{ │ <class 'transformers.integrations.peft.PeftAdapterMixin'>, │ <class 'torch.nn.modules.module.Module'>, │ <class 'transformers.generation.utils.GenerationMixin'>, │ <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>, │ <class 'transformers.modeling_utils.PreTrainedModel'>, │ <class 'transformers.utils.hub.PushToHubMixin'>, │ <class 'transformers.modeling_utils.ModuleUtilsMixin'>, │ <class 'transformers.models.gpt2.modeling_gpt2.GPT2PreTrainedModel'> }
def get_default(param: Parameter) -> Any:
"""Return the parameter's default value or None if not specified."""
return param.default if param.default is not param.empty else None
def get_field_annotations(func_or_method: Callable[..., Any]) -> Tuple[List[Tuple[str, Any, Any]], Dict[str, Any]]:
if not inspect.isroutine(func_or_method):
raise ValueError("Expected a function or method")
required_fields = []
optional_fields = []
annotations = {}
try:
sig: Signature = inspect.signature(func_or_method)
type_hints: Dict[str, Any] = get_type_hints(func_or_method)
except ValueError:
raise ValueError("Object does not support signature or type hints extraction.") from None
for name, param in sig.parameters.items():
if name == "self":
continue
type_hint = type_hints.get(name, Any)
annotations[name] = type_hint
if param.default is param.empty:
required_fields.append((name, type_hint, Ellipsis))
else:
default_value = param.default
optional_fields.append((name, type_hint, default_value))
fields = required_fields + optional_fields
return fields, annotations
# TODO: Tuple[str, Any, Any] should be Tuple[str, Any, ellipsis]
def get_constructor_field_annotations(
cls: Type[Any], include_bases: bool = True
) -> Tuple[List[Tuple[str, Any, Any]], Dict[str, Any]]:
fields = []
annotations = {}
classes_to_inspect = [cls] + list(get_base_classes(cls, include_self=False)) if include_bases else [cls]
for c in reversed(classes_to_inspect): # Reverse to respect MRO
if hasattr(c, "__init__"):
class_fields, class_annotations = get_field_annotations(c.__init__)
# Update fields and annotations with those from the current class,
# avoiding duplicates.
for field in class_fields:
if field[0] not in annotations:
fields.append(field) # noqa: PERF401
annotations.update(class_annotations)
return fields, annotations
fields, annotations = get_constructor_field_annotations(TrainingArguments, include_bases=False)
for field in fields:
assert len(field) == 3
print(f"{field[0]}, {field[1]}, {field[2]}")
assert get_field_annotations(TrainingArguments.__init__) == (fields, annotations)
output_dir, <class 'str'>, Ellipsis
overwrite_output_dir, <class 'bool'>, False
do_train, <class 'bool'>, False
do_eval, <class 'bool'>, False
do_predict, <class 'bool'>, False
evaluation_strategy, typing.Union[transformers.trainer_utils.IntervalStrategy, str], no
prediction_loss_only, <class 'bool'>, False
per_device_train_batch_size, <class 'int'>, 8
per_device_eval_batch_size, <class 'int'>, 8
per_gpu_train_batch_size, typing.Optional[int], None
per_gpu_eval_batch_size, typing.Optional[int], None
gradient_accumulation_steps, <class 'int'>, 1
eval_accumulation_steps, typing.Optional[int], None
eval_delay, typing.Optional[float], 0
learning_rate, <class 'float'>, 5e-05
weight_decay, <class 'float'>, 0.0
adam_beta1, <class 'float'>, 0.9
adam_beta2, <class 'float'>, 0.999
adam_epsilon, <class 'float'>, 1e-08
max_grad_norm, <class 'float'>, 1.0
num_train_epochs, <class 'float'>, 3.0
max_steps, <class 'int'>, -1
lr_scheduler_type, typing.Union[transformers.trainer_utils.SchedulerType, str], linear
lr_scheduler_kwargs, typing.Optional[typing.Dict], <factory>
warmup_ratio, <class 'float'>, 0.0
warmup_steps, <class 'int'>, 0
log_level, typing.Optional[str], passive
log_level_replica, typing.Optional[str], warning
log_on_each_node, <class 'bool'>, True
logging_dir, typing.Optional[str], None
logging_strategy, typing.Union[transformers.trainer_utils.IntervalStrategy, str], steps
logging_first_step, <class 'bool'>, False
logging_steps, <class 'float'>, 500
logging_nan_inf_filter, <class 'bool'>, True
save_strategy, typing.Union[transformers.trainer_utils.IntervalStrategy, str], steps
save_steps, <class 'float'>, 500
save_total_limit, typing.Optional[int], None
save_safetensors, typing.Optional[bool], True
save_on_each_node, <class 'bool'>, False
save_only_model, <class 'bool'>, False
no_cuda, <class 'bool'>, False
use_cpu, <class 'bool'>, False
use_mps_device, <class 'bool'>, False
seed, <class 'int'>, 42
data_seed, typing.Optional[int], None
jit_mode_eval, <class 'bool'>, False
use_ipex, <class 'bool'>, False
bf16, <class 'bool'>, False
fp16, <class 'bool'>, False
fp16_opt_level, <class 'str'>, O1
half_precision_backend, <class 'str'>, auto
bf16_full_eval, <class 'bool'>, False
fp16_full_eval, <class 'bool'>, False
tf32, typing.Optional[bool], None
local_rank, <class 'int'>, -1
ddp_backend, typing.Optional[str], None
tpu_num_cores, typing.Optional[int], None
tpu_metrics_debug, <class 'bool'>, False
debug, typing.Union[str, typing.List[transformers.debug_utils.DebugOption]],
dataloader_drop_last, <class 'bool'>, False
eval_steps, typing.Optional[float], None
dataloader_num_workers, <class 'int'>, 0
dataloader_prefetch_factor, typing.Optional[int], None
past_index, <class 'int'>, -1
run_name, typing.Optional[str], None
disable_tqdm, typing.Optional[bool], None
remove_unused_columns, typing.Optional[bool], True
label_names, typing.Optional[typing.List[str]], None
load_best_model_at_end, typing.Optional[bool], False
metric_for_best_model, typing.Optional[str], None
greater_is_better, typing.Optional[bool], None
ignore_data_skip, <class 'bool'>, False
fsdp, typing.Union[typing.List[transformers.trainer_utils.FSDPOption], str, NoneType],
fsdp_min_num_params, <class 'int'>, 0
fsdp_config, typing.Union[dict, str, NoneType], None
fsdp_transformer_layer_cls_to_wrap, typing.Optional[str], None
accelerator_config, typing.Optional[str], None
deepspeed, typing.Optional[str], None
label_smoothing_factor, <class 'float'>, 0.0
optim, typing.Union[transformers.training_args.OptimizerNames, str], adamw_torch
optim_args, typing.Optional[str], None
adafactor, <class 'bool'>, False
group_by_length, <class 'bool'>, False
length_column_name, typing.Optional[str], length
report_to, typing.Optional[typing.List[str]], None
ddp_find_unused_parameters, typing.Optional[bool], None
ddp_bucket_cap_mb, typing.Optional[int], None
ddp_broadcast_buffers, typing.Optional[bool], None
dataloader_pin_memory, <class 'bool'>, True
dataloader_persistent_workers, <class 'bool'>, False
skip_memory_metrics, <class 'bool'>, True
use_legacy_prediction_loop, <class 'bool'>, False
push_to_hub, <class 'bool'>, False
resume_from_checkpoint, typing.Optional[str], None
hub_model_id, typing.Optional[str], None
hub_strategy, typing.Union[transformers.trainer_utils.HubStrategy, str], every_save
hub_token, typing.Optional[str], None
hub_private_repo, <class 'bool'>, False
hub_always_push, <class 'bool'>, False
gradient_checkpointing, <class 'bool'>, False
gradient_checkpointing_kwargs, typing.Optional[dict], None
include_inputs_for_metrics, <class 'bool'>, False
fp16_backend, <class 'str'>, auto
push_to_hub_model_id, typing.Optional[str], None
push_to_hub_organization, typing.Optional[str], None
push_to_hub_token, typing.Optional[str], None
mp_parameters, <class 'str'>,
auto_find_batch_size, <class 'bool'>, False
full_determinism, <class 'bool'>, False
torchdynamo, typing.Optional[str], None
ray_scope, typing.Optional[str], last
ddp_timeout, typing.Optional[int], 1800
torch_compile, <class 'bool'>, False
torch_compile_backend, typing.Optional[str], None
torch_compile_mode, typing.Optional[str], None
dispatch_batches, typing.Optional[bool], None
split_batches, typing.Optional[bool], None
include_tokens_per_second, typing.Optional[bool], False
include_num_input_tokens_seen, typing.Optional[bool], False
neftune_noise_alpha, typing.Optional[float], None
Warning: it does not play too well with dataclass and pydantic classes
because they have more complex bells and whistles. However, because of the perks
of dataclass and pydantic, we can just use
property
like model_fields to get all fields and their types.
As we can see from above, we did not handle lr_scheduler_kwargs well:
lr_scheduler_kwargs, typing.Optional[typing.Dict], <factory>
where <factory> is the default value of the parameter. But it is actually
referring to the default_factory of the dataclass field, which can be a default
dict etc.
def type_hint_to_str(type_hint: Any) -> str:
"""
Convert a type hint into its string representation.
"""
if hasattr(type_hint, '__name__'):
return type_hint.__name__
elif hasattr(type_hint, '_name') and type_hint._name is not None:
return str(type_hint._name)
elif type(type_hint) == _GenericAlias: # For Python 3.8+
# Handles complex types, e.g., List[int], Union[str, int]
origin = type_hint_to_str(type_hint.__origin__)
args = ', '.join(type_hint_to_str(arg) for arg in type_hint.__args__)
return f"{origin}[{args}]"
else:
# Fallback for unhandled types
return str(type_hint)
def create_config_class_str(fields: List[Tuple[str, Any, Any]]) -> str:
lines = ["class Config:"]
if not fields:
lines.append(" ...")
else:
init_params = ["self"]
init_body = []
for name, type_hint, default in fields:
type_hint_str = type_hint_to_str(type_hint)
if default is Ellipsis: # Required argument
param_str = f"{name}: {type_hint_str}"
elif default is field:
param_str = f"{name}: {type_hint_str} = field(default_factory=dict)"
else:
default_repr = repr(default) if default is not None else 'None'
param_str = f"{name}: {type_hint_str} = {default_repr}"
init_params.append(param_str)
init_body.append(f" self.{name} = {name}")
lines.append(f" def __init__({', '.join(init_params)}):")
lines.extend(init_body)
return '\n'.join(lines)
config_class_str = create_config_class_str(fields)
print(config_class_str)
class Config:
def __init__(self, output_dir: str, overwrite_output_dir: bool = False, do_train: bool = False, do_eval: bool = False, do_predict: bool = False, evaluation_strategy: typing.Union[transformers.trainer_utils.IntervalStrategy, str] = 'no', prediction_loss_only: bool = False, per_device_train_batch_size: int = 8, per_device_eval_batch_size: int = 8, per_gpu_train_batch_size: typing.Optional[int] = None, per_gpu_eval_batch_size: typing.Optional[int] = None, gradient_accumulation_steps: int = 1, eval_accumulation_steps: typing.Optional[int] = None, eval_delay: typing.Optional[float] = 0, learning_rate: float = 5e-05, weight_decay: float = 0.0, adam_beta1: float = 0.9, adam_beta2: float = 0.999, adam_epsilon: float = 1e-08, max_grad_norm: float = 1.0, num_train_epochs: float = 3.0, max_steps: int = -1, lr_scheduler_type: typing.Union[transformers.trainer_utils.SchedulerType, str] = 'linear', lr_scheduler_kwargs: typing.Optional[typing.Dict] = <factory>, warmup_ratio: float = 0.0, warmup_steps: int = 0, log_level: typing.Optional[str] = 'passive', log_level_replica: typing.Optional[str] = 'warning', log_on_each_node: bool = True, logging_dir: typing.Optional[str] = None, logging_strategy: typing.Union[transformers.trainer_utils.IntervalStrategy, str] = 'steps', logging_first_step: bool = False, logging_steps: float = 500, logging_nan_inf_filter: bool = True, save_strategy: typing.Union[transformers.trainer_utils.IntervalStrategy, str] = 'steps', save_steps: float = 500, save_total_limit: typing.Optional[int] = None, save_safetensors: typing.Optional[bool] = True, save_on_each_node: bool = False, save_only_model: bool = False, no_cuda: bool = False, use_cpu: bool = False, use_mps_device: bool = False, seed: int = 42, data_seed: typing.Optional[int] = None, jit_mode_eval: bool = False, use_ipex: bool = False, bf16: bool = False, fp16: bool = False, fp16_opt_level: str = 'O1', half_precision_backend: str = 'auto', bf16_full_eval: bool = False, fp16_full_eval: bool = False, tf32: typing.Optional[bool] = None, local_rank: int = -1, ddp_backend: typing.Optional[str] = None, tpu_num_cores: typing.Optional[int] = None, tpu_metrics_debug: bool = False, debug: typing.Union[str, typing.List[transformers.debug_utils.DebugOption]] = '', dataloader_drop_last: bool = False, eval_steps: typing.Optional[float] = None, dataloader_num_workers: int = 0, dataloader_prefetch_factor: typing.Optional[int] = None, past_index: int = -1, run_name: typing.Optional[str] = None, disable_tqdm: typing.Optional[bool] = None, remove_unused_columns: typing.Optional[bool] = True, label_names: typing.Optional[typing.List[str]] = None, load_best_model_at_end: typing.Optional[bool] = False, metric_for_best_model: typing.Optional[str] = None, greater_is_better: typing.Optional[bool] = None, ignore_data_skip: bool = False, fsdp: typing.Union[typing.List[transformers.trainer_utils.FSDPOption], str, NoneType] = '', fsdp_min_num_params: int = 0, fsdp_config: typing.Union[dict, str, NoneType] = None, fsdp_transformer_layer_cls_to_wrap: typing.Optional[str] = None, accelerator_config: typing.Optional[str] = None, deepspeed: typing.Optional[str] = None, label_smoothing_factor: float = 0.0, optim: typing.Union[transformers.training_args.OptimizerNames, str] = 'adamw_torch', optim_args: typing.Optional[str] = None, adafactor: bool = False, group_by_length: bool = False, length_column_name: typing.Optional[str] = 'length', report_to: typing.Optional[typing.List[str]] = None, ddp_find_unused_parameters: typing.Optional[bool] = None, ddp_bucket_cap_mb: typing.Optional[int] = None, ddp_broadcast_buffers: typing.Optional[bool] = None, dataloader_pin_memory: bool = True, dataloader_persistent_workers: bool = False, skip_memory_metrics: bool = True, use_legacy_prediction_loop: bool = False, push_to_hub: bool = False, resume_from_checkpoint: typing.Optional[str] = None, hub_model_id: typing.Optional[str] = None, hub_strategy: typing.Union[transformers.trainer_utils.HubStrategy, str] = 'every_save', hub_token: typing.Optional[str] = None, hub_private_repo: bool = False, hub_always_push: bool = False, gradient_checkpointing: bool = False, gradient_checkpointing_kwargs: typing.Optional[dict] = None, include_inputs_for_metrics: bool = False, fp16_backend: str = 'auto', push_to_hub_model_id: typing.Optional[str] = None, push_to_hub_organization: typing.Optional[str] = None, push_to_hub_token: typing.Optional[str] = None, mp_parameters: str = '', auto_find_batch_size: bool = False, full_determinism: bool = False, torchdynamo: typing.Optional[str] = None, ray_scope: typing.Optional[str] = 'last', ddp_timeout: typing.Optional[int] = 1800, torch_compile: bool = False, torch_compile_backend: typing.Optional[str] = None, torch_compile_mode: typing.Optional[str] = None, dispatch_batches: typing.Optional[bool] = None, split_batches: typing.Optional[bool] = None, include_tokens_per_second: typing.Optional[bool] = False, include_num_input_tokens_seen: typing.Optional[bool] = False, neftune_noise_alpha: typing.Optional[float] = None):
self.output_dir = output_dir
self.overwrite_output_dir = overwrite_output_dir
self.do_train = do_train
self.do_eval = do_eval
self.do_predict = do_predict
self.evaluation_strategy = evaluation_strategy
self.prediction_loss_only = prediction_loss_only
self.per_device_train_batch_size = per_device_train_batch_size
self.per_device_eval_batch_size = per_device_eval_batch_size
self.per_gpu_train_batch_size = per_gpu_train_batch_size
self.per_gpu_eval_batch_size = per_gpu_eval_batch_size
self.gradient_accumulation_steps = gradient_accumulation_steps
self.eval_accumulation_steps = eval_accumulation_steps
self.eval_delay = eval_delay
self.learning_rate = learning_rate
self.weight_decay = weight_decay
self.adam_beta1 = adam_beta1
self.adam_beta2 = adam_beta2
self.adam_epsilon = adam_epsilon
self.max_grad_norm = max_grad_norm
self.num_train_epochs = num_train_epochs
self.max_steps = max_steps
self.lr_scheduler_type = lr_scheduler_type
self.lr_scheduler_kwargs = lr_scheduler_kwargs
self.warmup_ratio = warmup_ratio
self.warmup_steps = warmup_steps
self.log_level = log_level
self.log_level_replica = log_level_replica
self.log_on_each_node = log_on_each_node
self.logging_dir = logging_dir
self.logging_strategy = logging_strategy
self.logging_first_step = logging_first_step
self.logging_steps = logging_steps
self.logging_nan_inf_filter = logging_nan_inf_filter
self.save_strategy = save_strategy
self.save_steps = save_steps
self.save_total_limit = save_total_limit
self.save_safetensors = save_safetensors
self.save_on_each_node = save_on_each_node
self.save_only_model = save_only_model
self.no_cuda = no_cuda
self.use_cpu = use_cpu
self.use_mps_device = use_mps_device
self.seed = seed
self.data_seed = data_seed
self.jit_mode_eval = jit_mode_eval
self.use_ipex = use_ipex
self.bf16 = bf16
self.fp16 = fp16
self.fp16_opt_level = fp16_opt_level
self.half_precision_backend = half_precision_backend
self.bf16_full_eval = bf16_full_eval
self.fp16_full_eval = fp16_full_eval
self.tf32 = tf32
self.local_rank = local_rank
self.ddp_backend = ddp_backend
self.tpu_num_cores = tpu_num_cores
self.tpu_metrics_debug = tpu_metrics_debug
self.debug = debug
self.dataloader_drop_last = dataloader_drop_last
self.eval_steps = eval_steps
self.dataloader_num_workers = dataloader_num_workers
self.dataloader_prefetch_factor = dataloader_prefetch_factor
self.past_index = past_index
self.run_name = run_name
self.disable_tqdm = disable_tqdm
self.remove_unused_columns = remove_unused_columns
self.label_names = label_names
self.load_best_model_at_end = load_best_model_at_end
self.metric_for_best_model = metric_for_best_model
self.greater_is_better = greater_is_better
self.ignore_data_skip = ignore_data_skip
self.fsdp = fsdp
self.fsdp_min_num_params = fsdp_min_num_params
self.fsdp_config = fsdp_config
self.fsdp_transformer_layer_cls_to_wrap = fsdp_transformer_layer_cls_to_wrap
self.accelerator_config = accelerator_config
self.deepspeed = deepspeed
self.label_smoothing_factor = label_smoothing_factor
self.optim = optim
self.optim_args = optim_args
self.adafactor = adafactor
self.group_by_length = group_by_length
self.length_column_name = length_column_name
self.report_to = report_to
self.ddp_find_unused_parameters = ddp_find_unused_parameters
self.ddp_bucket_cap_mb = ddp_bucket_cap_mb
self.ddp_broadcast_buffers = ddp_broadcast_buffers
self.dataloader_pin_memory = dataloader_pin_memory
self.dataloader_persistent_workers = dataloader_persistent_workers
self.skip_memory_metrics = skip_memory_metrics
self.use_legacy_prediction_loop = use_legacy_prediction_loop
self.push_to_hub = push_to_hub
self.resume_from_checkpoint = resume_from_checkpoint
self.hub_model_id = hub_model_id
self.hub_strategy = hub_strategy
self.hub_token = hub_token
self.hub_private_repo = hub_private_repo
self.hub_always_push = hub_always_push
self.gradient_checkpointing = gradient_checkpointing
self.gradient_checkpointing_kwargs = gradient_checkpointing_kwargs
self.include_inputs_for_metrics = include_inputs_for_metrics
self.fp16_backend = fp16_backend
self.push_to_hub_model_id = push_to_hub_model_id
self.push_to_hub_organization = push_to_hub_organization
self.push_to_hub_token = push_to_hub_token
self.mp_parameters = mp_parameters
self.auto_find_batch_size = auto_find_batch_size
self.full_determinism = full_determinism
self.torchdynamo = torchdynamo
self.ray_scope = ray_scope
self.ddp_timeout = ddp_timeout
self.torch_compile = torch_compile
self.torch_compile_backend = torch_compile_backend
self.torch_compile_mode = torch_compile_mode
self.dispatch_batches = dispatch_batches
self.split_batches = split_batches
self.include_tokens_per_second = include_tokens_per_second
self.include_num_input_tokens_seen = include_num_input_tokens_seen
self.neftune_noise_alpha = neftune_noise_alpha
Using this as is will yield a SyntaxError because of the <factory> issue
highlighted above. We can use on a “normal” class Trainer.
fields, annotations = get_constructor_field_annotations(Trainer, include_bases=False)
config_class_str = create_config_class_str(fields)
print(config_class_str)
class Config:
def __init__(self, model: typing.Union[transformers.modeling_utils.PreTrainedModel, torch.nn.modules.module.Module, NoneType] = None, args: typing.Optional[transformers.training_args.TrainingArguments] = None, data_collator: typing.Optional[DataCollator] = None, train_dataset: typing.Optional[torch.utils.data.dataset.Dataset] = None, eval_dataset: typing.Union[torch.utils.data.dataset.Dataset, typing.Dict[str, torch.utils.data.dataset.Dataset], NoneType] = None, tokenizer: typing.Optional[transformers.tokenization_utils_base.PreTrainedTokenizerBase] = None, model_init: typing.Optional[typing.Callable[[], transformers.modeling_utils.PreTrainedModel]] = None, compute_metrics: typing.Optional[typing.Callable[[transformers.trainer_utils.EvalPrediction], typing.Dict]] = None, callbacks: typing.Optional[typing.List[transformers.trainer_callback.TrainerCallback]] = None, optimizers: Tuple = (None, None), preprocess_logits_for_metrics: typing.Optional[typing.Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None):
self.model = model
self.args = args
self.data_collator = data_collator
self.train_dataset = train_dataset
self.eval_dataset = eval_dataset
self.tokenizer = tokenizer
self.model_init = model_init
self.compute_metrics = compute_metrics
self.callbacks = callbacks
self.optimizers = optimizers
self.preprocess_logits_for_metrics = preprocess_logits_for_metrics
import transformers
import typing
import torch
from transformers import DataCollator
NoneType = type(None)
config_class_str = create_config_class_str(fields)
# Execute the generated class definition string
namespace = {}
exec(config_class_str, globals(), namespace)
# Extract the newly created class from the namespace
ConfigClass = namespace['Config']
inspect.signature(ConfigClass.__init__)
<Signature (self, model: Union[transformers.modeling_utils.PreTrainedModel, torch.nn.modules.module.Module, NoneType] = None, args: Optional[transformers.training_args.TrainingArguments] = None, data_collator: Optional[DataCollator] = None, train_dataset: Optional[torch.utils.data.dataset.Dataset] = None, eval_dataset: Union[torch.utils.data.dataset.Dataset, Dict[str, torch.utils.data.dataset.Dataset], NoneType] = None, tokenizer: Optional[transformers.tokenization_utils_base.PreTrainedTokenizerBase] = None, model_init: Optional[Callable[[], transformers.modeling_utils.PreTrainedModel]] = None, compute_metrics: Optional[Callable[[transformers.trainer_utils.EvalPrediction], Dict]] = None, callbacks: Optional[List[transformers.trainer_callback.TrainerCallback]] = None, optimizers: Tuple = (None, None), preprocess_logits_for_metrics: Optional[Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None)>