typing.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627
  1. import functools
  2. import operator
  3. import sys
  4. import typing
  5. from collections.abc import Callable
  6. from os import PathLike
  7. from typing import ( # type: ignore
  8. TYPE_CHECKING,
  9. AbstractSet,
  10. Any,
  11. Callable as TypingCallable,
  12. ClassVar,
  13. Dict,
  14. ForwardRef,
  15. Generator,
  16. Iterable,
  17. List,
  18. Mapping,
  19. NewType,
  20. Optional,
  21. Sequence,
  22. Set,
  23. Tuple,
  24. Type,
  25. TypeVar,
  26. Union,
  27. _eval_type,
  28. cast,
  29. get_type_hints,
  30. )
  31. from typing_extensions import (
  32. Annotated,
  33. Final,
  34. Literal,
  35. NotRequired as TypedDictNotRequired,
  36. Required as TypedDictRequired,
  37. )
  38. try:
  39. from typing import _TypingBase as typing_base # type: ignore
  40. except ImportError:
  41. from typing import _Final as typing_base # type: ignore
  42. try:
  43. from typing import GenericAlias as TypingGenericAlias # type: ignore
  44. except ImportError:
  45. # python < 3.9 does not have GenericAlias (list[int], tuple[str, ...] and so on)
  46. TypingGenericAlias = ()
  47. try:
  48. from types import UnionType as TypesUnionType # type: ignore
  49. except ImportError:
  50. # python < 3.10 does not have UnionType (str | int, byte | bool and so on)
  51. TypesUnionType = ()
  52. if sys.version_info < (3, 9):
  53. def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any:
  54. return type_._evaluate(globalns, localns)
  55. elif sys.version_info < (3, 12, 4):
  56. def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any:
  57. # Even though it is the right signature for python 3.9, mypy complains with
  58. # `error: Too many arguments for "_evaluate" of "ForwardRef"` hence the cast...
  59. # Python 3.13/3.12.4+ made `recursive_guard` a kwarg, so name it explicitly to avoid:
  60. # TypeError: ForwardRef._evaluate() missing 1 required keyword-only argument: 'recursive_guard'
  61. return cast(Any, type_)._evaluate(globalns, localns, recursive_guard=set())
  62. elif sys.version_info < (3, 14):
  63. def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any:
  64. # Pydantic 1.x will not support PEP 695 syntax, but provide `type_params` to avoid
  65. # warnings:
  66. return cast(Any, type_)._evaluate(globalns, localns, type_params=(), recursive_guard=set())
  67. else:
  68. def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any:
  69. # Pydantic 1.x will not support PEP 695 syntax, but provide `type_params` to avoid
  70. # warnings:
  71. return typing.evaluate_forward_ref(
  72. type_,
  73. globals=globalns,
  74. locals=localns,
  75. type_params=(),
  76. _recursive_guard=set(),
  77. )
  78. if sys.version_info < (3, 9):
  79. # Ensure we always get all the whole `Annotated` hint, not just the annotated type.
  80. # For 3.7 to 3.8, `get_type_hints` doesn't recognize `typing_extensions.Annotated`,
  81. # so it already returns the full annotation
  82. get_all_type_hints = get_type_hints
  83. else:
  84. def get_all_type_hints(obj: Any, globalns: Any = None, localns: Any = None) -> Any:
  85. return get_type_hints(obj, globalns, localns, include_extras=True)
  86. _T = TypeVar('_T')
  87. AnyCallable = TypingCallable[..., Any]
  88. NoArgAnyCallable = TypingCallable[[], Any]
  89. # workaround for https://github.com/python/mypy/issues/9496
  90. AnyArgTCallable = TypingCallable[..., _T]
  91. # Annotated[...] is implemented by returning an instance of one of these classes, depending on
  92. # python/typing_extensions version.
  93. AnnotatedTypeNames = {'AnnotatedMeta', '_AnnotatedAlias'}
  94. LITERAL_TYPES: Set[Any] = {Literal}
  95. if hasattr(typing, 'Literal'):
  96. LITERAL_TYPES.add(typing.Literal)
  97. if sys.version_info < (3, 8):
  98. def get_origin(t: Type[Any]) -> Optional[Type[Any]]:
  99. if type(t).__name__ in AnnotatedTypeNames:
  100. # weirdly this is a runtime requirement, as well as for mypy
  101. return cast(Type[Any], Annotated)
  102. return getattr(t, '__origin__', None)
  103. else:
  104. from typing import get_origin as _typing_get_origin
  105. def get_origin(tp: Type[Any]) -> Optional[Type[Any]]:
  106. """
  107. We can't directly use `typing.get_origin` since we need a fallback to support
  108. custom generic classes like `ConstrainedList`
  109. It should be useless once https://github.com/cython/cython/issues/3537 is
  110. solved and https://github.com/pydantic/pydantic/pull/1753 is merged.
  111. """
  112. if type(tp).__name__ in AnnotatedTypeNames:
  113. return cast(Type[Any], Annotated) # mypy complains about _SpecialForm
  114. return _typing_get_origin(tp) or getattr(tp, '__origin__', None)
  115. if sys.version_info < (3, 8):
  116. from typing import _GenericAlias
  117. def get_args(t: Type[Any]) -> Tuple[Any, ...]:
  118. """Compatibility version of get_args for python 3.7.
  119. Mostly compatible with the python 3.8 `typing` module version
  120. and able to handle almost all use cases.
  121. """
  122. if type(t).__name__ in AnnotatedTypeNames:
  123. return t.__args__ + t.__metadata__
  124. if isinstance(t, _GenericAlias):
  125. res = t.__args__
  126. if t.__origin__ is Callable and res and res[0] is not Ellipsis:
  127. res = (list(res[:-1]), res[-1])
  128. return res
  129. return getattr(t, '__args__', ())
  130. else:
  131. from typing import get_args as _typing_get_args
  132. def _generic_get_args(tp: Type[Any]) -> Tuple[Any, ...]:
  133. """
  134. In python 3.9, `typing.Dict`, `typing.List`, ...
  135. do have an empty `__args__` by default (instead of the generic ~T for example).
  136. In order to still support `Dict` for example and consider it as `Dict[Any, Any]`,
  137. we retrieve the `_nparams` value that tells us how many parameters it needs.
  138. """
  139. if hasattr(tp, '_nparams'):
  140. return (Any,) * tp._nparams
  141. # Special case for `tuple[()]`, which used to return ((),) with `typing.Tuple`
  142. # in python 3.10- but now returns () for `tuple` and `Tuple`.
  143. # This will probably be clarified in pydantic v2
  144. try:
  145. if tp == Tuple[()] or sys.version_info >= (3, 9) and tp == tuple[()]: # type: ignore[misc]
  146. return ((),)
  147. # there is a TypeError when compiled with cython
  148. except TypeError: # pragma: no cover
  149. pass
  150. return ()
  151. def get_args(tp: Type[Any]) -> Tuple[Any, ...]:
  152. """Get type arguments with all substitutions performed.
  153. For unions, basic simplifications used by Union constructor are performed.
  154. Examples::
  155. get_args(Dict[str, int]) == (str, int)
  156. get_args(int) == ()
  157. get_args(Union[int, Union[T, int], str][int]) == (int, str)
  158. get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int])
  159. get_args(Callable[[], T][int]) == ([], int)
  160. """
  161. if type(tp).__name__ in AnnotatedTypeNames:
  162. return tp.__args__ + tp.__metadata__
  163. # the fallback is needed for the same reasons as `get_origin` (see above)
  164. return _typing_get_args(tp) or getattr(tp, '__args__', ()) or _generic_get_args(tp)
  165. if sys.version_info < (3, 9):
  166. def convert_generics(tp: Type[Any]) -> Type[Any]:
  167. """Python 3.9 and older only supports generics from `typing` module.
  168. They convert strings to ForwardRef automatically.
  169. Examples::
  170. typing.List['Hero'] == typing.List[ForwardRef('Hero')]
  171. """
  172. return tp
  173. else:
  174. def convert_generics(tp: Type[Any]) -> Type[Any]:
  175. """
  176. Recursively searches for `str` type hints and replaces them with ForwardRef.
  177. Examples::
  178. convert_generics(list['Hero']) == list[ForwardRef('Hero')]
  179. convert_generics(dict['Hero', 'Team']) == dict[ForwardRef('Hero'), ForwardRef('Team')]
  180. convert_generics(typing.Dict['Hero', 'Team']) == typing.Dict[ForwardRef('Hero'), ForwardRef('Team')]
  181. convert_generics(list[str | 'Hero'] | int) == list[str | ForwardRef('Hero')] | int
  182. """
  183. origin = get_origin(tp)
  184. if not origin or not hasattr(tp, '__args__'):
  185. return tp
  186. args = get_args(tp)
  187. # typing.Annotated needs special treatment
  188. if origin is Annotated:
  189. return Annotated[(convert_generics(args[0]), *args[1:])] # type: ignore
  190. # recursively replace `str` instances inside of `GenericAlias` with `ForwardRef(arg)`
  191. converted = tuple(
  192. ForwardRef(arg) if isinstance(arg, str) and isinstance(tp, TypingGenericAlias) else convert_generics(arg)
  193. for arg in args
  194. )
  195. if converted == args:
  196. return tp
  197. elif isinstance(tp, TypingGenericAlias):
  198. return TypingGenericAlias(origin, converted)
  199. elif isinstance(tp, TypesUnionType):
  200. # recreate types.UnionType (PEP604, Python >= 3.10)
  201. return functools.reduce(operator.or_, converted) # type: ignore
  202. else:
  203. try:
  204. setattr(tp, '__args__', converted)
  205. except AttributeError:
  206. pass
  207. return tp
  208. if sys.version_info < (3, 10):
  209. def is_union(tp: Optional[Type[Any]]) -> bool:
  210. return tp is Union
  211. WithArgsTypes = (TypingGenericAlias,)
  212. else:
  213. import types
  214. import typing
  215. def is_union(tp: Optional[Type[Any]]) -> bool:
  216. return tp is Union or tp is types.UnionType # noqa: E721
  217. WithArgsTypes = (typing._GenericAlias, types.GenericAlias, types.UnionType)
  218. StrPath = Union[str, PathLike]
  219. if TYPE_CHECKING:
  220. from pydantic.v1.fields import ModelField
  221. TupleGenerator = Generator[Tuple[str, Any], None, None]
  222. DictStrAny = Dict[str, Any]
  223. DictAny = Dict[Any, Any]
  224. SetStr = Set[str]
  225. ListStr = List[str]
  226. IntStr = Union[int, str]
  227. AbstractSetIntStr = AbstractSet[IntStr]
  228. DictIntStrAny = Dict[IntStr, Any]
  229. MappingIntStrAny = Mapping[IntStr, Any]
  230. CallableGenerator = Generator[AnyCallable, None, None]
  231. ReprArgs = Sequence[Tuple[Optional[str], Any]]
  232. MYPY = False
  233. if MYPY:
  234. AnyClassMethod = classmethod[Any]
  235. else:
  236. # classmethod[TargetType, CallableParamSpecType, CallableReturnType]
  237. AnyClassMethod = classmethod[Any, Any, Any]
  238. __all__ = (
  239. 'AnyCallable',
  240. 'NoArgAnyCallable',
  241. 'NoneType',
  242. 'is_none_type',
  243. 'display_as_type',
  244. 'resolve_annotations',
  245. 'is_callable_type',
  246. 'is_literal_type',
  247. 'all_literal_values',
  248. 'is_namedtuple',
  249. 'is_typeddict',
  250. 'is_typeddict_special',
  251. 'is_new_type',
  252. 'new_type_supertype',
  253. 'is_classvar',
  254. 'is_finalvar',
  255. 'update_field_forward_refs',
  256. 'update_model_forward_refs',
  257. 'TupleGenerator',
  258. 'DictStrAny',
  259. 'DictAny',
  260. 'SetStr',
  261. 'ListStr',
  262. 'IntStr',
  263. 'AbstractSetIntStr',
  264. 'DictIntStrAny',
  265. 'CallableGenerator',
  266. 'ReprArgs',
  267. 'AnyClassMethod',
  268. 'CallableGenerator',
  269. 'WithArgsTypes',
  270. 'get_args',
  271. 'get_origin',
  272. 'get_sub_types',
  273. 'typing_base',
  274. 'get_all_type_hints',
  275. 'is_union',
  276. 'StrPath',
  277. 'MappingIntStrAny',
  278. )
  279. NoneType = None.__class__
  280. NONE_TYPES: Tuple[Any, Any, Any] = (None, NoneType, Literal[None])
  281. if sys.version_info < (3, 8):
  282. # Even though this implementation is slower, we need it for python 3.7:
  283. # In python 3.7 "Literal" is not a builtin type and uses a different
  284. # mechanism.
  285. # for this reason `Literal[None] is Literal[None]` evaluates to `False`,
  286. # breaking the faster implementation used for the other python versions.
  287. def is_none_type(type_: Any) -> bool:
  288. return type_ in NONE_TYPES
  289. elif sys.version_info[:2] == (3, 8):
  290. def is_none_type(type_: Any) -> bool:
  291. for none_type in NONE_TYPES:
  292. if type_ is none_type:
  293. return True
  294. # With python 3.8, specifically 3.8.10, Literal "is" check sare very flakey
  295. # can change on very subtle changes like use of types in other modules,
  296. # hopefully this check avoids that issue.
  297. if is_literal_type(type_): # pragma: no cover
  298. return all_literal_values(type_) == (None,)
  299. return False
  300. else:
  301. def is_none_type(type_: Any) -> bool:
  302. return type_ in NONE_TYPES
  303. def display_as_type(v: Type[Any]) -> str:
  304. if not isinstance(v, typing_base) and not isinstance(v, WithArgsTypes) and not isinstance(v, type):
  305. v = v.__class__
  306. if is_union(get_origin(v)):
  307. return f'Union[{", ".join(map(display_as_type, get_args(v)))}]'
  308. if isinstance(v, WithArgsTypes):
  309. # Generic alias are constructs like `list[int]`
  310. return str(v).replace('typing.', '')
  311. try:
  312. return v.__name__
  313. except AttributeError:
  314. # happens with typing objects
  315. return str(v).replace('typing.', '')
  316. def resolve_annotations(raw_annotations: Dict[str, Type[Any]], module_name: Optional[str]) -> Dict[str, Type[Any]]:
  317. """
  318. Partially taken from typing.get_type_hints.
  319. Resolve string or ForwardRef annotations into type objects if possible.
  320. """
  321. base_globals: Optional[Dict[str, Any]] = None
  322. if module_name:
  323. try:
  324. module = sys.modules[module_name]
  325. except KeyError:
  326. # happens occasionally, see https://github.com/pydantic/pydantic/issues/2363
  327. pass
  328. else:
  329. base_globals = module.__dict__
  330. annotations = {}
  331. for name, value in raw_annotations.items():
  332. if isinstance(value, str):
  333. if (3, 10) > sys.version_info >= (3, 9, 8) or sys.version_info >= (3, 10, 1):
  334. value = ForwardRef(value, is_argument=False, is_class=True)
  335. else:
  336. value = ForwardRef(value, is_argument=False)
  337. try:
  338. if sys.version_info >= (3, 13):
  339. value = _eval_type(value, base_globals, None, type_params=())
  340. else:
  341. value = _eval_type(value, base_globals, None)
  342. except NameError:
  343. # this is ok, it can be fixed with update_forward_refs
  344. pass
  345. annotations[name] = value
  346. return annotations
  347. def is_callable_type(type_: Type[Any]) -> bool:
  348. return type_ is Callable or get_origin(type_) is Callable
  349. def is_literal_type(type_: Type[Any]) -> bool:
  350. return Literal is not None and get_origin(type_) in LITERAL_TYPES
  351. def literal_values(type_: Type[Any]) -> Tuple[Any, ...]:
  352. return get_args(type_)
  353. def all_literal_values(type_: Type[Any]) -> Tuple[Any, ...]:
  354. """
  355. This method is used to retrieve all Literal values as
  356. Literal can be used recursively (see https://www.python.org/dev/peps/pep-0586)
  357. e.g. `Literal[Literal[Literal[1, 2, 3], "foo"], 5, None]`
  358. """
  359. if not is_literal_type(type_):
  360. return (type_,)
  361. values = literal_values(type_)
  362. return tuple(x for value in values for x in all_literal_values(value))
  363. def is_namedtuple(type_: Type[Any]) -> bool:
  364. """
  365. Check if a given class is a named tuple.
  366. It can be either a `typing.NamedTuple` or `collections.namedtuple`
  367. """
  368. from pydantic.v1.utils import lenient_issubclass
  369. return lenient_issubclass(type_, tuple) and hasattr(type_, '_fields')
  370. def is_typeddict(type_: Type[Any]) -> bool:
  371. """
  372. Check if a given class is a typed dict (from `typing` or `typing_extensions`)
  373. In 3.10, there will be a public method (https://docs.python.org/3.10/library/typing.html#typing.is_typeddict)
  374. """
  375. from pydantic.v1.utils import lenient_issubclass
  376. return lenient_issubclass(type_, dict) and hasattr(type_, '__total__')
  377. def _check_typeddict_special(type_: Any) -> bool:
  378. return type_ is TypedDictRequired or type_ is TypedDictNotRequired
  379. def is_typeddict_special(type_: Any) -> bool:
  380. """
  381. Check if type is a TypedDict special form (Required or NotRequired).
  382. """
  383. return _check_typeddict_special(type_) or _check_typeddict_special(get_origin(type_))
  384. test_type = NewType('test_type', str)
  385. def is_new_type(type_: Type[Any]) -> bool:
  386. """
  387. Check whether type_ was created using typing.NewType
  388. """
  389. return isinstance(type_, test_type.__class__) and hasattr(type_, '__supertype__') # type: ignore
  390. def new_type_supertype(type_: Type[Any]) -> Type[Any]:
  391. while hasattr(type_, '__supertype__'):
  392. type_ = type_.__supertype__
  393. return type_
  394. def _check_classvar(v: Optional[Type[Any]]) -> bool:
  395. if v is None:
  396. return False
  397. return v.__class__ == ClassVar.__class__ and getattr(v, '_name', None) == 'ClassVar'
  398. def _check_finalvar(v: Optional[Type[Any]]) -> bool:
  399. """
  400. Check if a given type is a `typing.Final` type.
  401. """
  402. if v is None:
  403. return False
  404. return v.__class__ == Final.__class__ and (sys.version_info < (3, 8) or getattr(v, '_name', None) == 'Final')
  405. def is_classvar(ann_type: Type[Any]) -> bool:
  406. if _check_classvar(ann_type) or _check_classvar(get_origin(ann_type)):
  407. return True
  408. # this is an ugly workaround for class vars that contain forward references and are therefore themselves
  409. # forward references, see #3679
  410. if ann_type.__class__ == ForwardRef and ann_type.__forward_arg__.startswith('ClassVar['):
  411. return True
  412. return False
  413. def is_finalvar(ann_type: Type[Any]) -> bool:
  414. return _check_finalvar(ann_type) or _check_finalvar(get_origin(ann_type))
  415. def update_field_forward_refs(field: 'ModelField', globalns: Any, localns: Any) -> None:
  416. """
  417. Try to update ForwardRefs on fields based on this ModelField, globalns and localns.
  418. """
  419. prepare = False
  420. if field.type_.__class__ == ForwardRef:
  421. prepare = True
  422. field.type_ = evaluate_forwardref(field.type_, globalns, localns or None)
  423. if field.outer_type_.__class__ == ForwardRef:
  424. prepare = True
  425. field.outer_type_ = evaluate_forwardref(field.outer_type_, globalns, localns or None)
  426. if prepare:
  427. field.prepare()
  428. if field.sub_fields:
  429. for sub_f in field.sub_fields:
  430. update_field_forward_refs(sub_f, globalns=globalns, localns=localns)
  431. if field.discriminator_key is not None:
  432. field.prepare_discriminated_union_sub_fields()
  433. def update_model_forward_refs(
  434. model: Type[Any],
  435. fields: Iterable['ModelField'],
  436. json_encoders: Dict[Union[Type[Any], str, ForwardRef], AnyCallable],
  437. localns: 'DictStrAny',
  438. exc_to_suppress: Tuple[Type[BaseException], ...] = (),
  439. ) -> None:
  440. """
  441. Try to update model fields ForwardRefs based on model and localns.
  442. """
  443. if model.__module__ in sys.modules:
  444. globalns = sys.modules[model.__module__].__dict__.copy()
  445. else:
  446. globalns = {}
  447. globalns.setdefault(model.__name__, model)
  448. for f in fields:
  449. try:
  450. update_field_forward_refs(f, globalns=globalns, localns=localns)
  451. except exc_to_suppress:
  452. pass
  453. for key in set(json_encoders.keys()):
  454. if isinstance(key, str):
  455. fr: ForwardRef = ForwardRef(key)
  456. elif isinstance(key, ForwardRef):
  457. fr = key
  458. else:
  459. continue
  460. try:
  461. new_key = evaluate_forwardref(fr, globalns, localns or None)
  462. except exc_to_suppress: # pragma: no cover
  463. continue
  464. json_encoders[new_key] = json_encoders.pop(key)
  465. def get_class(type_: Type[Any]) -> Union[None, bool, Type[Any]]:
  466. """
  467. Tries to get the class of a Type[T] annotation. Returns True if Type is used
  468. without brackets. Otherwise returns None.
  469. """
  470. if type_ is type:
  471. return True
  472. if get_origin(type_) is None:
  473. return None
  474. args = get_args(type_)
  475. if not args or not isinstance(args[0], type):
  476. return True
  477. else:
  478. return args[0]
  479. def get_sub_types(tp: Any) -> List[Any]:
  480. """
  481. Return all the types that are allowed by type `tp`
  482. `tp` can be a `Union` of allowed types or an `Annotated` type
  483. """
  484. origin = get_origin(tp)
  485. if origin is Annotated:
  486. return get_sub_types(get_args(tp)[0])
  487. elif is_union(origin):
  488. return [x for t in get_args(tp) for x in get_sub_types(t)]
  489. else:
  490. return [tp]