Update tokenize-rt==5.2.0

This commit is contained in:
JonnyWong16 2023-08-24 11:59:15 -07:00
commit 6a5eeca39f
No known key found for this signature in database
GPG key ID: B1F1F9807184697A

View file

@ -18,7 +18,7 @@ if ( # pragma: no branch
callable(getattr(tokenize, '_compile', None))
): # pragma: <3.10 cover
from functools import lru_cache
tokenize._compile = lru_cache()(tokenize._compile)
tokenize._compile = lru_cache(tokenize._compile)
ESCAPED_NL = 'ESCAPED_NL'
UNIMPORTANT_WS = 'UNIMPORTANT_WS'
@ -40,6 +40,9 @@ class Token(NamedTuple):
def offset(self) -> Offset:
return Offset(self.line, self.utf8_byte_offset)
def matches(self, *, name: str, src: str) -> bool:
return self.name == name and self.src == src
_string_re = re.compile('^([^\'"]*)(.*)$', re.DOTALL)
_escaped_nl_re = re.compile(r'\\(\n|\r\n|\r)')