diff --git a/lib/tokenize_rt.py b/lib/tokenize_rt.py index b225a9f0..f0922c16 100644 --- a/lib/tokenize_rt.py +++ b/lib/tokenize_rt.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import io import keyword @@ -6,20 +8,17 @@ import sys import tokenize from typing import Generator from typing import Iterable -from typing import List from typing import NamedTuple -from typing import Optional from typing import Pattern from typing import Sequence -from typing import Tuple # this is a performance hack. see https://bugs.python.org/issue43014 -if ( +if ( # pragma: no branch sys.version_info < (3, 10) and callable(getattr(tokenize, '_compile', None)) -): # pragma: no cover ( Offset: @@ -43,11 +42,10 @@ class Token(NamedTuple): _string_re = re.compile('^([^\'"]*)(.*)$', re.DOTALL) -_string_prefixes = frozenset('bfru') _escaped_nl_re = re.compile(r'\\(\n|\r\n|\r)') -def _re_partition(regex: Pattern[str], s: str) -> Tuple[str, str, str]: +def _re_partition(regex: Pattern[str], s: str) -> tuple[str, str, str]: match = regex.search(s) if match: return s[:match.start()], s[slice(*match.span())], s[match.end():] @@ -55,7 +53,7 @@ def _re_partition(regex: Pattern[str], s: str) -> Tuple[str, str, str]: return (s, '', '') -def src_to_tokens(src: str) -> List[Token]: +def src_to_tokens(src: str) -> list[Token]: tokenize_target = io.StringIO(src) lines = ('',) + tuple(tokenize_target) @@ -98,33 +96,7 @@ def src_to_tokens(src: str) -> List[Token]: end_offset += len(newtok.encode()) tok_name = tokenize.tok_name[tok_type] - # when a string prefix is not recognized, the tokenizer produces a - # NAME token followed by a STRING token - if ( - tok_name == 'STRING' and - tokens and - tokens[-1].name == 'NAME' and - frozenset(tokens[-1].src.lower()) <= _string_prefixes - ): - newsrc = tokens[-1].src + tok_text - tokens[-1] = tokens[-1]._replace(src=newsrc, name=tok_name) - # produce octal literals as a single token in python 3 as well - elif ( - tok_name == 'NUMBER' and - tokens and - tokens[-1].name == 'NUMBER' - ): - tokens[-1] = tokens[-1]._replace(src=tokens[-1].src + tok_text) - # produce long literals as a single token in python 3 as well - elif ( - tok_name == 'NAME' and - tok_text.lower() == 'l' and - tokens and - tokens[-1].name == 'NUMBER' - ): - tokens[-1] = tokens[-1]._replace(src=tokens[-1].src + tok_text) - else: - tokens.append(Token(tok_name, tok_text, sline, end_offset)) + tokens.append(Token(tok_name, tok_text, sline, end_offset)) last_line, last_col = eline, ecol if sline != eline: end_offset = len(lines[last_line][:last_col].encode()) @@ -140,19 +112,19 @@ def tokens_to_src(tokens: Iterable[Token]) -> str: def reversed_enumerate( tokens: Sequence[Token], -) -> Generator[Tuple[int, Token], None, None]: +) -> Generator[tuple[int, Token], None, None]: for i in reversed(range(len(tokens))): yield i, tokens[i] -def parse_string_literal(src: str) -> Tuple[str, str]: +def parse_string_literal(src: str) -> tuple[str, str]: """parse a string literal's source into (prefix, string)""" match = _string_re.match(src) assert match is not None return match.group(1), match.group(2) -def rfind_string_parts(tokens: Sequence[Token], i: int) -> Tuple[int, ...]: +def rfind_string_parts(tokens: Sequence[Token], i: int) -> tuple[int, ...]: """find the indicies of the string parts of a (joined) string literal - `i` should start at the end of the string literal @@ -195,7 +167,7 @@ def rfind_string_parts(tokens: Sequence[Token], i: int) -> Tuple[int, ...]: return tuple(reversed(ret)) -def main(argv: Optional[Sequence[str]] = None) -> int: +def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filename') args = parser.parse_args(argv) @@ -210,4 +182,4 @@ def main(argv: Optional[Sequence[str]] = None) -> int: if __name__ == '__main__': - exit(main()) + raise SystemExit(main()) diff --git a/requirements.txt b/requirements.txt index 17d503fd..cff0f4b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,7 +42,7 @@ simplejson==3.17.6 six==1.16.0 soupsieve==2.3.2.post1 tempora==5.0.2 -tokenize-rt==4.2.1 +tokenize-rt==5.0.0 tzdata==2022.6 tzlocal==4.2 urllib3==1.26.12