From 597c294a5dc730687934ee70cb42bed958df8f12 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:46:29 +0100 Subject: [PATCH 01/32] Use diagnostic line for missing-brace errors --- Lib/test/test_fstring.py | 14 ++++++++++++++ ...-09-24-18-20-00.gh-issue-153569.diagnostics.rst | 1 + Parser/lexer/string.c | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 63006266bcd039e..72c80b8bfbfecc5 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -645,6 +645,20 @@ def test_unclosed_multiline_replacement_field(self): f"{prefix[-1]}-string: expecting '}}' to close '{{' " f"on line {lineno}") + def test_unclosed_replacement_field_quote_line(self): + for prefix in ('f', 't', 'rf', 'rt'): + for quote in ('"', "'"): + triple = quote * 3 + for suffix in ('', '\nx'): + source = prefix + triple + '{1' + triple + suffix + with self.subTest(source=source): + with self.assertRaises(SyntaxError) as cm: + compile(source, '', 'exec') + self.assertEqual( + cm.exception.msg, + f"{prefix[-1]}-string: expecting '}}'") + self.assertEqual(cm.exception.lineno, 1) + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_mismatched_parens(self): self.assertAllRaise(SyntaxError, r"closing parenthesis '\}' " diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst new file mode 100644 index 000000000000000..2b0aae1a3d2b4c4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst @@ -0,0 +1 @@ +Correct missing-brace error messages for multiline f-strings and t-strings. diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index a48cedf270362ab..41dc3c66edff09f 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -370,7 +370,7 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) assert(level >= 0 && level < tok->level); assert(tok->parenstack[level] == '{'); int lineno = tok->parenlinenostack[level]; - if (lineno != tok->lineno) { + if (lineno != location.lineno) { _PyTokenizer_syntaxerror_at( tok, line, cursor_offset, location.lineno, -1, -1, "%c-string: expecting '}' to close '{' on line %d", From 9df1aa006f2bad80abea73b64d35fcd5c257f19e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:58:16 +0100 Subject: [PATCH 02/32] Keep implicit prepared newline metadata in the reader --- Modules/_testinternalcapi/tokenizer.c | 34 ++++++--------- Parser/tokenizer/decoder.c | 3 +- Parser/tokenizer/reader.c | 8 ++-- Parser/tokenizer/reader_internal.h | 1 + Parser/tokenizer/source.c | 61 +++------------------------ Parser/tokenizer/source.h | 11 +---- 6 files changed, 27 insertions(+), 91 deletions(-) diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index 1f89c12f223c7d9..fd131abe985937c 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -46,23 +46,17 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), } if (check_system_error( - _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, + _PyTok_SourceAppendLine(&source, "", 0) < 0, "accepted empty source line") < 0 || check_system_error( - _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0, + _PyTok_SourceAppendLine(&source, "a\nb\n", 4) < 0, "accepted multiple source lines") < 0 || - check_system_error( - _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0, - "accepted missing implicit newline") < 0 || check(_PyTok_SourceAppendLine( - &source, "alpha\n", 6, 0) == 0, + &source, "alpha\n", 6) == 0, "wrong first source offset") < 0 || check(_PyTok_SourceAppendLine( - &source, "\xce\xb2\n", 3, 1) == 6, - "wrong second source offset") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, 1) && - _PyTok_SourceLineIsImplicit(&source, 2), - "wrong implicit newline flags") < 0) { + &source, "\xce\xb2\n", 3) == 6, + "wrong second source offset") < 0) { goto error; } @@ -81,9 +75,9 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), } _PyTok_SourceClear(&source); - if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 || + if (_PyTok_SourceAppendLine(&source, "tail", 4) < 0 || check_system_error( - _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0, + _PyTok_SourceAppendLine(&source, "x\n", 2) < 0, "appended after unterminated source line") < 0) { goto error; } @@ -107,7 +101,7 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module), _PyTok_SourceText source; _PyTok_SourceInit(&source); for (int i = 0; i < 260; i++) { - if (_PyTok_SourceAppendLine(&source, "x\n", 2, 1) < 0) { + if (_PyTok_SourceAppendLine(&source, "x\n", 2) < 0) { goto error; } } @@ -121,10 +115,8 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module), goto error; } for (int i = 0; i < 260; i++) { - if (check(_PyTok_SourceAppendLine(&source, "y\n", 2, 0) == 520 + 2 * i, - "wrong source offset after discard") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, i + 1), - "discard preserved implicit newline flag") < 0) { + if (check(_PyTok_SourceAppendLine(&source, "y\n", 2) == 520 + 2 * i, + "wrong source offset after discard") < 0) { goto error; } } @@ -133,18 +125,18 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module), goto error; } _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "tail", 4, 0) == 1040, + if (check(_PyTok_SourceAppendLine(&source, "tail", 4) == 1040, "wrong source offset after repeated discard") < 0) { goto error; } _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) == 1044, + if (check(_PyTok_SourceAppendLine(&source, "z\n", 2) == 1044, "cannot append after discarding unterminated line") < 0) { goto error; } _PyTok_SourceDiscard(&source); source.base_offset = PY_SSIZE_T_MAX - 1; - if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) < 0 && + if (check(_PyTok_SourceAppendLine(&source, "z\n", 2) < 0 && PyErr_ExceptionMatches(PyExc_MemoryError), "accepted overflowing logical source offset") < 0) { goto error; diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 69c3bb371add960..d3c3f7202e9e336 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -354,12 +354,13 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, line = normalized; } _PyTok_Off appended = _PyTok_SourceAppendLine( - &tok->source, line, line_len, implicit); + &tok->source, line, line_len); if (appended < 0) { tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) ? E_NOMEM : E_ERROR; goto error; } + tok->reader->prepared_final_newline_is_implicit = implicit; pos += raw_line_len; } PyMem_Free(normalized); diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index c003deba18dd8f2..fd7225f63b37660 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -142,7 +142,6 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk) if (tok->lineno >= tok->source.nlines) { return _PYTOK_READ_EOF; } - int lineno = tok->lineno + 1; const char *start = _PyLexer_BufferPointer(tok, tok->inp); const char *newline = memchr( start, '\n', tok->source.bytes + tok->source.len - start); @@ -151,8 +150,8 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk) chunk->data = (char *)start; chunk->len = tok->source.bytes + end - start; chunk->ownership = _PYTOK_CHUNK_BORROWED; - chunk->implicit_newline = _PyTok_SourceLineIsImplicit( - &tok->source, lineno); + chunk->implicit_newline = end == tok->source.len && + tok->reader->prepared_final_newline_is_implicit; return _PYTOK_READ_LINE; } @@ -618,8 +617,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) reset_streaming_buffer(tok); } _PyTok_Off source_start = _PyTok_SourceAppendLine( - &tok->source, chunk.data, chunk.len, - chunk.implicit_newline); + &tok->source, chunk.data, chunk.len); if (source_start < 0) { _PyTok_ChunkClear(&chunk); tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 390b6ed9d8581a5..9638fef88cd593b 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -53,6 +53,7 @@ typedef struct _PyTok_Reader { Py_ssize_t decoded_len; Py_ssize_t decoded_cap; _PyTok_ReaderKind kind; + unsigned char prepared_final_newline_is_implicit; unsigned char decoded_tail_is_implicit; unsigned char file_initialized; unsigned char file_eof; diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index d69eab93923e81c..d08e09c597d7c68 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -12,7 +12,6 @@ void _PyTok_SourceClear(_PyTok_SourceText *source) { PyMem_Free(source->bytes); - PyMem_Free(source->implicit_lines); _PyTok_SourceInit(source); } @@ -25,10 +24,6 @@ _PyTok_SourceDiscard(_PyTok_SourceText *source) if (source->bytes != NULL) { source->bytes[0] = '\0'; } - if (source->implicit_lines != NULL) { - Py_ssize_t used = source->nlines / 8 + (source->nlines % 8 != 0); - memset(source->implicit_lines, 0, Py_MIN(used, source->implicit_cap)); - } source->nlines = 0; } @@ -71,35 +66,9 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) return 0; } -static int -reserve_implicit_lines(_PyTok_SourceText *source, int nlines) -{ - Py_ssize_t needed = nlines / 8 + (nlines % 8 != 0); - if (needed <= source->implicit_cap) { - return 0; - } - Py_ssize_t cap = source->implicit_cap > 0 ? source->implicit_cap : 16; - while (cap < needed) { - if (cap > PY_SSIZE_T_MAX / 2) { - cap = needed; - break; - } - cap *= 2; - } - unsigned char *lines = PyMem_Realloc(source->implicit_lines, cap); - if (lines == NULL) { - PyErr_NoMemory(); - return -1; - } - memset(lines + source->implicit_cap, 0, cap - source->implicit_cap); - source->implicit_lines = lines; - source->implicit_cap = cap; - return 0; -} - static int validate_line(const _PyTok_SourceText *source, const char *bytes, - Py_ssize_t len, int implicit_newline) + Py_ssize_t len) { if (len <= 0 || bytes == NULL || (source->nlines > 0 && @@ -108,8 +77,7 @@ validate_line(const _PyTok_SourceText *source, const char *bytes, return -1; } const char *newline = memchr(bytes, '\n', len); - if ((newline != NULL && newline != bytes + len - 1) || - (implicit_newline && newline == NULL)) { + if (newline != NULL && newline != bytes + len - 1) { PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line"); return -1; } @@ -123,9 +91,9 @@ validate_line(const _PyTok_SourceText *source, const char *bytes, _PyTok_Off _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, - Py_ssize_t len, int implicit_newline) + Py_ssize_t len) { - if (validate_line(source, bytes, len, implicit_newline) < 0) { + if (validate_line(source, bytes, len) < 0) { return -1; } if (source->len > PY_SSIZE_T_MAX - len - 1 || @@ -133,9 +101,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, PyErr_NoMemory(); return -1; } - int nlines = source->nlines + 1; - if ((implicit_newline && reserve_implicit_lines(source, nlines) < 0) || - reserve_bytes(source, source->len + len + 1) < 0) { + if (reserve_bytes(source, source->len + len + 1) < 0) { return -1; } @@ -143,11 +109,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, memcpy(source->bytes + start, bytes, len); source->len += len; source->bytes[source->len] = '\0'; - if (implicit_newline) { - source->implicit_lines[(nlines - 1) / 8] |= - (unsigned char)(1U << ((nlines - 1) & 7)); - } - source->nlines = nlines; + source->nlines++; return source->base_offset + start; } @@ -170,14 +132,3 @@ _PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno, *len = (newline != NULL ? newline : end) - line; return line; } - -int -_PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno) -{ - if (lineno < 1 || lineno > source->nlines || - (lineno - 1) / 8 >= source->implicit_cap) { - return 0; - } - return (source->implicit_lines[(lineno - 1) / 8] >> - ((lineno - 1) & 7)) & 1; -} diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index 9576419aade1d46..66980a41fb7c481 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -10,9 +10,7 @@ typedef struct { _PyTok_Off base_offset; _PyTok_Off len; _PyTok_Off cap; - unsigned char *implicit_lines; int nlines; - Py_ssize_t implicit_cap; } _PyTok_SourceText; static inline const char * @@ -29,19 +27,14 @@ PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *); PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *); /* Append one nonempty logical line and return its start offset. The input may contain one newline, as its final byte. An unterminated line must be the - final line. implicit_newline means that the final newline was synthesized. - The input must not point into source storage. */ + final line. The input must not point into source storage. */ PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine( - _PyTok_SourceText *source, const char *bytes, Py_ssize_t len, - int implicit_newline); + _PyTok_SourceText *source, const char *bytes, Py_ssize_t len); /* Return borrowed bytes excluding '\n', writing the byte length to *len. Line numbers are 1-based and clamp to the first or final line; a trailing '\n' adds an empty final line. The view need not be NUL-terminated. This does not set an exception. Append, discard, and clear invalidate the view. */ PyAPI_FUNC(const char *) _PyTok_SourceLineView( const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len); -/* Return false for invalid line numbers and the virtual EOF line. */ -PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit( - const _PyTok_SourceText *, int); #endif From 578b0c57c75a274e6bb2453ba4c5f94fd3ad8468 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:57:02 +0100 Subject: [PATCH 03/32] Reuse parsed formatted string kind in lexer --- Parser/lexer/lexer.c | 5 ++++- Parser/lexer/lexer_internal.h | 3 ++- Parser/lexer/string.c | 30 +++++------------------------- 3 files changed, 11 insertions(+), 27 deletions(-) diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 4652103107367ef..a4fdb9484abd8de 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -314,7 +314,10 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token // Handle valid f or t string creation: if (saw_f || saw_t) { - return _PyLexer_scan_fstring_start(tok, token, c); + ftstring_kind kind = saw_t + ? (saw_r ? RAW_TSTRING : TSTRING) + : (saw_r ? RAW_FSTRING : FSTRING); + return _PyLexer_scan_fstring_start(tok, token, c, kind); } return _PyLexer_scan_string(tok, token, c); } diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h index f0d9576d95f8ff3..652512a3e88c22d 100644 --- a/Parser/lexer/lexer_internal.h +++ b/Parser/lexer/lexer_internal.h @@ -66,7 +66,8 @@ int _PyLexer_close_ftstring_expr( void _PyLexer_mark_ftstring_debug(struct tok_state *, ftstring_state *); int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int); int _PyLexer_scan_number(struct tok_state *, struct token *, int, int); -int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int); +int _PyLexer_scan_fstring_start( + struct tok_state *, struct token *, int, ftstring_kind); int _PyLexer_scan_string(struct tok_state *, struct token *, int); int _PyLexer_get_normal(struct tok_state *, ftstring_state *, struct token *); int _PyLexer_get_ftstring(struct tok_state *, ftstring_state *, struct token *); diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index 41dc3c66edff09f..00c6543341ce75c 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -254,7 +254,8 @@ _PyLexer_check_string_prefixes(struct tok_state *tok, } int -_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) +_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, + int c, ftstring_kind kind) { _PyTok_Off p_start = -1; _PyTok_Off p_end = -1; @@ -293,30 +294,9 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) state->start_loc = tok->start_loc; state->expr_span = (_PyTok_Span){-1, -1}; - int raw = 0; - int tstring = 0; - switch (*_PyLexer_BufferPointer(tok, tok->start)) { - case 'T': - case 't': - raw = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 'r'; - tstring = 1; - break; - case 'F': - case 'f': - raw = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 'r'; - break; - case 'R': - case 'r': - raw = 1; - tstring = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 't'; - break; - default: - Py_UNREACHABLE(); - } - state->kind = tstring - ? (raw ? RAW_TSTRING : TSTRING) - : (raw ? RAW_FSTRING : FSTRING); - return tstring ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START); + state->kind = kind; + return _PyLexer_IsTString(kind) + ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START); } int From 072439ad580927b07dc2a1a96df0736b1dd26784 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:57:50 +0100 Subject: [PATCH 04/32] Transfer owned tokenizer encoding strings --- Parser/tokenizer/decoder.c | 31 ++++++++++++------------------- Parser/tokenizer/reader.c | 14 +++++--------- 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index d3c3f7202e9e336..1d438406bc7b461 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -179,17 +179,14 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next) if (cursor == start) { continue; } - char *found = _PyTok_CopyBytes(start, cursor - start); - if (found == NULL) { - return -1; - } + Py_ssize_t encoding_len = cursor - start; char normalized[13]; int n; - for (n = 0; n < 12 && found[n] != '\0'; n++) { - normalized[n] = found[n] == '_' ? '-' : Py_TOLOWER(found[n]); + for (n = 0; n < 12 && n < encoding_len; n++) { + normalized[n] = start[n] == '_' ? '-' : Py_TOLOWER(start[n]); } normalized[n] = '\0'; - const char *canonical = found; + const char *canonical = start; if (strcmp(normalized, "utf-8") == 0 || strncmp(normalized, "utf-8-", 6) == 0) { canonical = "utf-8"; @@ -202,14 +199,13 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next) strncmp(normalized, "iso-latin-1-", 12) == 0) { canonical = "iso-8859-1"; } - if (canonical != found) { - PyMem_Free(found); - found = _PyTok_CopyBytes(canonical, strlen(canonical)); - if (found == NULL) { - return -1; - } + if (canonical != start) { + encoding_len = strlen(canonical); + } + *encoding = _PyTok_CopyBytes(canonical, encoding_len); + if (*encoding == NULL) { + return -1; } - *encoding = found; *scan_next = 0; return 0; } @@ -269,11 +265,8 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, PyMem_Free(cookie); return _PYTOK_ENCODING_ERROR; } - if (!bom && _PyTok_SetEncoding(tok, cookie) < 0) { - PyMem_Free(cookie); - return _PYTOK_ENCODING_ERROR; - } - PyMem_Free(cookie); + PyMem_Free(tok->encoding); + tok->encoding = cookie; return _PYTOK_ENCODING_DONE; } diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index fd7225f63b37660..222118577f01e85 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -1,6 +1,7 @@ #include "Python.h" #include "pycore_fileutils.h" #include "pycore_pystate.h" +#include "pycore_runtime.h" #include "errcode.h" #include "helpers.h" @@ -794,21 +795,16 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) fclose(fp); return NULL; } - tok->filename = filename != NULL - ? Py_NewRef(filename) : PyUnicode_FromString(""); - if (tok->filename == NULL) { - fclose(fp); - _PyTokenizer_Free(tok); - return NULL; - } + _Py_DECLARE_STR(anon_string, ""); + tok->filename = Py_NewRef(filename != NULL ? filename : &_Py_STR(anon_string)); if (initialize_file(tok) < 0) { fclose(fp); _PyTokenizer_Free(tok); return NULL; } fclose(fp); - char *encoding = tok->encoding == NULL - ? NULL : _PyTok_CopyBytes(tok->encoding, strlen(tok->encoding)); + char *encoding = tok->encoding; + tok->encoding = NULL; _PyTokenizer_Free(tok); return encoding; } From 29f8d4a699b79bcd0e9d8532f6299f5804dcb1bc Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:57:50 +0100 Subject: [PATCH 05/32] Remove unused tokenizer state and redundant branches --- Parser/lexer/string.c | 2 +- Parser/tokenizer/reader.c | 5 +---- Parser/tokenizer/types.h | 6 ------ Python/Python-tokenize.c | 29 +++++++---------------------- 4 files changed, 9 insertions(+), 33 deletions(-) diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index 00c6543341ce75c..3c38e1ab7c32cec 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -13,7 +13,7 @@ string_error_token(struct tok_state *tok, struct token *token, { tok->diagnostic = (_PyTokenizer_Diagnostic){ .location = {location.lineno, location.byte_col + 1}, - .text_span = _PyTok_SpanFromBounds(start - location.byte_col, tok->inp), + .text_span = {start - location.byte_col, tok->inp}, }; int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, -1, -1); token->start_loc = location; diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 222118577f01e85..75e3bda929884fc 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -182,7 +182,7 @@ read_file_line(struct tok_state *tok, _PyTok_Chunk *chunk) break; } } - int implicit = len == 0 || reader->file_buffer[len - 1] != '\n'; + int implicit = reader->file_buffer[len - 1] != '\n'; chunk->data = reader->file_buffer; chunk->len = len; chunk->implicit_newline = implicit; @@ -466,9 +466,6 @@ next_readline(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_ERROR; } } - if (pop_decoded_line(reader, chunk)) { - return _PYTOK_READ_LINE; - } } } diff --git a/Parser/tokenizer/types.h b/Parser/tokenizer/types.h index c12b6de449c7008..b488cb01618d35f 100644 --- a/Parser/tokenizer/types.h +++ b/Parser/tokenizer/types.h @@ -18,12 +18,6 @@ typedef struct { int byte_col; } _PyTok_Loc; -static inline _PyTok_Span -_PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end) -{ - return (_PyTok_Span){start, end}; -} - static inline int _PyTok_SpanIsValid(_PyTok_Span span) { diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 3802709732e29da..ced88dc9ecebfb5 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -37,7 +37,6 @@ typedef struct /* Needed to cache line for performance */ PyObject *last_line; Py_ssize_t last_lineno; - Py_ssize_t last_end_lineno; Py_ssize_t byte_col_offset_diff; } tokenizeriterobject; @@ -79,18 +78,15 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, self->last_line = NULL; self->byte_col_offset_diff = 0; self->last_lineno = 0; - self->last_end_lineno = 0; return (PyObject *)self; } -static int +static void _tokenizer_error(tokenizeriterobject *it) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); - if (PyErr_Occurred()) { - return -1; - } + assert(!PyErr_Occurred()); const char *msg = NULL; PyObject* errtype = PyExc_SyntaxError; @@ -105,19 +101,17 @@ _tokenizer_error(tokenizeriterobject *it) PyErr_SyntaxLocationObject( info.filename, info.location.lineno, (int)Py_MAX(0, info.input_span.end - info.input_span.start)); - return -1; + return; case E_DEDENT: msg = "unindent does not match any outer indentation level"; errtype = PyExc_IndentationError; break; case E_INTR: - if (!PyErr_Occurred()) { - PyErr_SetNone(PyExc_KeyboardInterrupt); - } - return -1; + PyErr_SetNone(PyExc_KeyboardInterrupt); + return; case E_NOMEM: PyErr_NoMemory(); - return -1; + return; case E_TABSPACE: errtype = PyExc_TabError; msg = "inconsistent use of tabs and spaces in indentation"; @@ -138,7 +132,6 @@ _tokenizer_error(tokenizeriterobject *it) PyObject* error_line = NULL; PyObject* tmp = NULL; PyObject* value = NULL; - int result = 0; Py_ssize_t input_size; const char *input = _PyTokenizer_SpanView( @@ -148,30 +141,25 @@ _tokenizer_error(tokenizeriterobject *it) size -= 1; // Remove the newline character from the end of the line error_line = PyUnicode_DecodeUTF8(input, size, "replace"); if (!error_line) { - result = -1; goto exit; } Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, input_size); if (offset == -1) { - result = -1; goto exit; } tmp = Py_BuildValue("(OnnOOO)", info.filename, info.location.lineno, offset, error_line, Py_None, Py_None); if (!tmp) { - result = -1; goto exit; } errstr = PyUnicode_FromString(msg); if (!errstr) { - result = -1; goto exit; } value = _PyTuple_FromPair(errstr, tmp); if (!value) { - result = -1; goto exit; } @@ -182,7 +170,6 @@ _tokenizer_error(tokenizeriterobject *it) Py_XDECREF(error_line); Py_XDECREF(tmp); Py_XDECREF(value); - return result; } static PyObject * @@ -243,7 +230,6 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, } } it->last_lineno = lineno; - it->last_end_lineno = end_lineno; } static PyObject * @@ -266,9 +252,8 @@ tokenizeriter_next(PyObject *op) } goto exit; } - if (it->done || type == ERRORTOKEN) { + if (it->done) { PyErr_SetString(PyExc_StopIteration, "EOF"); - it->done = 1; goto exit; } _PyToken_View view; From c911eb490e50fbeb571304fbd12b052fa4250225 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:56:26 +0100 Subject: [PATCH 06/32] Avoid rescanning partial decoded tokenizer lines --- Lib/test/test_tokenize.py | 14 ++++++++++++++ ...-09-24-18-20-00.gh-issue-153569.diagnostics.rst | 1 + Parser/tokenizer/reader.c | 11 +++++++++-- Parser/tokenizer/reader_internal.h | 1 + 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index cb8f58511aab877..95c49ec578cd015 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2341,6 +2341,20 @@ def test_utf8_decoder_spans_readline_calls(self): tokenize.TokenInfo(token.ENDMARKER, "", (2, 0), (2, 0), ""), ]) + def test_utf8_decoder_spans_many_readline_calls(self): + for prefix in (b"", b"previous\n"): + with self.subTest(prefix=prefix): + chunks = ([prefix + b"x\xc3"] + [b"\xa9\xc3"] * 100 + + [b"\xa9\n", b"z\xc3", b"\xa9\n", b""]) + source = b"".join(chunks) + expected = list(_tokenize.TokenizerIter( + BytesIO(source).readline, encoding="utf-8", extra_tokens=True + )) + tokens = list(_tokenize.TokenizerIter( + iter(chunks).__next__, encoding="utf-8", extra_tokens=True + )) + self.assertEqual(tokens, expected) + def test_utf8_decoder_replaces_incomplete_input_at_eof(self): expected = [ tokenize.TokenInfo(token.NAME, "x�", (1, 0), (1, 2), "x�"), diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst index 2b0aae1a3d2b4c4..4ae20f3436d238d 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst @@ -1 +1,2 @@ Correct missing-brace error messages for multiline f-strings and t-strings. +Improve tokenization of UTF-8 input split across many readline calls. diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 75e3bda929884fc..c686b39a99567d0 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -72,6 +72,7 @@ append_decoded(_PyTok_Reader *reader, const char *data, Py_ssize_t len) Py_ssize_t remaining = reader->decoded_len - reader->decoded_pos; memmove(reader->decoded, reader->decoded + reader->decoded_pos, (size_t)remaining); + reader->decoded_scan -= reader->decoded_pos; reader->decoded_pos = 0; reader->decoded_len = remaining; } @@ -105,13 +106,17 @@ static int pop_decoded_line(_PyTok_Reader *reader, _PyTok_Chunk *chunk) { assert(reader->decoded_pos >= 0 && reader->decoded_pos <= reader->decoded_len); + assert(reader->decoded_scan >= reader->decoded_pos && + reader->decoded_scan <= reader->decoded_len); if (reader->decoded_pos == reader->decoded_len) { return 0; } char *start = reader->decoded + reader->decoded_pos; - char *newline = memchr(start, '\n', - reader->decoded_len - reader->decoded_pos); + // Previously scanned bytes cannot contain a newline. + char *newline = memchr(reader->decoded + reader->decoded_scan, '\n', + reader->decoded_len - reader->decoded_scan); if (newline == NULL) { + reader->decoded_scan = reader->decoded_len; return 0; } Py_ssize_t len = newline - start + 1; @@ -119,10 +124,12 @@ pop_decoded_line(_PyTok_Reader *reader, _PyTok_Chunk *chunk) chunk->len = len; chunk->ownership = _PYTOK_CHUNK_BORROWED; reader->decoded_pos += len; + reader->decoded_scan = reader->decoded_pos; chunk->implicit_newline = reader->decoded_pos == reader->decoded_len && reader->decoded_tail_is_implicit; if (reader->decoded_pos == reader->decoded_len) { reader->decoded_pos = reader->decoded_len = 0; + reader->decoded_scan = 0; reader->decoded_tail_is_implicit = 0; } return 1; diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 9638fef88cd593b..fff2cc9db86be40 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -50,6 +50,7 @@ typedef struct _PyTok_Reader { char *decoded; Py_ssize_t decoded_pos; + Py_ssize_t decoded_scan; Py_ssize_t decoded_len; Py_ssize_t decoded_cap; _PyTok_ReaderKind kind; From f0149a8fff06f445c0eb9ecae49e1d1805da5f24 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:58:01 +0100 Subject: [PATCH 07/32] Reuse prepared tokenizer line terminators for CR detection --- Parser/tokenizer/decoder.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 1d438406bc7b461..653b5748b5729ad 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -322,12 +322,14 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, data[pos + raw_line_len - 1] == '\r'; int add_newline = add_final_newline && pos + raw_line_len == len && !terminated; - int normalize = add_newline || - (!preserve_crlf && - memchr(data + pos, '\r', raw_line_len) != NULL); - const char *line = data + pos; Py_ssize_t line_len = raw_line_len; + // raw_line_length stops at the first CR or LF, so any CR is in + // the line terminator. + int normalize = add_newline || + (!preserve_crlf && + (line[line_len - 1] == '\r' || + (line_len > 1 && line[line_len - 2] == '\r'))); int implicit = 0; if (normalize) { if (line_len > PY_SSIZE_T_MAX - 2) { From c5465b306c1fa850a4483beaeb5ebc1037e320d0 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 18:00:36 +0100 Subject: [PATCH 08/32] Reuse interactive tokenizer chunks without carriage returns --- Lib/test/test_repl.py | 13 +++++++++++++ Parser/tokenizer/reader.c | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 21e603c561d73c0..e1ce17b4dd4256c 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -214,6 +214,19 @@ def test_multiline_fstring_source_reallocation(self): self.assertEqual(p.returncode, 0) self.assertIn(">>> 3\n>>> ", output) + def test_long_non_ascii_input(self): + value = "é漢" * 5000 + for newline in ("\n", "\r\n"): + with self.subTest(newline=newline): + p = spawn_repl(encoding="utf-8") + p.stdin.write( + f"value = {value!r}{newline}" + f"print(len(value), value[:2]){newline}" + ) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + self.assertIn(">>> 10000 é漢\n>>> ", output) + def test_close_stdin(self): user_input = dedent(''' import os diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index c686b39a99567d0..68fdb2f37e4c927 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -507,6 +507,10 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) _PyTok_ChunkClear(&decoded); return _PYTOK_READ_ERROR; } + if (memchr(decoded.data, '\r', decoded.len) == NULL) { + *chunk = decoded; + return _PYTOK_READ_LINE; + } chunk->data = _PyTok_NormalizeNewlines( decoded.data, decoded.len, 0, 0, &chunk->len, NULL); From 02764ba2b1ce074dfa074a8637dba162b0a20f3b Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:57:12 +0100 Subject: [PATCH 09/32] Release tokenizer constructor resources on allocation failure --- Python/Python-tokenize.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index ced88dc9ecebfb5..882ef134c9e8f91 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -56,21 +56,18 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, int extra_tokens, const char *encoding) /*[clinic end generated code: output=7501a1211683ce16 input=f7dddf8a613ae8bd]*/ { - tokenizeriterobject *self = (tokenizeriterobject *)type->tp_alloc(type, 0); - if (self == NULL) { - return NULL; - } - PyObject *filename = PyUnicode_FromString(""); - if (filename == NULL) { + struct tok_state *tok = _PyTokenizer_FromReadline(readline, encoding); + if (tok == NULL) { return NULL; } - self->tok = _PyTokenizer_FromReadline(readline, encoding); - if (self->tok == NULL) { - Py_DECREF(filename); + _Py_DECLARE_STR(anon_string, ""); + _PyTokenizer_SetContext(tok, &_Py_STR(anon_string), NULL); + tokenizeriterobject *self = (tokenizeriterobject *)type->tp_alloc(type, 0); + if (self == NULL) { + _PyTokenizer_Free(tok); return NULL; } - _PyTokenizer_SetContext(self->tok, filename, NULL); - Py_DECREF(filename); + self->tok = tok; _PyTokenizer_SetOptions(self->tok, extra_tokens, 0); self->extra_tokens = extra_tokens; self->done = 0; From c04c0bbfebad218ea4443df661d81ee4c7cfe865 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:57:50 +0100 Subject: [PATCH 10/32] Narrow interactive newline normalization wrapper --- Parser/tokenizer/decoder.c | 21 +++++++-------------- Parser/tokenizer/reader.c | 3 +-- Parser/tokenizer/reader_internal.h | 3 +-- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 653b5748b5729ad..d30d5e7a0227c5e 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -77,7 +77,7 @@ chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk, static void normalize_newlines_into(char *result, const char *data, Py_ssize_t len, int preserve_crlf, int add_final_newline, - Py_ssize_t *out_len, int *implicit_newline) + Py_ssize_t *out_len) { Py_ssize_t write = 0; if (preserve_crlf || memchr(data, '\r', len) == NULL) { @@ -97,21 +97,16 @@ normalize_newlines_into(char *result, const char *data, Py_ssize_t len, result[write++] = c; } } - int implicit = add_final_newline && write > 0 && result[write - 1] != '\n'; - if (implicit) { + if (add_final_newline && write > 0 && result[write - 1] != '\n') { result[write++] = '\n'; } result[write] = '\0'; *out_len = write; - if (implicit_newline != NULL) { - *implicit_newline = implicit; - } } char * -_PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf, - int add_final_newline, Py_ssize_t *out_len, - int *implicit_newline) +_PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, + Py_ssize_t *out_len) { if (len > PY_SSIZE_T_MAX - 2) { PyErr_NoMemory(); @@ -122,8 +117,7 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf, PyErr_NoMemory(); return NULL; } - normalize_newlines_into(result, data, len, preserve_crlf, - add_final_newline, out_len, implicit_newline); + normalize_newlines_into(result, data, len, 0, 0, out_len); return result; } @@ -330,7 +324,6 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, (!preserve_crlf && (line[line_len - 1] == '\r' || (line_len > 1 && line[line_len - 2] == '\r'))); - int implicit = 0; if (normalize) { if (line_len > PY_SSIZE_T_MAX - 2) { PyErr_NoMemory(); @@ -345,7 +338,7 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, } normalize_newlines_into(normalized, line, line_len, preserve_crlf, add_newline, - &line_len, &implicit); + &line_len); line = normalized; } _PyTok_Off appended = _PyTok_SourceAppendLine( @@ -355,7 +348,7 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, ? E_NOMEM : E_ERROR; goto error; } - tok->reader->prepared_final_newline_is_implicit = implicit; + tok->reader->prepared_final_newline_is_implicit = add_newline; pos += raw_line_len; } PyMem_Free(normalized); diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 68fdb2f37e4c927..9859c650248287c 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -512,8 +512,7 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_LINE; } chunk->data = _PyTok_NormalizeNewlines( - decoded.data, decoded.len, 0, 0, - &chunk->len, NULL); + decoded.data, decoded.len, &chunk->len); _PyTok_ChunkClear(&decoded); if (chunk->data == NULL) { PyErr_NoMemory(); diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index fff2cc9db86be40..26e5957b68af70c 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -69,8 +69,7 @@ int _PyTok_ReserveBuffer(char **, Py_ssize_t *, Py_ssize_t, Py_ssize_t); char *_PyTok_CopyBytes(const char *, Py_ssize_t); int _PyTok_DecodeOnce( struct tok_state *, _PyTok_Chunk *, const char *, const char *); -char *_PyTok_NormalizeNewlines( - const char *, Py_ssize_t, int, int, Py_ssize_t *, int *); +char *_PyTok_NormalizeNewlines(const char *, Py_ssize_t, Py_ssize_t *); void _PyTok_ChunkClear(_PyTok_Chunk *); int _PyTok_SetEncoding(struct tok_state *, const char *); _PyTok_EncodingResult _PyTok_DetectEncoding( From b44bde7fbad0be081dd126bbd864d44cfcf932c8 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 18:40:03 +0100 Subject: [PATCH 11/32] Close duplicated tokenizer descriptors when fdopen fails --- Parser/tokenizer/reader.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 9859c650248287c..494aca408615760 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -786,7 +786,16 @@ static FILE * fdopen_borrow(int fd) { int copy = _Py_dup(fd); - return copy < 0 ? NULL : fdopen(copy, "r"); + if (copy < 0) { + return NULL; + } + FILE *fp = fdopen(copy, "r"); + if (fp == NULL) { + int saved_errno = errno; + close(copy); + errno = saved_errno; + } + return fp; } #endif From 1e44edcb9344b0aff14e24c8facb29294769765d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 18:40:03 +0100 Subject: [PATCH 12/32] Preserve cached tokenizer lines when newline allocation fails --- Python/Python-tokenize.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 882ef134c9e8f91..cedb1f7e3f160e4 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -309,8 +309,8 @@ tokenizeriter_next(PyObject *op) type = OP; } else if (type == NEWLINE) { - Py_DECREF(str); if (!view.implicit_newline) { + Py_DECREF(str); assert(token_start != NULL); if (token_start[0] == '\r') { str = PyUnicode_FromString("\r\n"); @@ -328,7 +328,6 @@ tokenizeriter_next(PyObject *op) } if (str == NULL) { - Py_DECREF(line); goto exit; } } From f995e1ac928f81a56bdca6e78a3cebb732d28d06 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 18:40:54 +0100 Subject: [PATCH 13/32] Collect cycles involving tokenizer readline callbacks --- Lib/test/test_tokenize.py | 13 +++++++++++++ Parser/tokenizer/reader.c | 18 ++++++++++++++++++ Parser/tokenizer/tokenizer.h | 1 + Python/Python-tokenize.c | 13 ++++++++++++- 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 95c49ec578cd015..de3a7fc3503432c 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -8,6 +8,7 @@ import token import tokenize import unittest +import weakref from io import BytesIO, StringIO from textwrap import dedent from unittest import TestCase, mock @@ -2268,6 +2269,18 @@ def check_tokenize(self, s, expected): ) self.assertEqual(result, expected.rstrip().splitlines()) + def test_readline_reference_cycle(self): + class Readline: + def __call__(self): + return "" + + readline = Readline() + readline.iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) + ref = weakref.ref(readline) + del readline + support.gc_collect() + self.assertIsNone(ref()) + def test_encoding(self): def readline(encoding): yield "1+1".encode(encoding) diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 494aca408615760..298a9e64a46188f 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -38,6 +38,24 @@ _PyTok_ReaderFree(struct tok_state *tok) tok->reader = NULL; } +int +_PyTokenizer_Traverse(struct tok_state *tok, visitproc visit, void *arg) +{ + Py_VISIT(tok->filename); + Py_VISIT(tok->module); + _PyTok_Reader *reader = tok->reader; + Py_VISIT(reader->readline); + Py_VISIT(reader->decoder); + for (int i = 0; + i < (int)Py_ARRAY_LENGTH(reader->prefetched_lines); i++) { + _PyTok_Chunk *chunk = &reader->prefetched_lines[i]; + if (chunk->ownership == _PYTOK_CHUNK_PYOBJECT) { + Py_VISIT(chunk->owner); + } + } + return 0; +} + int _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed, Py_ssize_t initial_capacity) diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h index 82a84830d7cbfe7..2ef87add3646c3c 100644 --- a/Parser/tokenizer/tokenizer.h +++ b/Parser/tokenizer/tokenizer.h @@ -58,6 +58,7 @@ typedef struct { Errors are returned as ERRORTOKEN, with or without a Python exception. */ void _PyTokenizer_Get(struct tok_state *, struct token *); void _PyTokenizer_Free(struct tok_state *); +int _PyTokenizer_Traverse(struct tok_state *, visitproc, void *); void _PyTokenizer_raise_init_error(PyObject *filename); void _PyToken_Init(struct token *); static inline void diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index cedb1f7e3f160e4..a10200bcd5ff6df 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -348,15 +348,26 @@ tokenizeriter_dealloc(PyObject *op) { tokenizeriterobject *it = (tokenizeriterobject*)op; PyTypeObject *tp = Py_TYPE(it); + PyObject_GC_UnTrack(it); Py_XDECREF(it->last_line); _PyTokenizer_Free(it->tok); tp->tp_free(it); Py_DECREF(tp); } +static int +tokenizeriter_traverse(PyObject *op, visitproc visit, void *arg) +{ + tokenizeriterobject *it = (tokenizeriterobject *)op; + Py_VISIT(Py_TYPE(it)); + Py_VISIT(it->last_line); + return _PyTokenizer_Traverse(it->tok, visit, arg); +} + static PyType_Slot tokenizeriter_slots[] = { {Py_tp_new, tokenizeriter_new}, {Py_tp_dealloc, tokenizeriter_dealloc}, + {Py_tp_traverse, tokenizeriter_traverse}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, tokenizeriter_next}, @@ -366,7 +377,7 @@ static PyType_Slot tokenizeriter_slots[] = { static PyType_Spec tokenizeriter_spec = { .name = "_tokenize.TokenizerIter", .basicsize = sizeof(tokenizeriterobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), .slots = tokenizeriter_slots, }; From 6e43c2713c15eabfa0be823ffb3cb3cd629eae6c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 18:57:59 +0100 Subject: [PATCH 14/32] Preserve buffered text when decoding source files --- Lib/test/test_source_encoding.py | 5 +++++ .../2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst | 1 + Parser/tokenizer/reader.c | 3 ++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index ec98e609c4e98f9..63f3126c99925cc 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -135,6 +135,11 @@ def test_stateful_file_decoder_spans_lines(self): ) self._assert_python_file_ok(source) + @support.requires_subprocess() + def test_stateful_file_decoder_preserves_buffered_text(self): + source = b"# coding: hz\nx~\ny = 1\nassert xy == 1\n" + self._assert_python_file_ok(source) + @support.requires_subprocess() def test_stateful_file_decoder_finalizes_before_implicit_newline(self): source = b"# coding: hz\n# ~{1dA?" diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst index 4ae20f3436d238d..a1b233bc758f951 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst @@ -1,2 +1,3 @@ Correct missing-brace error messages for multiline f-strings and t-strings. Improve tokenization of UTF-8 input split across many readline calls. +Preserve source order when reading files with stateful encodings. diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 298a9e64a46188f..e692ae8ac2a24fe 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -350,7 +350,8 @@ next_file(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_LINE; } int decoded = _PyTok_DecodeChunk(tok, &input, 0); - if (decoded == 0 && chunk_is_line(&input)) { + if (decoded == 0 && reader->decoded_pos == reader->decoded_len && + chunk_is_line(&input)) { *chunk = input; return _PYTOK_READ_LINE; } From a36c95ca9e28f5aaf69c879f1623b99cf8e6fe87 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 18:58:00 +0100 Subject: [PATCH 15/32] Clear the transferred syntax error line before cleanup --- Parser/pegen_errors.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 74ab56c912ae1af..6997a1158e9310f 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -327,6 +327,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, } tmp = Py_BuildValue("(OnnNnn)", info.filename, lineno, col_number, error_line, end_lineno, end_col_number); + error_line = NULL; if (!tmp) { goto error; } From ffd9ac333cae95d081aa8217aea0b4167232c458 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 18:58:00 +0100 Subject: [PATCH 16/32] Propagate tokenizer column conversion failures --- Parser/pegen.c | 3 +++ Python/Python-tokenize.c | 19 ++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Parser/pegen.c b/Parser/pegen.c index 7b228ca874bc0ba..db1b4454ddc4d87 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -36,6 +36,9 @@ Py_ssize_t _PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset) { const unsigned char *data = (const unsigned char*)PyUnicode_AsUTF8(line); + if (data == NULL) { + return -1; + } Py_ssize_t len = 0; while (col_offset < end_col_offset) { diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index a10200bcd5ff6df..f9b8a949ab5649e 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -190,7 +190,7 @@ _get_current_line(tokenizeriterobject *it, int current_lineno, return line; } -static void +static int _get_col_offsets(tokenizeriterobject *it, const struct token *token, const char *token_start, const char *line_start, const char *end_line_start, PyObject *line, int line_changed, @@ -206,6 +206,9 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, byte_offset = token_start - line_start; if (line_changed) { *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset); + if (*col_offset < 0) { + return -1; + } it->byte_col_offset_diff = byte_offset - *col_offset; } else { @@ -218,15 +221,22 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, if (lineno == end_lineno) { // Avoid rescanning the prefix of a very long line. Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); + if (token_col_offset < 0) { + return -1; + } *end_col_offset = *col_offset + token_col_offset; it->byte_col_offset_diff += token_end - token_start - token_col_offset; } else { *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(end_line_start, end_byte_offset); + if (*end_col_offset < 0) { + return -1; + } it->byte_col_offset_diff += end_byte_offset - *end_col_offset; } } it->last_lineno = lineno; + return 0; } static PyObject * @@ -295,8 +305,11 @@ tokenizeriter_next(PyObject *op) Py_ssize_t end_lineno = token.end_loc.lineno; Py_ssize_t col_offset = -1; Py_ssize_t end_col_offset = -1; - _get_col_offsets(it, &token, token_start, view.line, view.end_line, line, - line_changed, &col_offset, &end_col_offset); + if (_get_col_offsets(it, &token, token_start, view.line, view.end_line, line, + line_changed, &col_offset, &end_col_offset) < 0) { + Py_DECREF(str); + goto exit; + } if (it->extra_tokens) { if (is_trailing_token) { From 6fefac84a1d5561c3ddbbf72e312d9ad5b0c1805 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 19:21:09 +0100 Subject: [PATCH 17/32] Allocate parser repetition buffers on first match --- Parser/parser.c | 780 +++++------------- .../peg_generator/pegen/c_generator_rules.py | 13 +- 2 files changed, 198 insertions(+), 595 deletions(-) diff --git a/Parser/parser.c b/Parser/parser.c index 896c089e3360035..9f1ee169d0ad886 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -27350,14 +27350,8 @@ _loop0_1_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // NEWLINE if (p->error_indicator) { @@ -27372,7 +27366,7 @@ _loop0_1_rule(Parser *p) { _res = newline_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -27417,14 +27411,8 @@ _loop1_2_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // statement if (p->error_indicator) { @@ -27439,7 +27427,7 @@ _loop1_2_rule(Parser *p) { _res = statement_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -27489,14 +27477,8 @@ _loop0_3_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ';' simple_stmt if (p->error_indicator) { @@ -27520,7 +27502,7 @@ _loop0_3_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28043,14 +28025,8 @@ _loop1_12_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (star_targets '=') if (p->error_indicator) { @@ -28065,7 +28041,7 @@ _loop1_12_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28115,14 +28091,8 @@ _loop0_13_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' NAME if (p->error_indicator) { @@ -28146,7 +28116,7 @@ _loop0_13_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28335,14 +28305,8 @@ _loop0_17_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('.' | '...') if (p->error_indicator) { @@ -28357,7 +28321,7 @@ _loop0_17_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28402,14 +28366,8 @@ _loop1_18_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('.' | '...') if (p->error_indicator) { @@ -28424,7 +28382,7 @@ _loop1_18_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28474,14 +28432,8 @@ _loop0_19_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' import_from_as_name if (p->error_indicator) { @@ -28505,7 +28457,7 @@ _loop0_19_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28637,14 +28589,8 @@ _loop0_22_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' dotted_as_name if (p->error_indicator) { @@ -28668,7 +28614,7 @@ _loop0_22_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28754,14 +28700,8 @@ _loop1_24_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('@' named_expression NEWLINE) if (p->error_indicator) { @@ -28776,7 +28716,7 @@ _loop1_24_rule(Parser *p) { _res = _tmp_159_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28921,14 +28861,8 @@ _loop0_27_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_no_default if (p->error_indicator) { @@ -28943,7 +28877,7 @@ _loop0_27_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28988,14 +28922,8 @@ _loop0_28_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_with_default if (p->error_indicator) { @@ -29010,7 +28938,7 @@ _loop0_28_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29055,14 +28983,8 @@ _loop1_29_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_no_default if (p->error_indicator) { @@ -29077,7 +28999,7 @@ _loop1_29_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29127,14 +29049,8 @@ _loop1_30_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_with_default if (p->error_indicator) { @@ -29149,7 +29065,7 @@ _loop1_30_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29199,14 +29115,8 @@ _loop0_31_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_maybe_default if (p->error_indicator) { @@ -29221,7 +29131,7 @@ _loop0_31_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29266,14 +29176,8 @@ _loop1_32_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_maybe_default if (p->error_indicator) { @@ -29288,7 +29192,7 @@ _loop1_32_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29338,14 +29242,8 @@ _loop0_33_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' with_item if (p->error_indicator) { @@ -29369,7 +29267,7 @@ _loop0_33_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29531,14 +29429,8 @@ _loop1_36_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // except_block if (p->error_indicator) { @@ -29553,7 +29445,7 @@ _loop1_36_rule(Parser *p) { _res = except_block_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29603,14 +29495,8 @@ _loop1_37_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // except_star_block if (p->error_indicator) { @@ -29625,7 +29511,7 @@ _loop1_37_rule(Parser *p) { _res = except_star_block_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29675,14 +29561,8 @@ _loop1_38_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // case_block if (p->error_indicator) { @@ -29697,7 +29577,7 @@ _loop1_38_rule(Parser *p) { _res = case_block_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29747,14 +29627,8 @@ _loop0_39_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // '|' closed_pattern if (p->error_indicator) { @@ -29778,7 +29652,7 @@ _loop0_39_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30073,14 +29947,8 @@ _loop0_44_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' maybe_star_pattern if (p->error_indicator) { @@ -30104,7 +29972,7 @@ _loop0_44_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30190,14 +30058,8 @@ _loop0_46_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' key_value_pattern if (p->error_indicator) { @@ -30221,7 +30083,7 @@ _loop0_46_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30364,14 +30226,8 @@ _loop0_49_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' pattern if (p->error_indicator) { @@ -30395,7 +30251,7 @@ _loop0_49_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30481,14 +30337,8 @@ _loop0_51_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' keyword_pattern if (p->error_indicator) { @@ -30512,7 +30362,7 @@ _loop0_51_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30598,14 +30448,8 @@ _loop0_53_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' type_param if (p->error_indicator) { @@ -30629,7 +30473,7 @@ _loop0_53_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30715,14 +30559,8 @@ _loop1_55_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' expression) if (p->error_indicator) { @@ -30737,7 +30575,7 @@ _loop1_55_rule(Parser *p) { _res = _tmp_16_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30787,14 +30625,8 @@ _loop1_56_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' star_expression) if (p->error_indicator) { @@ -30809,7 +30641,7 @@ _loop1_56_rule(Parser *p) { _res = _tmp_160_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30859,14 +30691,8 @@ _loop0_57_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' star_named_expression if (p->error_indicator) { @@ -30890,7 +30716,7 @@ _loop0_57_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30976,14 +30802,8 @@ _loop1_59_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('or' conjunction) if (p->error_indicator) { @@ -30998,7 +30818,7 @@ _loop1_59_rule(Parser *p) { _res = _tmp_161_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31048,14 +30868,8 @@ _loop1_60_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('and' inversion) if (p->error_indicator) { @@ -31070,7 +30884,7 @@ _loop1_60_rule(Parser *p) { _res = _tmp_162_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31120,14 +30934,8 @@ _loop1_61_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // compare_op_bitwise_or_pair if (p->error_indicator) { @@ -31142,7 +30950,7 @@ _loop1_61_rule(Parser *p) { _res = compare_op_bitwise_or_pair_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31235,14 +31043,8 @@ _loop0_63_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (slice | starred_expression) if (p->error_indicator) { @@ -31266,7 +31068,7 @@ _loop0_63_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31683,14 +31485,8 @@ _loop0_70_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_no_default if (p->error_indicator) { @@ -31705,7 +31501,7 @@ _loop0_70_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31750,14 +31546,8 @@ _loop0_71_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_with_default if (p->error_indicator) { @@ -31772,7 +31562,7 @@ _loop0_71_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31817,14 +31607,8 @@ _loop1_72_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_no_default if (p->error_indicator) { @@ -31839,7 +31623,7 @@ _loop1_72_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31889,14 +31673,8 @@ _loop1_73_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_with_default if (p->error_indicator) { @@ -31911,7 +31689,7 @@ _loop1_73_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31961,14 +31739,8 @@ _loop0_74_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_maybe_default if (p->error_indicator) { @@ -31983,7 +31755,7 @@ _loop0_74_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32028,14 +31800,8 @@ _loop1_75_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_maybe_default if (p->error_indicator) { @@ -32050,7 +31816,7 @@ _loop1_75_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32100,14 +31866,8 @@ _loop0_76_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // fstring_format_spec if (p->error_indicator) { @@ -32122,7 +31882,7 @@ _loop0_76_rule(Parser *p) { _res = fstring_format_spec_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32167,14 +31927,8 @@ _loop0_77_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // fstring_middle if (p->error_indicator) { @@ -32189,7 +31943,7 @@ _loop0_77_rule(Parser *p) { _res = fstring_middle_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32234,14 +31988,8 @@ _loop0_78_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // tstring_format_spec if (p->error_indicator) { @@ -32256,7 +32004,7 @@ _loop0_78_rule(Parser *p) { _res = tstring_format_spec_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32301,14 +32049,8 @@ _loop0_79_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // tstring_middle if (p->error_indicator) { @@ -32323,7 +32065,7 @@ _loop0_79_rule(Parser *p) { _res = tstring_middle_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32368,14 +32110,8 @@ _loop1_80_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (fstring | string) if (p->error_indicator) { @@ -32390,7 +32126,7 @@ _loop1_80_rule(Parser *p) { _res = _tmp_154_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32440,14 +32176,8 @@ _loop1_81_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // tstring if (p->error_indicator) { @@ -32462,7 +32192,7 @@ _loop1_81_rule(Parser *p) { _res = tstring_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32512,14 +32242,8 @@ _loop0_82_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' display_item if (p->error_indicator) { @@ -32543,7 +32267,7 @@ _loop0_82_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32678,14 +32402,8 @@ _loop0_85_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' double_starred_kvpair if (p->error_indicator) { @@ -32709,7 +32427,7 @@ _loop0_85_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32795,14 +32513,8 @@ _loop1_87_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // for_if_clause if (p->error_indicator) { @@ -32817,7 +32529,7 @@ _loop1_87_rule(Parser *p) { _res = for_if_clause_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32867,14 +32579,8 @@ _loop0_88_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('if' disjunction) if (p->error_indicator) { @@ -32889,7 +32595,7 @@ _loop0_88_rule(Parser *p) { _res = _tmp_164_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -33012,14 +32718,8 @@ _loop0_90_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (starred_expression | (assignment_expression | expression !':=') !'=') if (p->error_indicator) { @@ -33043,7 +32743,7 @@ _loop0_90_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -33176,14 +32876,8 @@ _loop0_93_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' kwarg_or_starred if (p->error_indicator) { @@ -33207,7 +32901,7 @@ _loop0_93_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -33339,14 +33033,8 @@ _loop0_96_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' kwarg_or_double_starred if (p->error_indicator) { @@ -33370,7 +33058,7 @@ _loop0_96_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -33456,14 +33144,8 @@ _loop0_98_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' star_target) if (p->error_indicator) { @@ -33478,7 +33160,7 @@ _loop0_98_rule(Parser *p) { _res = _tmp_166_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -33639,14 +33321,8 @@ _loop0_101_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' del_target if (p->error_indicator) { @@ -33670,7 +33346,7 @@ _loop0_101_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -33756,14 +33432,8 @@ _loop0_103_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' expression if (p->error_indicator) { @@ -33787,7 +33457,7 @@ _loop0_103_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -33973,14 +33643,8 @@ _loop0_107_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (starred_expression !'=') if (p->error_indicator) { @@ -34004,7 +33668,7 @@ _loop0_107_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -34289,14 +33953,8 @@ _loop1_113_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (!STRING expression_without_invalid) if (p->error_indicator) { @@ -34311,7 +33969,7 @@ _loop1_113_rule(Parser *p) { _res = _tmp_169_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -34744,14 +34402,8 @@ _loop0_119_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // star_named_expressions if (p->error_indicator) { @@ -34766,7 +34418,7 @@ _loop0_119_rule(Parser *p) { _res = star_named_expressions_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -34811,14 +34463,8 @@ _loop0_120_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (star_targets '=') if (p->error_indicator) { @@ -34833,7 +34479,7 @@ _loop0_120_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -35356,14 +35002,8 @@ _loop0_129_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' lambda_param if (p->error_indicator) { @@ -35387,7 +35027,7 @@ _loop0_129_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -35692,14 +35332,8 @@ _loop0_135_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' dotted_name if (p->error_indicator) { @@ -35723,7 +35357,7 @@ _loop0_135_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -35850,14 +35484,8 @@ _loop0_138_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (expression ['as' star_target]) if (p->error_indicator) { @@ -35881,7 +35509,7 @@ _loop0_138_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -35967,14 +35595,8 @@ _loop0_140_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (expressions ['as' star_target]) if (p->error_indicator) { @@ -35998,7 +35620,7 @@ _loop0_140_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -36141,14 +35763,8 @@ _loop0_143_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // block if (p->error_indicator) { @@ -36163,7 +35779,7 @@ _loop0_143_rule(Parser *p) { _res = block_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -37794,14 +37410,8 @@ _loop0_172_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' bitwise_or) if (p->error_indicator) { @@ -37816,7 +37426,7 @@ _loop0_172_rule(Parser *p) { _res = _tmp_177_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); diff --git a/Tools/peg_generator/pegen/c_generator_rules.py b/Tools/peg_generator/pegen/c_generator_rules.py index 6dd598beb1b39c4..43685e6d211b87e 100644 --- a/Tools/peg_generator/pegen/c_generator_rules.py +++ b/Tools/peg_generator/pegen/c_generator_rules.py @@ -31,12 +31,6 @@ def emit(self, value: str) -> None: self.writer.print("p->level--;") self.writer.print(f"return {value};") - def check_memory(self, expr: str) -> None: - self.writer.print(f"if ({expr}) {{") - with self.writer.indent(): - self.no_memory() - self.writer.print("}") - def no_memory(self) -> None: self.writer.print("p->error_indicator = 1;") self.writer.print("PyErr_NoMemory();") @@ -55,15 +49,14 @@ def __init__(self, writer: CWriter, returns: _CReturnEmitter): self.error_returns = returns.with_cleanup(self._release) def initialize(self) -> None: - self._print("void **_children = PyMem_Malloc(sizeof(void *));") - self._returns.check_memory("!_children") - self._print("Py_ssize_t _children_capacity = 1;") + self._print("void **_children = NULL;") + self._print("Py_ssize_t _children_capacity = 0;") self._print("Py_ssize_t _n = 0;") def append(self, value: str) -> None: self._print("if (_n == _children_capacity) {") with self._indent(): - self._print("_children_capacity *= 2;") + self._print("_children_capacity = _children_capacity ? _children_capacity * 2 : 1;") self._print( "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));" ) From cc4ae2a791d939599bc94db09e4c97a9810c367e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 19:20:09 +0100 Subject: [PATCH 18/32] Allocate type comment storage only when needed --- Parser/pegen.c | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/Parser/pegen.c b/Parser/pegen.c index db1b4454ddc4d87..8c747062d8ac199 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -147,20 +147,10 @@ init_normalization(Parser *p) return 1; } -static int -growable_comment_array_init(growable_comment_array *arr, size_t initial_size) { - assert(initial_size > 0); - arr->items = PyMem_Malloc(initial_size * sizeof(*arr->items)); - arr->size = initial_size; - arr->num_items = 0; - - return arr->items != NULL; -} - static int growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) { if (arr->num_items >= arr->size) { - size_t new_size = arr->size * 2; + size_t new_size = arr->size ? arr->size * 2 : 10; void *new_items_array = PyMem_Realloc(arr->items, new_size * sizeof(*arr->items)); if (!new_items_array) { return 0; @@ -324,6 +314,7 @@ _PyPegen_fill_token(Parser *p) tag[len] = '\0'; // Ownership of tag passes to the growable array if (!growable_comment_array_add(&p->type_ignore_comments, new_token.end_loc.lineno, tag)) { + PyMem_Free(tag); PyErr_NoMemory(); goto error; } @@ -947,12 +938,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, PyMem_Free(p); return (Parser *) PyErr_NoMemory(); } - if (!growable_comment_array_init(&p->type_ignore_comments, 10)) { - PyMem_Free(p->tokens[0]); - PyMem_Free(p->tokens); - PyMem_Free(p); - return (Parser *) PyErr_NoMemory(); - } + p->type_ignore_comments = (growable_comment_array){0}; p->mark = 0; p->fill = 0; From fca5aac134055451f88657a0295579c38d5dfd27 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 19:22:19 +0100 Subject: [PATCH 19/32] Reuse decoded multiline text when counting token columns --- Python/Python-tokenize.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index f9b8a949ab5649e..c6f690e27724311 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -228,7 +228,8 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, it->byte_col_offset_diff += token_end - token_start - token_col_offset; } else { - *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(end_line_start, end_byte_offset); + *end_col_offset = _PyPegen_byte_offset_to_character_offset_line( + line, end_line_start - line_start, token_end - line_start); if (*end_col_offset < 0) { return -1; } From 98ced0aa3fcb60ae1d5df8434b42ed3c054ef0f4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 19:23:26 +0100 Subject: [PATCH 20/32] Skip empty decoded tokenizer chunks --- Parser/tokenizer/reader.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index e692ae8ac2a24fe..a49946eeed25ede 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -86,6 +86,9 @@ _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed, static int append_decoded(_PyTok_Reader *reader, const char *data, Py_ssize_t len) { + if (len == 0) { + return 0; + } if (reader->decoded_pos > 0) { Py_ssize_t remaining = reader->decoded_len - reader->decoded_pos; memmove(reader->decoded, reader->decoded + reader->decoded_pos, From b8eba43fa63c8cf1296b88a1c606c8e5bf915a8d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 19:44:09 +0100 Subject: [PATCH 21/32] Check parser array growth before allocating --- Lib/test/test_type_comments.py | 10 + Parser/parser.c | 910 +++++++++++++++--- Parser/pegen.c | 29 +- Parser/pegen.h | 2 + .../peg_generator/pegen/c_generator_rules.py | 13 +- 5 files changed, 822 insertions(+), 142 deletions(-) diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index d827ac271085bdd..832a1e283e0c279 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -338,6 +338,16 @@ def test_ignores(self): tree = self.classic_parse(ignores) self.assertEqual(tree.type_ignores, []) + def test_many_ignores(self): + comment_count = 25 + tags = [f"[tag_{index}]" for index in range(comment_count)] + source = "".join(f"pass # type: ignore{tag}\n" for tag in tags) + for tree in self.parse_all(source): + self.assertEqual( + [(item.lineno, item.tag) for item in tree.type_ignores], + list(enumerate(tags, start=1)), + ) + def test_longargs(self): for tree in self.parse_all(longargs, minver=8): for t in tree.body: diff --git a/Parser/parser.c b/Parser/parser.c index 9f1ee169d0ad886..6a18fb5a531ef67 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -27366,8 +27366,17 @@ _loop0_1_rule(Parser *p) { _res = newline_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27376,6 +27385,7 @@ _loop0_1_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -27427,8 +27437,17 @@ _loop1_2_rule(Parser *p) { _res = statement_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27437,6 +27456,7 @@ _loop1_2_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -27502,8 +27522,17 @@ _loop0_3_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27512,6 +27541,7 @@ _loop0_3_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28041,8 +28071,17 @@ _loop1_12_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28051,6 +28090,7 @@ _loop1_12_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28116,8 +28156,17 @@ _loop0_13_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28126,6 +28175,7 @@ _loop0_13_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28321,8 +28371,17 @@ _loop0_17_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28331,6 +28390,7 @@ _loop0_17_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28382,8 +28442,17 @@ _loop1_18_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28392,6 +28461,7 @@ _loop1_18_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28457,8 +28527,17 @@ _loop0_19_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28467,6 +28546,7 @@ _loop0_19_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28614,8 +28694,17 @@ _loop0_22_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28624,6 +28713,7 @@ _loop0_22_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28716,8 +28806,17 @@ _loop1_24_rule(Parser *p) { _res = _tmp_159_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28726,6 +28825,7 @@ _loop1_24_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28877,8 +28977,17 @@ _loop0_27_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28887,6 +28996,7 @@ _loop0_27_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28938,8 +29048,17 @@ _loop0_28_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28948,6 +29067,7 @@ _loop0_28_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28999,8 +29119,17 @@ _loop1_29_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29009,6 +29138,7 @@ _loop1_29_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29065,8 +29195,17 @@ _loop1_30_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29075,6 +29214,7 @@ _loop1_30_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29131,8 +29271,17 @@ _loop0_31_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29141,6 +29290,7 @@ _loop0_31_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29192,8 +29342,17 @@ _loop1_32_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29202,6 +29361,7 @@ _loop1_32_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29267,8 +29427,17 @@ _loop0_33_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29277,6 +29446,7 @@ _loop0_33_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29445,8 +29615,17 @@ _loop1_36_rule(Parser *p) { _res = except_block_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29455,6 +29634,7 @@ _loop1_36_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29511,8 +29691,17 @@ _loop1_37_rule(Parser *p) { _res = except_star_block_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29521,6 +29710,7 @@ _loop1_37_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29577,8 +29767,17 @@ _loop1_38_rule(Parser *p) { _res = case_block_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29587,6 +29786,7 @@ _loop1_38_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29652,8 +29852,17 @@ _loop0_39_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29662,6 +29871,7 @@ _loop0_39_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29972,8 +30182,17 @@ _loop0_44_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29982,6 +30201,7 @@ _loop0_44_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30083,8 +30303,17 @@ _loop0_46_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30093,6 +30322,7 @@ _loop0_46_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30251,8 +30481,17 @@ _loop0_49_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30261,6 +30500,7 @@ _loop0_49_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30362,8 +30602,17 @@ _loop0_51_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30372,6 +30621,7 @@ _loop0_51_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30473,8 +30723,17 @@ _loop0_53_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30483,6 +30742,7 @@ _loop0_53_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30575,8 +30835,17 @@ _loop1_55_rule(Parser *p) { _res = _tmp_16_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30585,6 +30854,7 @@ _loop1_55_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30641,8 +30911,17 @@ _loop1_56_rule(Parser *p) { _res = _tmp_160_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30651,6 +30930,7 @@ _loop1_56_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30716,8 +30996,17 @@ _loop0_57_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30726,6 +31015,7 @@ _loop0_57_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30818,8 +31108,17 @@ _loop1_59_rule(Parser *p) { _res = _tmp_161_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30828,6 +31127,7 @@ _loop1_59_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30884,8 +31184,17 @@ _loop1_60_rule(Parser *p) { _res = _tmp_162_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30894,6 +31203,7 @@ _loop1_60_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30950,8 +31260,17 @@ _loop1_61_rule(Parser *p) { _res = compare_op_bitwise_or_pair_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30960,6 +31279,7 @@ _loop1_61_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31068,8 +31388,17 @@ _loop0_63_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31078,6 +31407,7 @@ _loop0_63_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31501,8 +31831,17 @@ _loop0_70_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31511,6 +31850,7 @@ _loop0_70_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31562,8 +31902,17 @@ _loop0_71_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31572,6 +31921,7 @@ _loop0_71_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31623,8 +31973,17 @@ _loop1_72_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31633,6 +31992,7 @@ _loop1_72_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31689,8 +32049,17 @@ _loop1_73_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31699,6 +32068,7 @@ _loop1_73_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31755,8 +32125,17 @@ _loop0_74_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31765,6 +32144,7 @@ _loop0_74_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31816,8 +32196,17 @@ _loop1_75_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31826,6 +32215,7 @@ _loop1_75_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31882,8 +32272,17 @@ _loop0_76_rule(Parser *p) { _res = fstring_format_spec_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31892,6 +32291,7 @@ _loop0_76_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31943,8 +32343,17 @@ _loop0_77_rule(Parser *p) { _res = fstring_middle_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31953,6 +32362,7 @@ _loop0_77_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32004,8 +32414,17 @@ _loop0_78_rule(Parser *p) { _res = tstring_format_spec_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32014,6 +32433,7 @@ _loop0_78_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32065,8 +32485,17 @@ _loop0_79_rule(Parser *p) { _res = tstring_middle_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32075,6 +32504,7 @@ _loop0_79_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32126,8 +32556,17 @@ _loop1_80_rule(Parser *p) { _res = _tmp_154_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32136,6 +32575,7 @@ _loop1_80_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32192,8 +32632,17 @@ _loop1_81_rule(Parser *p) { _res = tstring_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32202,6 +32651,7 @@ _loop1_81_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32267,8 +32717,17 @@ _loop0_82_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32277,6 +32736,7 @@ _loop0_82_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32427,8 +32887,17 @@ _loop0_85_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32437,6 +32906,7 @@ _loop0_85_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32529,8 +32999,17 @@ _loop1_87_rule(Parser *p) { _res = for_if_clause_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32539,6 +33018,7 @@ _loop1_87_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32595,8 +33075,17 @@ _loop0_88_rule(Parser *p) { _res = _tmp_164_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32605,6 +33094,7 @@ _loop0_88_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32743,8 +33233,17 @@ _loop0_90_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32753,6 +33252,7 @@ _loop0_90_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32901,8 +33401,17 @@ _loop0_93_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32911,6 +33420,7 @@ _loop0_93_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33058,8 +33568,17 @@ _loop0_96_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33068,6 +33587,7 @@ _loop0_96_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33160,8 +33680,17 @@ _loop0_98_rule(Parser *p) { _res = _tmp_166_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33170,6 +33699,7 @@ _loop0_98_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33346,8 +33876,17 @@ _loop0_101_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33356,6 +33895,7 @@ _loop0_101_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33457,8 +33997,17 @@ _loop0_103_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33467,6 +34016,7 @@ _loop0_103_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33668,8 +34218,17 @@ _loop0_107_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33678,6 +34237,7 @@ _loop0_107_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33969,8 +34529,17 @@ _loop1_113_rule(Parser *p) { _res = _tmp_169_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33979,6 +34548,7 @@ _loop1_113_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -34418,8 +34988,17 @@ _loop0_119_rule(Parser *p) { _res = star_named_expressions_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -34428,6 +35007,7 @@ _loop0_119_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -34479,8 +35059,17 @@ _loop0_120_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -34489,6 +35078,7 @@ _loop0_120_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -35027,8 +35617,17 @@ _loop0_129_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35037,6 +35636,7 @@ _loop0_129_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -35357,8 +35957,17 @@ _loop0_135_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35367,6 +35976,7 @@ _loop0_135_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -35509,8 +36119,17 @@ _loop0_138_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35519,6 +36138,7 @@ _loop0_138_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -35620,8 +36240,17 @@ _loop0_140_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35630,6 +36259,7 @@ _loop0_140_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -35779,8 +36409,17 @@ _loop0_143_rule(Parser *p) { _res = block_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35789,6 +36428,7 @@ _loop0_143_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -37426,8 +38066,17 @@ _loop0_172_rule(Parser *p) { _res = _tmp_177_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -37436,6 +38085,7 @@ _loop0_172_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; diff --git a/Parser/pegen.c b/Parser/pegen.c index 8c747062d8ac199..979c707e093e81f 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -13,6 +13,7 @@ #define IDENTIFIER_CACHE_SIZE 2048 // Must be a power of two. #define IDENTIFIER_CACHE_MAX_PROBES 8 +#define TYPE_IGNORE_INITIAL_CAPACITY 10 struct _identifier_cache_entry { const char *key; // Borrowed from arena-owned token bytes. @@ -148,10 +149,18 @@ init_normalization(Parser *p) } static int -growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) { +growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) +{ if (arr->num_items >= arr->size) { - size_t new_size = arr->size ? arr->size * 2 : 10; - void *new_items_array = PyMem_Realloc(arr->items, new_size * sizeof(*arr->items)); + if (arr->size > (size_t)PY_SSIZE_T_MAX / + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*arr->items))) { + return 0; + } + size_t new_size = arr->size == 0 + ? TYPE_IGNORE_INITIAL_CAPACITY + : arr->size * PEGEN_ARRAY_GROWTH_FACTOR; + void *new_items_array = PyMem_Realloc( + arr->items, new_size * sizeof(*arr->items)); if (!new_items_array) { return 0; } @@ -166,8 +175,9 @@ growable_comment_array_add(growable_comment_array *arr, int lineno, char *commen } static void -growable_comment_array_deallocate(growable_comment_array *arr) { - for (unsigned i = 0; i < arr->num_items; i++) { +growable_comment_array_deallocate(growable_comment_array *arr) +{ + for (size_t i = 0; i < arr->num_items; i++) { PyMem_Free(arr->items[i].comment); } PyMem_Free(arr->items); @@ -267,12 +277,13 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to static int _resize_tokens_array(Parser *p) { - if (p->size > INT_MAX / 2 || - (size_t)p->size > PY_SSIZE_T_MAX / (2 * sizeof(*p->tokens))) { + if (p->size > INT_MAX / PEGEN_ARRAY_GROWTH_FACTOR || + (size_t)p->size > PY_SSIZE_T_MAX / + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*p->tokens))) { PyErr_NoMemory(); return -1; } - int newsize = p->size * 2; + int newsize = p->size * PEGEN_ARRAY_GROWTH_FACTOR; Token **new_tokens = PyMem_Realloc(p->tokens, (size_t)newsize * sizeof(Token *)); if (new_tokens == NULL) { PyErr_NoMemory(); @@ -985,7 +996,7 @@ _PyPegen_Parser_Free(Parser *p) Py_XDECREF(p->normalize); // Resizes allocate blocks starting at indices 1, 2, 4, and so on. PyMem_Free(p->tokens[0]); - for (int i = 1; i < p->size; i *= 2) { + for (int i = 1; i < p->size; i *= PEGEN_ARRAY_GROWTH_FACTOR) { PyMem_Free(p->tokens[i]); } PyMem_Free(p->tokens); diff --git a/Parser/pegen.h b/Parser/pegen.h index eb3f1d8a3041a7c..2c5332b4b7e8281 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -29,6 +29,8 @@ #define CURRENT_POS (-5) +#define PEGEN_ARRAY_GROWTH_FACTOR 2 + typedef struct _memo { int type; void *node; diff --git a/Tools/peg_generator/pegen/c_generator_rules.py b/Tools/peg_generator/pegen/c_generator_rules.py index 43685e6d211b87e..86abea9c9a8c3a8 100644 --- a/Tools/peg_generator/pegen/c_generator_rules.py +++ b/Tools/peg_generator/pegen/c_generator_rules.py @@ -56,12 +56,19 @@ def initialize(self) -> None: def append(self, value: str) -> None: self._print("if (_n == _children_capacity) {") with self._indent(): - self._print("_children_capacity = _children_capacity ? _children_capacity * 2 : 1;") - self._print( - "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));" + self._check_memory( + "(size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / " + "(PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))" ) + self._print("Py_ssize_t _new_capacity = _children_capacity == 0") + with self._indent(): + self._print("? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR;") + self._print("void **_new_children = PyMem_Realloc(") + with self._indent(): + self._print("_children, _new_capacity * sizeof(*_children));") self._check_memory("!_new_children") self._print("_children = _new_children;") + self._print("_children_capacity = _new_capacity;") self._print("}") self._print(f"_children[_n++] = {value};") From a29e2fd153d88d7a267f5b577cfaae6e86601885 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 19:46:14 +0100 Subject: [PATCH 22/32] Name tokenizer bounds and derive byte lengths from their data --- Modules/_testinternalcapi/tokenizer.c | 53 ++++++++++++------ Parser/tokenizer/decoder.c | 77 +++++++++++++++------------ Parser/tokenizer/reader.c | 6 ++- Parser/tokenizer/reader_internal.h | 6 ++- Parser/tokenizer/source.c | 9 ++-- 5 files changed, 95 insertions(+), 56 deletions(-) diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index fd131abe985937c..94b5742d3825f6e 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -38,6 +38,12 @@ static PyObject * test_tokenizer_source(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { + const char multiple_lines[] = "a\nb\n"; + const char first_line[] = "alpha\n"; + const char second_line[] = "\xce\xb2\n"; + const char expected[] = "alpha\n\xce\xb2\n"; + const char tail[] = "tail"; + const char terminated_line[] = "x\n"; _PyTok_SourceText source; _PyTok_SourceInit(&source); @@ -49,13 +55,15 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), _PyTok_SourceAppendLine(&source, "", 0) < 0, "accepted empty source line") < 0 || check_system_error( - _PyTok_SourceAppendLine(&source, "a\nb\n", 4) < 0, + _PyTok_SourceAppendLine( + &source, multiple_lines, sizeof(multiple_lines) - 1) < 0, "accepted multiple source lines") < 0 || check(_PyTok_SourceAppendLine( - &source, "alpha\n", 6) == 0, + &source, first_line, sizeof(first_line) - 1) == 0, "wrong first source offset") < 0 || check(_PyTok_SourceAppendLine( - &source, "\xce\xb2\n", 3) == 6, + &source, second_line, sizeof(second_line) - 1) == + (Py_ssize_t)sizeof(first_line) - 1, "wrong second source offset") < 0) { goto error; } @@ -68,16 +76,17 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), goto error; } - if (check(source.len == 9 && - memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0, + if (check(source.len == (Py_ssize_t)sizeof(expected) - 1 && + memcmp(source.bytes, expected, sizeof(expected)) == 0, "wrong source contents") < 0) { goto error; } _PyTok_SourceClear(&source); - if (_PyTok_SourceAppendLine(&source, "tail", 4) < 0 || + if (_PyTok_SourceAppendLine(&source, tail, sizeof(tail) - 1) < 0 || check_system_error( - _PyTok_SourceAppendLine(&source, "x\n", 2) < 0, + _PyTok_SourceAppendLine( + &source, terminated_line, sizeof(terminated_line) - 1) < 0, "appended after unterminated source line") < 0) { goto error; } @@ -98,24 +107,34 @@ static PyObject * test_tokenizer_source_discard(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { + enum { LINE_COUNT = 260 }; + const char first_line[] = "x\n"; + const char second_line[] = "y\n"; + const char tail[] = "tail"; + const char final_line[] = "z\n"; + const _PyTok_Off first_batch_len = LINE_COUNT * (sizeof(first_line) - 1); + const _PyTok_Off second_batch_len = LINE_COUNT * (sizeof(second_line) - 1); + const _PyTok_Off discarded_len = first_batch_len + second_batch_len; _PyTok_SourceText source; _PyTok_SourceInit(&source); - for (int i = 0; i < 260; i++) { - if (_PyTok_SourceAppendLine(&source, "x\n", 2) < 0) { + for (int i = 0; i < LINE_COUNT; i++) { + if (_PyTok_SourceAppendLine(&source, first_line, sizeof(first_line) - 1) < 0) { goto error; } } char *bytes = source.bytes; _PyTok_Off capacity = source.cap; _PyTok_SourceDiscard(&source); - if (check(source.base_offset == 520 && source.len == 0 && + if (check(source.base_offset == first_batch_len && source.len == 0 && source.nlines == 0 && source.bytes == bytes && source.cap == capacity && source.bytes[0] == '\0', "discard did not preserve source allocation") < 0) { goto error; } - for (int i = 0; i < 260; i++) { - if (check(_PyTok_SourceAppendLine(&source, "y\n", 2) == 520 + 2 * i, + for (int i = 0; i < LINE_COUNT; i++) { + if (check(_PyTok_SourceAppendLine( + &source, second_line, sizeof(second_line) - 1) == + first_batch_len + ((Py_ssize_t)sizeof(second_line) - 1) * i, "wrong source offset after discard") < 0) { goto error; } @@ -125,18 +144,22 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module), goto error; } _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "tail", 4) == 1040, + if (check(_PyTok_SourceAppendLine( + &source, tail, sizeof(tail) - 1) == discarded_len, "wrong source offset after repeated discard") < 0) { goto error; } _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "z\n", 2) == 1044, + if (check(_PyTok_SourceAppendLine( + &source, final_line, sizeof(final_line) - 1) == + discarded_len + (Py_ssize_t)sizeof(tail) - 1, "cannot append after discarding unterminated line") < 0) { goto error; } _PyTok_SourceDiscard(&source); source.base_offset = PY_SSIZE_T_MAX - 1; - if (check(_PyTok_SourceAppendLine(&source, "z\n", 2) < 0 && + if (check(_PyTok_SourceAppendLine( + &source, final_line, sizeof(final_line) - 1) < 0 && PyErr_ExceptionMatches(PyExc_MemoryError), "accepted overflowing logical source offset") < 0) { goto error; diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index d30d5e7a0227c5e..0af8cc31a3b947f 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -8,6 +8,14 @@ #include "helpers.h" #include "../lexer/state.h" +#define UNICODE_BOM 0xFEFF +#define UTF16_LE_BOM "\xff\xfe" +#define UTF16_BE_BOM "\xfe\xff" +#define UTF16_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF16_LE_BOM) - 1)) +#define UTF8_BOM "\xef\xbb\xbf" +#define UTF8_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF8_BOM) - 1)) +#define NORMALIZED_LINE_INITIAL_CAPACITY 256 + char * _PyTok_CopyBytes(const char *data, Py_ssize_t len) { @@ -60,9 +68,9 @@ chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk, return -1; } if (strip_bom && PyUnicode_GET_LENGTH(unicode) > 0 && - PyUnicode_ReadChar(unicode, 0) == 0xFEFF) { - utf8 += 3; - utf8_len -= 3; + PyUnicode_ReadChar(unicode, 0) == UNICODE_BOM) { + utf8 += UTF8_BOM_LENGTH; + utf8_len -= UTF8_BOM_LENGTH; } chunk_release_data(chunk); chunk->owner = unicode; @@ -72,8 +80,8 @@ chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk, return 0; } -// The caller must provide len + 2 bytes: normalization can append a final -// newline and always writes a NUL terminator. +// The caller provides len + add_final_newline + 1 bytes for an optional +// final newline and the NUL terminator. static void normalize_newlines_into(char *result, const char *data, Py_ssize_t len, int preserve_crlf, int add_final_newline, @@ -108,11 +116,11 @@ char * _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, Py_ssize_t *out_len) { - if (len > PY_SSIZE_T_MAX - 2) { + if (len < 0 || len == PY_SSIZE_T_MAX) { PyErr_NoMemory(); return NULL; } - char *result = PyMem_Malloc((size_t)len + 2); + char *result = PyMem_Malloc((size_t)len + 1); if (result == NULL) { PyErr_NoMemory(); return NULL; @@ -152,11 +160,12 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next) return 0; } } - for (; i + 6 < len; i++) { - if (memcmp(line + i, "coding", 6) != 0) { + const Py_ssize_t marker_len = sizeof("coding") - 1; + for (; i < len - marker_len; i++) { + if (memcmp(line + i, "coding", marker_len) != 0) { continue; } - const char *cursor = line + i + 6; + const char *cursor = line + i + marker_len; if (*cursor != ':' && *cursor != '=') { continue; } @@ -174,23 +183,26 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next) continue; } Py_ssize_t encoding_len = cursor - start; - char normalized[13]; - int n; - for (n = 0; n < 12 && n < encoding_len; n++) { + // Include the longest recognized prefix and its NUL terminator. + char normalized[sizeof("iso-latin-1-")]; + const Py_ssize_t prefix_len = Py_MIN( + encoding_len, (Py_ssize_t)sizeof(normalized) - 1); + Py_ssize_t n; + for (n = 0; n < prefix_len; n++) { normalized[n] = start[n] == '_' ? '-' : Py_TOLOWER(start[n]); } normalized[n] = '\0'; const char *canonical = start; if (strcmp(normalized, "utf-8") == 0 || - strncmp(normalized, "utf-8-", 6) == 0) { + strncmp(normalized, "utf-8-", sizeof("utf-8-") - 1) == 0) { canonical = "utf-8"; } else if (strcmp(normalized, "latin-1") == 0 || strcmp(normalized, "iso-8859-1") == 0 || strcmp(normalized, "iso-latin-1") == 0 || - strncmp(normalized, "latin-1-", 8) == 0 || - strncmp(normalized, "iso-8859-1-", 11) == 0 || - strncmp(normalized, "iso-latin-1-", 12) == 0) { + strncmp(normalized, "latin-1-", sizeof("latin-1-") - 1) == 0 || + strncmp(normalized, "iso-8859-1-", sizeof("iso-8859-1-") - 1) == 0 || + strncmp(normalized, "iso-latin-1-", sizeof("iso-latin-1-") - 1) == 0) { canonical = "iso-8859-1"; } if (canonical != start) { @@ -211,17 +223,15 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, const _PyTok_Chunk *second, int final, Py_ssize_t *bom_len) { - int bom = first->len >= 3 && - (unsigned char)first->data[0] == 0xEF && - (unsigned char)first->data[1] == 0xBB && - (unsigned char)first->data[2] == 0xBF; - *bom_len = bom ? 3 : 0; + int bom = first->len >= UTF8_BOM_LENGTH && + memcmp(first->data, UTF8_BOM, UTF8_BOM_LENGTH) == 0; + *bom_len = bom ? UTF8_BOM_LENGTH : 0; char *cookie = NULL; int scan_next = 0; int cookie_line = 1; - const char *first_data = first->data + (bom ? 3 : 0); - Py_ssize_t first_len = first->len - (bom ? 3 : 0); + const char *first_data = first->data + *bom_len; + Py_ssize_t first_len = first->len - *bom_len; if (find_cookie(first_data, first_len, &cookie, &scan_next) < 0) { return _PYTOK_ENCODING_ERROR; } @@ -246,8 +256,8 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, } if (bom && strcmp(cookie, "utf-8") != 0) { const _PyTok_Chunk *line = cookie_line == 2 ? second : first; - const char *line_data = line->data + (cookie_line == 1 ? 3 : 0); - Py_ssize_t line_len = line->len - (cookie_line == 1 ? 3 : 0); + const char *line_data = line->data + (cookie_line == 1 ? *bom_len : 0); + Py_ssize_t line_len = line->len - (cookie_line == 1 ? *bom_len : 0); int end_col = (int)Py_MIN(line_len, INT_MAX); if (end_col > 0 && (line_data[end_col - 1] == '\n' || line_data[end_col - 1] == '\r')) { @@ -325,14 +335,15 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, (line[line_len - 1] == '\r' || (line_len > 1 && line[line_len - 2] == '\r'))); if (normalize) { - if (line_len > PY_SSIZE_T_MAX - 2) { + if (line_len > PY_SSIZE_T_MAX - add_newline - 1) { PyErr_NoMemory(); tok->done = E_NOMEM; goto error; } // Reserve space for an optional final '\n' and the NUL terminator. - Py_ssize_t needed = line_len + 2; - if (_PyTok_ReserveBuffer(&normalized, &capacity, needed, 256) < 0) { + Py_ssize_t needed = line_len + add_newline + 1; + if (_PyTok_ReserveBuffer(&normalized, &capacity, needed, + NORMALIZED_LINE_INITIAL_CAPACITY) < 0) { tok->done = E_NOMEM; goto error; } @@ -461,11 +472,9 @@ _PyTok_DecodeChunk(struct tok_state *tok, _PyTok_Chunk *chunk, int final) return 0; } int strip_bom = reader->kind == _PYTOK_READER_READLINE && - chunk->len >= 2 && - (((unsigned char)chunk->data[0] == 0xFF && - (unsigned char)chunk->data[1] == 0xFE) || - ((unsigned char)chunk->data[0] == 0xFE && - (unsigned char)chunk->data[1] == 0xFF)); + chunk->len >= UTF16_BOM_LENGTH && + (memcmp(chunk->data, UTF16_LE_BOM, UTF16_BOM_LENGTH) == 0 || + memcmp(chunk->data, UTF16_BE_BOM, UTF16_BOM_LENGTH) == 0); PyObject *input; if (chunk->ownership == _PYTOK_CHUNK_PYOBJECT && PyBytes_Check(chunk->owner) && diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index a49946eeed25ede..99436e1604af777 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -9,6 +9,8 @@ #include "reader_internal.h" #include "../lexer/state.h" +#define READER_BUFFER_GROWTH_FACTOR 2 + #ifdef HAVE_UNISTD_H # include #endif @@ -67,11 +69,11 @@ _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed, // Grow geometrically to avoid reallocating for every longer line. Py_ssize_t cap = *capacity > 0 ? *capacity : initial_capacity; while (cap < needed) { - if (cap > PY_SSIZE_T_MAX / 2) { + if (cap > PY_SSIZE_T_MAX / READER_BUFFER_GROWTH_FACTOR) { cap = needed; break; } - cap *= 2; + cap *= READER_BUFFER_GROWTH_FACTOR; } char *resized = PyMem_Realloc(*buffer, cap); if (resized == NULL) { diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 26e5957b68af70c..f74063de8d653a4 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -3,6 +3,8 @@ #include "Python.h" +#define _PYTOK_ENCODING_COOKIE_MAX_LINES 2 + typedef enum { _PYTOK_READER_PREPARED, _PYTOK_READER_FILE, @@ -46,11 +48,11 @@ typedef struct _PyTok_Reader { char *file_buffer; Py_ssize_t file_buffer_cap; - _PyTok_Chunk prefetched_lines[2]; + _PyTok_Chunk prefetched_lines[_PYTOK_ENCODING_COOKIE_MAX_LINES]; char *decoded; Py_ssize_t decoded_pos; - Py_ssize_t decoded_scan; + Py_ssize_t decoded_scan; // First byte not yet scanned for a newline. Py_ssize_t decoded_len; Py_ssize_t decoded_cap; _PyTok_ReaderKind kind; diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index d08e09c597d7c68..ebb7b05507c56ec 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -1,7 +1,10 @@ #include "Python.h" +#include "pycore_pymem.h" // PYMEM_DEADBYTE #include "source.h" +#define SOURCE_BUFFER_GROWTH_FACTOR 2 + void _PyTok_SourceInit(_PyTok_SourceText *source) { @@ -35,11 +38,11 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) } Py_ssize_t cap = source->cap > 0 ? source->cap : BUFSIZ; while (cap < needed) { - if (cap > PY_SSIZE_T_MAX / 2) { + if (cap > PY_SSIZE_T_MAX / SOURCE_BUFFER_GROWTH_FACTOR) { cap = needed; break; } - cap *= 2; + cap *= SOURCE_BUFFER_GROWTH_FACTOR; } char *bytes; #ifdef Py_DEBUG @@ -57,7 +60,7 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) } #ifdef Py_DEBUG if (source->bytes != NULL) { - memset(source->bytes, 0xDD, source->cap); + memset(source->bytes, PYMEM_DEADBYTE, source->cap); PyMem_Free(source->bytes); } #endif From 3587fb161a6eb3a24a57419347872f96f669d6f4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 19:46:14 +0100 Subject: [PATCH 23/32] Set the tokenizer exception when file input size overflows --- Parser/tokenizer/reader.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 99436e1604af777..6a0b7615f8df5de 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -192,9 +192,12 @@ read_file_line(struct tok_state *tok, _PyTok_Chunk *chunk) _PyTok_Reader *reader = tok->reader; Py_ssize_t len = 0; for (;;) { - if (len > PY_SSIZE_T_MAX - BUFSIZ || - _PyTok_ReserveBuffer(&reader->file_buffer, &reader->file_buffer_cap, - len + BUFSIZ, BUFSIZ) < 0) { + if (len > PY_SSIZE_T_MAX - BUFSIZ) { + PyErr_NoMemory(); + return _PYTOK_READ_ERROR; + } + if (_PyTok_ReserveBuffer(&reader->file_buffer, &reader->file_buffer_cap, + len + BUFSIZ, BUFSIZ) < 0) { return _PYTOK_READ_ERROR; } int available = (int)Py_MIN(reader->file_buffer_cap - len, INT_MAX); From 39f6d4c6a3874290525d70f0aa7a0319d2a1f0ab Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 20:48:09 +0100 Subject: [PATCH 24/32] gh-153569: Send exact line endings in the REPL test --- Lib/test/test_repl.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index e1ce17b4dd4256c..516d9486c731b30 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -219,10 +219,12 @@ def test_long_non_ascii_input(self): for newline in ("\n", "\r\n"): with self.subTest(newline=newline): p = spawn_repl(encoding="utf-8") - p.stdin.write( + # Send the requested line ending without Windows text-mode + # translation turning CRLF into CRCRLF. + p.stdin.buffer.write(( f"value = {value!r}{newline}" f"print(len(value), value[:2]){newline}" - ) + ).encode("utf-8")) output = kill_python(p) self.assertEqual(p.returncode, 0) self.assertIn(">>> 10000 é漢\n>>> ", output) From ba726350a15d85bce3e55befd64849acd509e558 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 23:45:30 +0100 Subject: [PATCH 25/32] Share parser repetition buffer growth --- Parser/parser.c | 845 ++---------------- Parser/pegen.c | 16 + Parser/pegen.h | 2 + .../peg_generator/pegen/c_generator_rules.py | 12 +- 4 files changed, 85 insertions(+), 790 deletions(-) diff --git a/Parser/parser.c b/Parser/parser.c index 6a18fb5a531ef67..441d23a0658147c 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -27366,17 +27366,7 @@ _loop0_1_rule(Parser *p) { _res = newline_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27385,7 +27375,6 @@ _loop0_1_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -27437,17 +27426,7 @@ _loop1_2_rule(Parser *p) { _res = statement_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27456,7 +27435,6 @@ _loop1_2_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -27522,17 +27500,7 @@ _loop0_3_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27541,7 +27509,6 @@ _loop0_3_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28071,17 +28038,7 @@ _loop1_12_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28090,7 +28047,6 @@ _loop1_12_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28156,17 +28112,7 @@ _loop0_13_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28175,7 +28121,6 @@ _loop0_13_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28371,17 +28316,7 @@ _loop0_17_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28390,7 +28325,6 @@ _loop0_17_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28442,17 +28376,7 @@ _loop1_18_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28461,7 +28385,6 @@ _loop1_18_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28527,17 +28450,7 @@ _loop0_19_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28546,7 +28459,6 @@ _loop0_19_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28694,17 +28606,7 @@ _loop0_22_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28713,7 +28615,6 @@ _loop0_22_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28806,17 +28707,7 @@ _loop1_24_rule(Parser *p) { _res = _tmp_159_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28825,7 +28716,6 @@ _loop1_24_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28977,17 +28867,7 @@ _loop0_27_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28996,7 +28876,6 @@ _loop0_27_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29048,17 +28927,7 @@ _loop0_28_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29067,7 +28936,6 @@ _loop0_28_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29119,17 +28987,7 @@ _loop1_29_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29138,7 +28996,6 @@ _loop1_29_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29195,17 +29052,7 @@ _loop1_30_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29214,7 +29061,6 @@ _loop1_30_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29271,17 +29117,7 @@ _loop0_31_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29290,7 +29126,6 @@ _loop0_31_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29342,17 +29177,7 @@ _loop1_32_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29361,7 +29186,6 @@ _loop1_32_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29427,17 +29251,7 @@ _loop0_33_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29446,7 +29260,6 @@ _loop0_33_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29615,17 +29428,7 @@ _loop1_36_rule(Parser *p) { _res = except_block_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29634,7 +29437,6 @@ _loop1_36_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29691,17 +29493,7 @@ _loop1_37_rule(Parser *p) { _res = except_star_block_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29710,7 +29502,6 @@ _loop1_37_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29767,17 +29558,7 @@ _loop1_38_rule(Parser *p) { _res = case_block_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29786,7 +29567,6 @@ _loop1_38_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29852,17 +29632,7 @@ _loop0_39_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29871,7 +29641,6 @@ _loop0_39_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30182,17 +29951,7 @@ _loop0_44_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30201,7 +29960,6 @@ _loop0_44_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30303,17 +30061,7 @@ _loop0_46_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30322,7 +30070,6 @@ _loop0_46_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30481,17 +30228,7 @@ _loop0_49_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30500,7 +30237,6 @@ _loop0_49_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30602,17 +30338,7 @@ _loop0_51_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30621,7 +30347,6 @@ _loop0_51_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30723,17 +30448,7 @@ _loop0_53_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30742,7 +30457,6 @@ _loop0_53_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30835,17 +30549,7 @@ _loop1_55_rule(Parser *p) { _res = _tmp_16_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30854,7 +30558,6 @@ _loop1_55_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30911,17 +30614,7 @@ _loop1_56_rule(Parser *p) { _res = _tmp_160_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30930,7 +30623,6 @@ _loop1_56_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30996,17 +30688,7 @@ _loop0_57_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31015,7 +30697,6 @@ _loop0_57_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31108,17 +30789,7 @@ _loop1_59_rule(Parser *p) { _res = _tmp_161_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31127,7 +30798,6 @@ _loop1_59_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31184,17 +30854,7 @@ _loop1_60_rule(Parser *p) { _res = _tmp_162_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31203,7 +30863,6 @@ _loop1_60_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31260,17 +30919,7 @@ _loop1_61_rule(Parser *p) { _res = compare_op_bitwise_or_pair_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31279,7 +30928,6 @@ _loop1_61_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31388,17 +31036,7 @@ _loop0_63_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31407,7 +31045,6 @@ _loop0_63_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31831,17 +31468,7 @@ _loop0_70_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31850,7 +31477,6 @@ _loop0_70_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31902,17 +31528,7 @@ _loop0_71_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31921,7 +31537,6 @@ _loop0_71_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31973,17 +31588,7 @@ _loop1_72_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31992,7 +31597,6 @@ _loop1_72_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32049,17 +31653,7 @@ _loop1_73_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32068,7 +31662,6 @@ _loop1_73_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32125,17 +31718,7 @@ _loop0_74_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32144,7 +31727,6 @@ _loop0_74_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32196,17 +31778,7 @@ _loop1_75_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32215,7 +31787,6 @@ _loop1_75_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32272,17 +31843,7 @@ _loop0_76_rule(Parser *p) { _res = fstring_format_spec_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32291,7 +31852,6 @@ _loop0_76_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32343,17 +31903,7 @@ _loop0_77_rule(Parser *p) { _res = fstring_middle_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32362,7 +31912,6 @@ _loop0_77_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32414,17 +31963,7 @@ _loop0_78_rule(Parser *p) { _res = tstring_format_spec_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32433,7 +31972,6 @@ _loop0_78_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32485,17 +32023,7 @@ _loop0_79_rule(Parser *p) { _res = tstring_middle_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32504,7 +32032,6 @@ _loop0_79_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32556,17 +32083,7 @@ _loop1_80_rule(Parser *p) { _res = _tmp_154_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32575,7 +32092,6 @@ _loop1_80_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32632,17 +32148,7 @@ _loop1_81_rule(Parser *p) { _res = tstring_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32651,7 +32157,6 @@ _loop1_81_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32717,17 +32222,7 @@ _loop0_82_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32736,7 +32231,6 @@ _loop0_82_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32887,17 +32381,7 @@ _loop0_85_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32906,7 +32390,6 @@ _loop0_85_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32999,17 +32482,7 @@ _loop1_87_rule(Parser *p) { _res = for_if_clause_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33018,7 +32491,6 @@ _loop1_87_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33075,17 +32547,7 @@ _loop0_88_rule(Parser *p) { _res = _tmp_164_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33094,7 +32556,6 @@ _loop0_88_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33233,17 +32694,7 @@ _loop0_90_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33252,7 +32703,6 @@ _loop0_90_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33401,17 +32851,7 @@ _loop0_93_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33420,7 +32860,6 @@ _loop0_93_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33568,17 +33007,7 @@ _loop0_96_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33587,7 +33016,6 @@ _loop0_96_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33680,17 +33108,7 @@ _loop0_98_rule(Parser *p) { _res = _tmp_166_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33699,7 +33117,6 @@ _loop0_98_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33876,17 +33293,7 @@ _loop0_101_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33895,7 +33302,6 @@ _loop0_101_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33997,17 +33403,7 @@ _loop0_103_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -34016,7 +33412,6 @@ _loop0_103_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -34218,17 +33613,7 @@ _loop0_107_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -34237,7 +33622,6 @@ _loop0_107_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -34529,17 +33913,7 @@ _loop1_113_rule(Parser *p) { _res = _tmp_169_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -34548,7 +33922,6 @@ _loop1_113_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -34988,17 +34361,7 @@ _loop0_119_rule(Parser *p) { _res = star_named_expressions_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35007,7 +34370,6 @@ _loop0_119_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -35059,17 +34421,7 @@ _loop0_120_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35078,7 +34430,6 @@ _loop0_120_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -35617,17 +34968,7 @@ _loop0_129_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35636,7 +34977,6 @@ _loop0_129_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -35957,17 +35297,7 @@ _loop0_135_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35976,7 +35306,6 @@ _loop0_135_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -36119,17 +35448,7 @@ _loop0_138_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -36138,7 +35457,6 @@ _loop0_138_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -36240,17 +35558,7 @@ _loop0_140_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -36259,7 +35567,6 @@ _loop0_140_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -36409,17 +35716,7 @@ _loop0_143_rule(Parser *p) { _res = block_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -36428,7 +35725,6 @@ _loop0_143_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -38066,17 +37362,7 @@ _loop0_172_rule(Parser *p) { _res = _tmp_177_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -38085,7 +37371,6 @@ _loop0_172_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; diff --git a/Parser/pegen.c b/Parser/pegen.c index 979c707e093e81f..9210f4c9f3f3eaf 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -148,6 +148,22 @@ init_normalization(Parser *p) return 1; } +void ** +_PyPegen_grow_loop_buffer(void **buffer, Py_ssize_t *capacity) +{ + if ((size_t)*capacity > (size_t)PY_SSIZE_T_MAX / + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*buffer))) { + return NULL; + } + Py_ssize_t new_capacity = *capacity == 0 + ? 1 : *capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **new_buffer = PyMem_Realloc(buffer, new_capacity * sizeof(*buffer)); + if (new_buffer != NULL) { + *capacity = new_capacity; + } + return new_buffer; +} + static int growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) { diff --git a/Parser/pegen.h b/Parser/pegen.h index 2c5332b4b7e8281..1a0a20290dc94d3 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -192,6 +192,8 @@ expr_ty _PyPegen_soft_keyword_token(Parser *p); expr_ty _PyPegen_fstring_middle_token(Parser* p); Token *_PyPegen_get_last_nonnwhitespace_token(Parser *); int _PyPegen_fill_token(Parser *p); +// Return NULL on failure, leaving the buffer and capacity unchanged. +void **_PyPegen_grow_loop_buffer(void **buffer, Py_ssize_t *capacity); expr_ty _PyPegen_name_token(Parser *p); expr_ty _PyPegen_number_token(Parser *p); void *_PyPegen_string_token(Parser *p); diff --git a/Tools/peg_generator/pegen/c_generator_rules.py b/Tools/peg_generator/pegen/c_generator_rules.py index 86abea9c9a8c3a8..a5a07d02dbc029e 100644 --- a/Tools/peg_generator/pegen/c_generator_rules.py +++ b/Tools/peg_generator/pegen/c_generator_rules.py @@ -56,19 +56,11 @@ def initialize(self) -> None: def append(self, value: str) -> None: self._print("if (_n == _children_capacity) {") with self._indent(): - self._check_memory( - "(size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / " - "(PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))" + self._print( + "void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity);" ) - self._print("Py_ssize_t _new_capacity = _children_capacity == 0") - with self._indent(): - self._print("? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR;") - self._print("void **_new_children = PyMem_Realloc(") - with self._indent(): - self._print("_children, _new_capacity * sizeof(*_children));") self._check_memory("!_new_children") self._print("_children = _new_children;") - self._print("_children_capacity = _new_capacity;") self._print("}") self._print(f"_children[_n++] = {value};") From a768c1d23773c8db2396e0d5ab6f866e06d50bbd Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 23:45:30 +0100 Subject: [PATCH 26/32] Fold Unicode input coverage into the existing REPL test --- Lib/test/test_repl.py | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 516d9486c731b30..3525dee84493acc 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -187,16 +187,19 @@ def read_until(marker, start=0): def test_lexer_buffer_realloc_with_null_start(self): # gh-144759: NULL pointer arithmetic when the lexer buffer grows # while parsing long input. - long_value = "a" * 2000 + long_value = "é漢" * 2000 user_input = dedent(f"""\ x = f'{{{long_value!r}}}' print(x) """) - p = spawn_repl() - p.stdin.write(user_input) - output = kill_python(p) - self.assertEqual(p.returncode, 0) - self.assertIn(long_value, output) + for newline in ("\n", "\r\n"): + with self.subTest(newline=newline): + p = spawn_repl(encoding="utf-8") + # Bypass Windows text-mode translation of CRLF to CRCRLF. + p.stdin.buffer.write(user_input.replace("\n", newline).encode("utf-8")) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + self.assertIn(long_value, output) @cpython_only def test_multiline_fstring_source_reallocation(self): @@ -214,21 +217,6 @@ def test_multiline_fstring_source_reallocation(self): self.assertEqual(p.returncode, 0) self.assertIn(">>> 3\n>>> ", output) - def test_long_non_ascii_input(self): - value = "é漢" * 5000 - for newline in ("\n", "\r\n"): - with self.subTest(newline=newline): - p = spawn_repl(encoding="utf-8") - # Send the requested line ending without Windows text-mode - # translation turning CRLF into CRCRLF. - p.stdin.buffer.write(( - f"value = {value!r}{newline}" - f"print(len(value), value[:2]){newline}" - ).encode("utf-8")) - output = kill_python(p) - self.assertEqual(p.returncode, 0) - self.assertIn(">>> 10000 é漢\n>>> ", output) - def test_close_stdin(self): user_input = dedent(''' import os From 7264275f87183b03f6b8b462f1c25be998746fde Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 25 Sep 2026 00:00:03 +0100 Subject: [PATCH 27/32] Keep parser array growth policy private --- Parser/pegen.c | 1 + Parser/pegen.h | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Parser/pegen.c b/Parser/pegen.c index 9210f4c9f3f3eaf..575f84a96f8b74a 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -13,6 +13,7 @@ #define IDENTIFIER_CACHE_SIZE 2048 // Must be a power of two. #define IDENTIFIER_CACHE_MAX_PROBES 8 +#define PEGEN_ARRAY_GROWTH_FACTOR 2 #define TYPE_IGNORE_INITIAL_CAPACITY 10 struct _identifier_cache_entry { diff --git a/Parser/pegen.h b/Parser/pegen.h index 1a0a20290dc94d3..825cc9197bfabf3 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -29,8 +29,6 @@ #define CURRENT_POS (-5) -#define PEGEN_ARRAY_GROWTH_FACTOR 2 - typedef struct _memo { int type; void *node; From e55d8b0adfcffb01474bcae21bb891767fd74510 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 25 Sep 2026 00:00:03 +0100 Subject: [PATCH 28/32] Drop obsolete bitmap stress from source discard test --- Modules/_testinternalcapi/tokenizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index 94b5742d3825f6e..0c2bdee2e803b22 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -107,7 +107,7 @@ static PyObject * test_tokenizer_source_discard(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { - enum { LINE_COUNT = 260 }; + enum { LINE_COUNT = 2 }; const char first_line[] = "x\n"; const char second_line[] = "y\n"; const char tail[] = "tail"; From 714d6dd6e226823279e91096df6466238d739ae7 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 25 Sep 2026 03:10:53 +0100 Subject: [PATCH 29/32] Preserve input failures while scanning string literals --- Lib/test/test_tokenize.py | 6 ++++++ .../2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst | 1 + Parser/lexer/string.c | 3 +++ 3 files changed, 10 insertions(+) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index de3a7fc3503432c..9fa294aa3618845 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2423,6 +2423,12 @@ def test_multiline_readline_chunk_with_unterminated_tail(self): ) self.assertEqual(readline.call_count, 2) + def test_readline_memory_error_in_string(self): + readline = mock.Mock(side_effect=['"""first\n', MemoryError]) + iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) + with self.assertRaises(MemoryError): + next(iterator) + def test_readline_callback_is_not_read_ahead(self): readline = mock.Mock(side_effect=["x\n", "y\n", ""]) iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst index a1b233bc758f951..7bd6b023fb32da5 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst @@ -1,3 +1,4 @@ Correct missing-brace error messages for multiline f-strings and t-strings. Improve tokenization of UTF-8 input split across many readline calls. Preserve source order when reading files with stateful encodings. +Preserve input errors when scanning string literals. diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index 3c38e1ab7c32cec..b481ee42f7b33b2 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -335,6 +335,9 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) break; } if (c == EOF || (quote_size == 1 && c == '\n')) { + if (tok_failed(tok)) { + return MAKE_TOKEN(ERRORTOKEN); + } int end_lineno = tok->lineno; _PyTok_Loc location = tok->start_loc; const char *line = _PyLexer_BufferPointer(tok, tok->start) - location.byte_col; From 657d92c9d02fc53e6fbd702934cc09f3d1533245 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 28 Sep 2026 23:23:33 +0100 Subject: [PATCH 30/32] fixup! Preserve input failures while scanning string literals --- Lib/test/test_source_encoding.py | 24 + Parser/parser.c | 520 +++++++----------- Parser/pegen.c | 18 +- Parser/pegen.h | 4 +- Parser/tokenizer/decoder.c | 81 ++- Parser/tokenizer/reader.c | 14 +- Python/Python-tokenize.c | 59 +- .../peg_generator/pegen/c_generator_rules.py | 10 +- 8 files changed, 292 insertions(+), 438 deletions(-) diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index 63f3126c99925cc..3c29002025575e1 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -29,6 +29,30 @@ def test_compilestring(self): exec(c, d) self.assertEqual(d['u'], '\xf3') + def test_compilestring_line_endings(self): + for newline in ('\n', '\r', '\r\n', '\r\r\n'): + expected = 'é\n\ntext' if newline == '\r\r\n' else 'é\ntext' + for suffix in ('', newline): + for mode in ('exec', 'eval'): + source = f"'''é{newline}text'''{suffix}" + if mode == 'exec': + source = 'value = ' + source + for encoding in (None, 'utf-8', 'latin-1'): + with self.subTest(newline=newline, suffix=suffix, + mode=mode, encoding=encoding): + input = source + if encoding is not None: + input = (f'# coding: {encoding}{newline}' + + source).encode(encoding) + code = compile(input, '', mode) + if mode == 'exec': + namespace = {} + exec(code, namespace) + value = namespace['value'] + else: + value = eval(code) + self.assertEqual(value, expected) + def test_issue2301(self): try: compile(b"# coding: cp932\nprint '\x94\x4e'", "dummy", "exec") diff --git a/Parser/parser.c b/Parser/parser.c index 441d23a0658147c..979907f4873b9dd 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -27366,15 +27366,13 @@ _loop0_1_rule(Parser *p) { _res = newline_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -27385,9 +27383,9 @@ _loop0_1_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -27426,15 +27424,13 @@ _loop1_2_rule(Parser *p) { _res = statement_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -27450,9 +27446,9 @@ _loop1_2_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -27500,15 +27496,13 @@ _loop0_3_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -27519,9 +27513,9 @@ _loop0_3_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28038,15 +28032,13 @@ _loop1_12_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28062,9 +28054,9 @@ _loop1_12_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28112,15 +28104,13 @@ _loop0_13_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28131,9 +28121,9 @@ _loop0_13_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28316,15 +28306,13 @@ _loop0_17_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28335,9 +28323,9 @@ _loop0_17_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28376,15 +28364,13 @@ _loop1_18_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28400,9 +28386,9 @@ _loop1_18_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28450,15 +28436,13 @@ _loop0_19_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28469,9 +28453,9 @@ _loop0_19_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28606,15 +28590,13 @@ _loop0_22_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28625,9 +28607,9 @@ _loop0_22_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28707,15 +28689,13 @@ _loop1_24_rule(Parser *p) { _res = _tmp_159_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28731,9 +28711,9 @@ _loop1_24_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28867,15 +28847,13 @@ _loop0_27_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28886,9 +28864,9 @@ _loop0_27_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28927,15 +28905,13 @@ _loop0_28_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28946,9 +28922,9 @@ _loop0_28_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28987,15 +28963,13 @@ _loop1_29_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29011,9 +28985,9 @@ _loop1_29_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29052,15 +29026,13 @@ _loop1_30_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29076,9 +29048,9 @@ _loop1_30_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29117,15 +29089,13 @@ _loop0_31_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29136,9 +29106,9 @@ _loop0_31_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29177,15 +29147,13 @@ _loop1_32_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29201,9 +29169,9 @@ _loop1_32_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29251,15 +29219,13 @@ _loop0_33_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29270,9 +29236,9 @@ _loop0_33_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29428,15 +29394,13 @@ _loop1_36_rule(Parser *p) { _res = except_block_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29452,9 +29416,9 @@ _loop1_36_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29493,15 +29457,13 @@ _loop1_37_rule(Parser *p) { _res = except_star_block_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29517,9 +29479,9 @@ _loop1_37_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29558,15 +29520,13 @@ _loop1_38_rule(Parser *p) { _res = case_block_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29582,9 +29542,9 @@ _loop1_38_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29632,15 +29592,13 @@ _loop0_39_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29651,9 +29609,9 @@ _loop0_39_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29951,15 +29909,13 @@ _loop0_44_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29970,9 +29926,9 @@ _loop0_44_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30061,15 +30017,13 @@ _loop0_46_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30080,9 +30034,9 @@ _loop0_46_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30228,15 +30182,13 @@ _loop0_49_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30247,9 +30199,9 @@ _loop0_49_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30338,15 +30290,13 @@ _loop0_51_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30357,9 +30307,9 @@ _loop0_51_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30448,15 +30398,13 @@ _loop0_53_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30467,9 +30415,9 @@ _loop0_53_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30549,15 +30497,13 @@ _loop1_55_rule(Parser *p) { _res = _tmp_16_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30573,9 +30519,9 @@ _loop1_55_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30614,15 +30560,13 @@ _loop1_56_rule(Parser *p) { _res = _tmp_160_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30638,9 +30582,9 @@ _loop1_56_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30688,15 +30632,13 @@ _loop0_57_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30707,9 +30649,9 @@ _loop0_57_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30789,15 +30731,13 @@ _loop1_59_rule(Parser *p) { _res = _tmp_161_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30813,9 +30753,9 @@ _loop1_59_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30854,15 +30794,13 @@ _loop1_60_rule(Parser *p) { _res = _tmp_162_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30878,9 +30816,9 @@ _loop1_60_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30919,15 +30857,13 @@ _loop1_61_rule(Parser *p) { _res = compare_op_bitwise_or_pair_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30943,9 +30879,9 @@ _loop1_61_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31036,15 +30972,13 @@ _loop0_63_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31055,9 +30989,9 @@ _loop0_63_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31468,15 +31402,13 @@ _loop0_70_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31487,9 +31419,9 @@ _loop0_70_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31528,15 +31460,13 @@ _loop0_71_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31547,9 +31477,9 @@ _loop0_71_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31588,15 +31518,13 @@ _loop1_72_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31612,9 +31540,9 @@ _loop1_72_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31653,15 +31581,13 @@ _loop1_73_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31677,9 +31603,9 @@ _loop1_73_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31718,15 +31644,13 @@ _loop0_74_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31737,9 +31661,9 @@ _loop0_74_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31778,15 +31702,13 @@ _loop1_75_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31802,9 +31724,9 @@ _loop1_75_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31843,15 +31765,13 @@ _loop0_76_rule(Parser *p) { _res = fstring_format_spec_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31862,9 +31782,9 @@ _loop0_76_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31903,15 +31823,13 @@ _loop0_77_rule(Parser *p) { _res = fstring_middle_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31922,9 +31840,9 @@ _loop0_77_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31963,15 +31881,13 @@ _loop0_78_rule(Parser *p) { _res = tstring_format_spec_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31982,9 +31898,9 @@ _loop0_78_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32023,15 +31939,13 @@ _loop0_79_rule(Parser *p) { _res = tstring_middle_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32042,9 +31956,9 @@ _loop0_79_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32083,15 +31997,13 @@ _loop1_80_rule(Parser *p) { _res = _tmp_154_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32107,9 +32019,9 @@ _loop1_80_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32148,15 +32060,13 @@ _loop1_81_rule(Parser *p) { _res = tstring_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32172,9 +32082,9 @@ _loop1_81_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32222,15 +32132,13 @@ _loop0_82_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32241,9 +32149,9 @@ _loop0_82_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32381,15 +32289,13 @@ _loop0_85_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32400,9 +32306,9 @@ _loop0_85_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32482,15 +32388,13 @@ _loop1_87_rule(Parser *p) { _res = for_if_clause_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32506,9 +32410,9 @@ _loop1_87_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32547,15 +32451,13 @@ _loop0_88_rule(Parser *p) { _res = _tmp_164_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32566,9 +32468,9 @@ _loop0_88_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32694,15 +32596,13 @@ _loop0_90_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32713,9 +32613,9 @@ _loop0_90_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32851,15 +32751,13 @@ _loop0_93_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32870,9 +32768,9 @@ _loop0_93_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33007,15 +32905,13 @@ _loop0_96_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33026,9 +32922,9 @@ _loop0_96_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33108,15 +33004,13 @@ _loop0_98_rule(Parser *p) { _res = _tmp_166_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33127,9 +33021,9 @@ _loop0_98_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33293,15 +33187,13 @@ _loop0_101_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33312,9 +33204,9 @@ _loop0_101_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33403,15 +33295,13 @@ _loop0_103_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33422,9 +33312,9 @@ _loop0_103_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33613,15 +33503,13 @@ _loop0_107_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33632,9 +33520,9 @@ _loop0_107_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33913,15 +33801,13 @@ _loop1_113_rule(Parser *p) { _res = _tmp_169_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33937,9 +33823,9 @@ _loop1_113_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -34361,15 +34247,13 @@ _loop0_119_rule(Parser *p) { _res = star_named_expressions_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -34380,9 +34264,9 @@ _loop0_119_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -34421,15 +34305,13 @@ _loop0_120_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -34440,9 +34322,9 @@ _loop0_120_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -34968,15 +34850,13 @@ _loop0_129_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -34987,9 +34867,9 @@ _loop0_129_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -35297,15 +35177,13 @@ _loop0_135_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -35316,9 +35194,9 @@ _loop0_135_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -35448,15 +35326,13 @@ _loop0_138_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -35467,9 +35343,9 @@ _loop0_138_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -35558,15 +35434,13 @@ _loop0_140_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -35577,9 +35451,9 @@ _loop0_140_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -35716,15 +35590,13 @@ _loop0_143_rule(Parser *p) { _res = block_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -35735,9 +35607,9 @@ _loop0_143_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -37362,15 +37234,13 @@ _loop0_172_rule(Parser *p) { _res = _tmp_177_var; if (_n == _children_capacity) { - void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -37381,9 +37251,9 @@ _loop0_172_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } diff --git a/Parser/pegen.c b/Parser/pegen.c index 575f84a96f8b74a..4ca42a407bcde80 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -149,20 +149,22 @@ init_normalization(Parser *p) return 1; } -void ** -_PyPegen_grow_loop_buffer(void **buffer, Py_ssize_t *capacity) +int +_PyPegen_grow_loop_buffer(void ***buffer, Py_ssize_t *capacity) { if ((size_t)*capacity > (size_t)PY_SSIZE_T_MAX / - (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*buffer))) { - return NULL; + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(**buffer))) { + return -1; } Py_ssize_t new_capacity = *capacity == 0 ? 1 : *capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **new_buffer = PyMem_Realloc(buffer, new_capacity * sizeof(*buffer)); - if (new_buffer != NULL) { - *capacity = new_capacity; + void **new_buffer = PyMem_Realloc(*buffer, new_capacity * sizeof(**buffer)); + if (new_buffer == NULL) { + return -1; } - return new_buffer; + *buffer = new_buffer; + *capacity = new_capacity; + return 0; } static int diff --git a/Parser/pegen.h b/Parser/pegen.h index 825cc9197bfabf3..7454289ef6df03d 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -190,8 +190,8 @@ expr_ty _PyPegen_soft_keyword_token(Parser *p); expr_ty _PyPegen_fstring_middle_token(Parser* p); Token *_PyPegen_get_last_nonnwhitespace_token(Parser *); int _PyPegen_fill_token(Parser *p); -// Return NULL on failure, leaving the buffer and capacity unchanged. -void **_PyPegen_grow_loop_buffer(void **buffer, Py_ssize_t *capacity); +// Return -1 on failure, leaving the buffer and capacity unchanged. +int _PyPegen_grow_loop_buffer(void ***buffer, Py_ssize_t *capacity); expr_ty _PyPegen_name_token(Parser *p); expr_ty _PyPegen_number_token(Parser *p); void *_PyPegen_string_token(Parser *p); diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 0af8cc31a3b947f..220fed65ad4e450 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -80,38 +80,6 @@ chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk, return 0; } -// The caller provides len + add_final_newline + 1 bytes for an optional -// final newline and the NUL terminator. -static void -normalize_newlines_into(char *result, const char *data, Py_ssize_t len, - int preserve_crlf, int add_final_newline, - Py_ssize_t *out_len) -{ - Py_ssize_t write = 0; - if (preserve_crlf || memchr(data, '\r', len) == NULL) { - // No translation needed: copy verbatim. - memcpy(result, data, len); - write = len; - } - else { - for (Py_ssize_t read = 0; read < len; read++) { - char c = data[read]; - if (c == '\r') { - if (read + 1 < len && data[read + 1] == '\n') { - read++; - } - c = '\n'; - } - result[write++] = c; - } - } - if (add_final_newline && write > 0 && result[write - 1] != '\n') { - result[write++] = '\n'; - } - result[write] = '\0'; - *out_len = write; -} - char * _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, Py_ssize_t *out_len) @@ -125,7 +93,19 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, PyErr_NoMemory(); return NULL; } - normalize_newlines_into(result, data, len, 0, 0, out_len); + Py_ssize_t write = 0; + for (Py_ssize_t read = 0; read < len; read++) { + char c = data[read]; + if (c == '\r') { + if (read + 1 < len && data[read + 1] == '\n') { + read++; + } + c = '\n'; + } + result[write++] = c; + } + result[write] = '\0'; + *out_len = write; return result; } @@ -245,13 +225,10 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, return _PYTOK_ENCODING_NEED_SECOND_LINE; } - if (bom) { - if (_PyTok_SetEncoding(tok, "utf-8") < 0) { - PyMem_Free(cookie); + if (cookie == NULL) { + if (bom && _PyTok_SetEncoding(tok, "utf-8") < 0) { return _PYTOK_ENCODING_ERROR; } - } - if (cookie == NULL) { return _PYTOK_ENCODING_DONE; } if (bom && strcmp(cookie, "utf-8") != 0) { @@ -330,26 +307,32 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, Py_ssize_t line_len = raw_line_len; // raw_line_length stops at the first CR or LF, so any CR is in // the line terminator. - int normalize = add_newline || - (!preserve_crlf && - (line[line_len - 1] == '\r' || - (line_len > 1 && line[line_len - 2] == '\r'))); - if (normalize) { - if (line_len > PY_SSIZE_T_MAX - add_newline - 1) { + Py_ssize_t terminator_len = 0; + if (!preserve_crlf) { + if (line[line_len - 1] == '\r') { + terminator_len = 1; + } + else if (line_len > 1 && line[line_len - 2] == '\r') { + terminator_len = 2; + } + } + if (terminator_len || add_newline) { + Py_ssize_t payload_len = line_len - terminator_len; + if (payload_len > PY_SSIZE_T_MAX - 2) { PyErr_NoMemory(); tok->done = E_NOMEM; goto error; } - // Reserve space for an optional final '\n' and the NUL terminator. - Py_ssize_t needed = line_len + add_newline + 1; + Py_ssize_t needed = payload_len + 2; if (_PyTok_ReserveBuffer(&normalized, &capacity, needed, NORMALIZED_LINE_INITIAL_CAPACITY) < 0) { tok->done = E_NOMEM; goto error; } - normalize_newlines_into(normalized, line, line_len, - preserve_crlf, add_newline, - &line_len); + memcpy(normalized, line, payload_len); + normalized[payload_len] = '\n'; + line_len = payload_len + 1; + normalized[line_len] = '\0'; line = normalized; } _PyTok_Off appended = _PyTok_SourceAppendLine( diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 6a0b7615f8df5de..36f910e3199756d 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -314,8 +314,7 @@ finalize_decoding(struct tok_state *tok) return -1; } } - if (reader->decoded_pos < reader->decoded_len && - append_implicit_newline(reader) < 0) { + if (append_implicit_newline(reader) < 0) { tok->done = E_NOMEM; return -1; } @@ -542,7 +541,6 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) decoded.data, decoded.len, &chunk->len); _PyTok_ChunkClear(&decoded); if (chunk->data == NULL) { - PyErr_NoMemory(); tok->done = E_NOMEM; return _PYTOK_READ_ERROR; } @@ -840,14 +838,12 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) } _Py_DECLARE_STR(anon_string, ""); tok->filename = Py_NewRef(filename != NULL ? filename : &_Py_STR(anon_string)); - if (initialize_file(tok) < 0) { - fclose(fp); - _PyTokenizer_Free(tok); - return NULL; + char *encoding = NULL; + if (initialize_file(tok) == 0) { + encoding = tok->encoding; + tok->encoding = NULL; } fclose(fp); - char *encoding = tok->encoding; - tok->encoding = NULL; _PyTokenizer_Free(tok); return encoding; } diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index c6f690e27724311..26c699a4420f73d 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -1,7 +1,6 @@ #include "Python.h" #include "errcode.h" #include "internal/pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION -#include "internal/pycore_tuple.h" // _PyTuple_FromPair #include "../Parser/tokenizer/tokenizer.h" #include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset() @@ -125,9 +124,7 @@ _tokenizer_error(tokenizeriterobject *it) msg = "unknown tokenization error"; } - PyObject* errstr = NULL; PyObject* error_line = NULL; - PyObject* tmp = NULL; PyObject* value = NULL; Py_ssize_t input_size; @@ -145,17 +142,9 @@ _tokenizer_error(tokenizeriterobject *it) if (offset == -1) { goto exit; } - tmp = Py_BuildValue("(OnnOOO)", info.filename, info.location.lineno, offset, error_line, Py_None, Py_None); - if (!tmp) { - goto exit; - } - - errstr = PyUnicode_FromString(msg); - if (!errstr) { - goto exit; - } - - value = _PyTuple_FromPair(errstr, tmp); + value = Py_BuildValue("(s(OinOOO))", msg, info.filename, + info.location.lineno, offset, error_line, + Py_None, Py_None); if (!value) { goto exit; } @@ -163,9 +152,7 @@ _tokenizer_error(tokenizeriterobject *it) PyErr_SetObject(errtype, value); exit: - Py_XDECREF(errstr); Py_XDECREF(error_line); - Py_XDECREF(tmp); Py_XDECREF(value); } @@ -174,36 +161,33 @@ _get_current_line(tokenizeriterobject *it, int current_lineno, const char *line_start, Py_ssize_t size, int *line_changed) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); - PyObject *line; if (current_lineno != it->last_lineno) { // Line has changed since last token, so we fetch the new line and cache it // in the iter object. Py_XDECREF(it->last_line); - line = PyUnicode_DecodeUTF8(line_start, size, "replace"); - it->last_line = line; + it->last_line = PyUnicode_DecodeUTF8(line_start, size, "replace"); it->byte_col_offset_diff = 0; } else { - line = it->last_line; *line_changed = 0; } - return line; + return it->last_line; } static int _get_col_offsets(tokenizeriterobject *it, const struct token *token, - const char *token_start, const char *line_start, - const char *end_line_start, PyObject *line, int line_changed, + const _PyToken_View *view, PyObject *line, int line_changed, Py_ssize_t *col_offset, Py_ssize_t *end_col_offset) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); + const char *token_start = view->text; const char *token_end = token_start == NULL - ? NULL : token_start + token->span.end - token->span.start; + ? NULL : token_start + view->length; Py_ssize_t lineno = token->start_loc.lineno; Py_ssize_t end_lineno = token->end_loc.lineno; Py_ssize_t byte_offset = -1; - if (token_start != NULL && token_start >= line_start) { - byte_offset = token_start - line_start; + if (token_start != NULL && token_start >= view->line) { + byte_offset = token_start - view->line; if (line_changed) { *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset); if (*col_offset < 0) { @@ -216,8 +200,8 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, } } - if (token_end != NULL && token_end >= end_line_start) { - Py_ssize_t end_byte_offset = token_end - end_line_start; + if (token_end != NULL && token_end >= view->end_line) { + Py_ssize_t end_byte_offset = token_end - view->end_line; if (lineno == end_lineno) { // Avoid rescanning the prefix of a very long line. Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); @@ -229,7 +213,7 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, } else { *end_col_offset = _PyPegen_byte_offset_to_character_offset_line( - line, end_line_start - line_start, token_end - line_start); + line, view->end_line - view->line, token_end - view->line); if (*end_col_offset < 0) { return -1; } @@ -248,6 +232,11 @@ tokenizeriter_next(PyObject *op) Py_BEGIN_CRITICAL_SECTION(it); + if (it->done) { + PyErr_SetString(PyExc_StopIteration, "EOF"); + goto unlock; + } + struct token token; _PyToken_Init(&token); @@ -260,10 +249,6 @@ tokenizeriter_next(PyObject *op) } goto exit; } - if (it->done) { - PyErr_SetString(PyExc_StopIteration, "EOF"); - goto exit; - } _PyToken_View view; _PyToken_GetView(it->tok, &token, &view); const char *token_start = view.text; @@ -279,10 +264,7 @@ tokenizeriter_next(PyObject *op) goto exit; } - int is_trailing_token = 0; - if (type == ENDMARKER || (type == DEDENT && view.at_eof)) { - is_trailing_token = 1; - } + int is_trailing_token = type == ENDMARKER || (type == DEDENT && view.at_eof); PyObject* line = NULL; int line_changed = 1; @@ -306,7 +288,7 @@ tokenizeriter_next(PyObject *op) Py_ssize_t end_lineno = token.end_loc.lineno; Py_ssize_t col_offset = -1; Py_ssize_t end_col_offset = -1; - if (_get_col_offsets(it, &token, token_start, view.line, view.end_line, line, + if (_get_col_offsets(it, &token, &view, line, line_changed, &col_offset, &end_col_offset) < 0) { Py_DECREF(str); goto exit; @@ -353,6 +335,7 @@ tokenizeriter_next(PyObject *op) it->done = 1; } +unlock: Py_END_CRITICAL_SECTION(); return result; } diff --git a/Tools/peg_generator/pegen/c_generator_rules.py b/Tools/peg_generator/pegen/c_generator_rules.py index a5a07d02dbc029e..f6cf2d954e4e7bb 100644 --- a/Tools/peg_generator/pegen/c_generator_rules.py +++ b/Tools/peg_generator/pegen/c_generator_rules.py @@ -45,7 +45,6 @@ class _LoopBuffer: def __init__(self, writer: CWriter, returns: _CReturnEmitter): self._print = writer.print self._indent = writer.indent - self._returns = returns self.error_returns = returns.with_cleanup(self._release) def initialize(self) -> None: @@ -56,11 +55,9 @@ def initialize(self) -> None: def append(self, value: str) -> None: self._print("if (_n == _children_capacity) {") with self._indent(): - self._print( - "void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity);" + self._check_memory( + "_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0" ) - self._check_memory("!_new_children") - self._print("_children = _new_children;") self._print("}") self._print(f"_children[_n++] = {value};") @@ -79,8 +76,7 @@ def finish(self, *, require_one: bool) -> str: def _check_memory(self, expr: str) -> None: self._print(f"if ({expr}) {{") with self._indent(): - self._print(self._release) - self._returns.no_memory() + self.error_returns.no_memory() self._print("}") From 39436f719af7d212db637edc71bc647d463bc2df Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 29 Sep 2026 01:00:46 +0100 Subject: [PATCH 31/32] fixup! fixup! Preserve input failures while scanning string literals --- Python/Python-tokenize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 26c699a4420f73d..e91141bd1b9e610 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -335,7 +335,7 @@ tokenizeriter_next(PyObject *op) it->done = 1; } -unlock: +unlock:; Py_END_CRITICAL_SECTION(); return result; } From 452e6fa7a38c1cc9cd55e3016c41f7eac6c4b321 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 29 Sep 2026 01:51:20 +0100 Subject: [PATCH 32/32] gh-153569: Remove assertion-only lexer bookkeeping --- Parser/lexer/string.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index b481ee42f7b33b2..99e45578b6ca16e 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -83,18 +83,17 @@ finish_ftstring_expr(struct tok_state *tok, ftstring_state *state, PyObject *res; if (comments != NULL && comments->count > 0) { Py_ssize_t stripped_size = expr_len; - _PyTok_Off previous_end = state->expr_span.start; Py_ssize_t comment_count = 0; for (Py_ssize_t i = 0; i < comments->count; i++) { _PyTok_Span comment = comments->spans[i]; assert(_PyTok_SpanIsValid(comment)); - assert(comment.start >= previous_end); + assert(comment.start >= (i == 0 ? state->expr_span.start + : comments->spans[i - 1].end)); if (comment.start >= state->expr_span.end) { break; } assert(comment.end <= state->expr_span.end); stripped_size -= comment.end - comment.start; - previous_end = comment.end; comment_count++; } char *stripped = PyMem_Malloc((size_t)stripped_size);