Skip to content

Commit 58e2f7f

Browse files
committed
Move byte_offset_to_character_offset to the parser API
1 parent 1a5a4cf commit 58e2f7f

File tree

4 files changed

+27
-27
lines changed

4 files changed

+27
-27
lines changed

Include/internal/pycore_traceback.h

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -87,27 +87,6 @@ PyAPI_FUNC(PyObject*) _PyTraceBack_FromFrame(
8787
PyObject *tb_next,
8888
PyFrameObject *frame);
8989

90-
static inline Py_ssize_t
91-
_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset)
92-
{
93-
const char *str = PyUnicode_AsUTF8(line);
94-
if (!str) {
95-
return 0;
96-
}
97-
Py_ssize_t len = strlen(str);
98-
if (col_offset > len + 1) {
99-
col_offset = len + 1;
100-
}
101-
assert(col_offset >= 0);
102-
PyObject *text = PyUnicode_DecodeUTF8(str, col_offset, "replace");
103-
if (!text) {
104-
return 0;
105-
}
106-
Py_ssize_t size = PyUnicode_GET_LENGTH(text);
107-
Py_DECREF(text);
108-
return size;
109-
}
110-
11190
#ifdef __cplusplus
11291
}
11392
#endif

Parser/pegen.c

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
#include <Python.h>
22
#include "pycore_ast.h" // _PyAST_Validate(),
3-
#include "pycore_traceback.h" // _byte_offset_to_character_offset(),
43
#include <errcode.h>
54
#include "tokenizer.h"
65

@@ -398,6 +397,27 @@ get_error_line(Parser *p, Py_ssize_t lineno)
398397
return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace");
399398
}
400399

400+
Py_ssize_t
401+
_PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset)
402+
{
403+
const char *str = PyUnicode_AsUTF8(line);
404+
if (!str) {
405+
return 0;
406+
}
407+
Py_ssize_t len = strlen(str);
408+
if (col_offset > len + 1) {
409+
col_offset = len + 1;
410+
}
411+
assert(col_offset >= 0);
412+
PyObject *text = PyUnicode_DecodeUTF8(str, col_offset, "replace");
413+
if (!text) {
414+
return 0;
415+
}
416+
Py_ssize_t size = PyUnicode_GET_LENGTH(text);
417+
Py_DECREF(text);
418+
return size;
419+
}
420+
401421
void *
402422
_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
403423
Py_ssize_t lineno, Py_ssize_t col_offset,
@@ -478,9 +498,9 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
478498
Py_ssize_t end_col_number = end_col_offset;
479499

480500
if (p->tok->encoding != NULL) {
481-
col_number = _byte_offset_to_character_offset(error_line, col_offset);
501+
col_number = _PyPegen_byte_offset_to_character_offset(error_line, col_offset);
482502
end_col_number = end_col_number > 0 ?
483-
_byte_offset_to_character_offset(error_line, end_col_offset) :
503+
_PyPegen_byte_offset_to_character_offset(error_line, end_col_offset) :
484504
end_col_number;
485505
}
486506
tmp = Py_BuildValue("(OiiNii)", p->tok->filename, lineno, col_number, error_line, end_lineno, end_col_number);

Parser/pegen.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ expr_ty _PyPegen_name_token(Parser *p);
139139
expr_ty _PyPegen_number_token(Parser *p);
140140
void *_PyPegen_string_token(Parser *p);
141141
const char *_PyPegen_get_expr_name(expr_ty);
142+
Py_ssize_t _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset);
142143
void *_PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...);
143144
void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
144145
Py_ssize_t lineno, Py_ssize_t col_offset,

Python/traceback.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
#include "pycore_interp.h" // PyInterpreterState.gc
88
#include "frameobject.h" // PyFrame_GetBack()
99
#include "pycore_frame.h" // _PyFrame_GetCode()
10-
#include "pycore_traceback.h" // _byte_offset_to_character_offset()
10+
#include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset()
1111
#include "structmember.h" // PyMemberDef
1212
#include "osdefs.h" // SEP
1313
#ifdef HAVE_FCNTL_H
@@ -556,8 +556,8 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen
556556
}
557557
// Convert the utf-8 byte offset to the actual character offset so we
558558
// print the right number of carets.
559-
Py_ssize_t start_offset = _byte_offset_to_character_offset(source_line, start_col_byte_offset);
560-
Py_ssize_t end_offset = _byte_offset_to_character_offset(source_line, end_col_byte_offset);
559+
Py_ssize_t start_offset = _PyPegen_byte_offset_to_character_offset(source_line, start_col_byte_offset);
560+
Py_ssize_t end_offset = _PyPegen_byte_offset_to_character_offset(source_line, end_col_byte_offset);
561561

562562
char offset = truncation;
563563
while (++offset <= start_offset) {

0 commit comments

Comments
 (0)