Skip to content

Commit ee94aa0

Browse files
bpo-45952: Get the C analyzer tool working again. (gh-29882)
There wasn't much that needed to be done. Mostly it was just a few new files that got added. https://bugs.python.org/issue45952
1 parent f27bef3 commit ee94aa0

File tree

6 files changed

+111
-99
lines changed

6 files changed

+111
-99
lines changed

Tools/c-analyzer/TODO

Lines changed: 14 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,11 @@
1+
# For up-to-date results, run:
2+
# ./python Tools/c-analyzer/c-analyzer.py check --format summary
3+
# or
4+
# ./python Tools/c-analyzer/c-analyzer.py analyze
5+
6+
7+
#######################################
8+
# non-PyObject (61)
19

210
# allocator (16)
311
Objects/obmalloc.c:_PyMem static PyMemAllocatorEx _PyMem
@@ -32,12 +40,7 @@ Objects/dictobject.c:empty_keys_struct static PyDictKe
3240
Python/fileutils.c:_Py_open_cloexec_works int _Py_open_cloexec_works
3341

3442

35-
# freelists
36-
Objects/dictobject.c:keys_free_list static PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST]
37-
Objects/dictobject.c:numfreekeys static int numfreekeys
38-
39-
40-
# other non-object (43)
43+
# other non-object (40)
4144
Modules/_tracemalloc.c:allocators static struct { PyMemAllocatorEx mem; PyMemAllocatorEx raw; PyMemAllocatorEx obj; } allocators
4245
Modules/_tracemalloc.c:tables_lock static PyThread_type_lock tables_lock
4346
Modules/_tracemalloc.c:tracemalloc_filenames static _Py_hashtable_t *tracemalloc_filenames
@@ -81,30 +84,7 @@ Python/pylifecycle.c:fatal_error():reentrant static int reen
8184

8285

8386
#######################################
84-
# PyObject (960)
85-
86-
# freelists (10 + 10)
87-
Modules/_collectionsmodule.c:freeblocks static block *freeblocks[MAXFREEBLOCKS]
88-
Modules/_collectionsmodule.c:numfreeblocks static Py_ssize_t numfreeblocks
89-
Objects/dictobject.c:free_list static PyDictObject *free_list[PyDict_MAXFREELIST]
90-
Objects/dictobject.c:numfree static int numfree
91-
Objects/exceptions.c:memerrors_freelist static PyBaseExceptionObject *memerrors_freelist
92-
Objects/exceptions.c:memerrors_numfree static int memerrors_numfree
93-
Objects/floatobject.c:free_list static PyFloatObject *free_list
94-
Objects/floatobject.c:numfree static int numfree
95-
Objects/frameobject.c:free_list static PyFrameObject *free_list
96-
Objects/frameobject.c:numfree static int numfree
97-
Objects/genobject.c:ag_asend_freelist static PyAsyncGenASend *ag_asend_freelist[_PyAsyncGen_MAXFREELIST]
98-
Objects/genobject.c:ag_asend_freelist_free static int ag_asend_freelist_free
99-
Objects/genobject.c:ag_value_freelist static _PyAsyncGenWrappedValue *ag_value_freelist[_PyAsyncGen_MAXFREELIST]
100-
Objects/genobject.c:ag_value_freelist_free static int ag_value_freelist_free
101-
Objects/listobject.c:free_list static PyListObject *free_list[PyList_MAXFREELIST]
102-
Objects/listobject.c:numfree static int numfree
103-
Objects/tupleobject.c:free_list static PyTupleObject *free_list[PyTuple_MAXSAVESIZE]
104-
Objects/tupleobject.c:numfree static int numfree[PyTuple_MAXSAVESIZE]
105-
Python/context.c:ctx_freelist static PyContext *ctx_freelist
106-
Python/context.c:ctx_freelist_len static int ctx_freelist_len
107-
87+
# PyObject (919)
10888

10989
# singletons (7)
11090
Objects/boolobject.c:_Py_FalseStruct static struct _longobject _Py_FalseStruct
@@ -116,16 +96,8 @@ Objects/object.c:_Py_NotImplementedStruct PyObject _Py_No
11696
Objects/sliceobject.c:_Py_EllipsisObject PyObject _Py_EllipsisObject
11797

11898

119-
# module vars (9)
120-
Modules/_functoolsmodule.c:kwd_mark static PyObject *kwd_mark
121-
Modules/_localemodule.c:Error static PyObject *Error
122-
Modules/_threadmodule.c:ThreadError static PyObject *ThreadError
99+
# module vars (1)
123100
Modules/_tracemalloc.c:unknown_filename static PyObject *unknown_filename
124-
Modules/signalmodule.c:DefaultHandler static PyObject *DefaultHandler
125-
Modules/signalmodule.c:IgnoreHandler static PyObject *IgnoreHandler
126-
Modules/signalmodule.c:IntHandler static PyObject *IntHandler
127-
Modules/signalmodule.c:ItimerError static PyObject *ItimerError
128-
Objects/exceptions.c:errnomap static PyObject *errnomap
129101

130102

131103
# other (non-cache) (5)
@@ -136,26 +108,15 @@ Modules/signalmodule.c:Handlers static volatile
136108
Objects/setobject.c:_dummy_struct static PyObject _dummy_struct
137109

138110

139-
# caches (5)
140-
Modules/posixmodule.c:posix_putenv_garbage static PyObject *posix_putenv_garbage
141-
Objects/sliceobject.c:slice_cache static PySliceObject *slice_cache
142-
Objects/typeobject.c:method_cache static struct method_cache_entry method_cache[1 << MCACHE_SIZE_EXP]
143-
Objects/unicodeobject.c:interned static PyObject *interned
111+
# caches (1)
144112
Python/import.c:extensions static PyObject *extensions
145113

146114

147-
# cached constants - non-str (15)
115+
# cached constants - non-str (6)
148116
Modules/_io/_iomodule.c:_PyIO_empty_bytes PyObject *_PyIO_empty_bytes
149117
Modules/_io/bufferedio.c:_PyIO_trap_eintr():eintr_int static PyObject *eintr_int
150-
Modules/posixmodule.c:billion static PyObject *billion
151-
Modules/posixmodule.c:wait_helper():struct_rusage static PyObject *struct_rusage
152-
Objects/bytesobject.c:characters static PyBytesObject *characters[UCHAR_MAX + 1]
153-
Objects/bytesobject.c:nullstring static PyBytesObject *nullstring
154-
Objects/codeobject.c:PyCode_NewEmpty():nulltuple static PyObject *nulltuple
155-
Objects/dictobject.c:empty_values static PyObject *empty_values[1]
118+
Objects/dictobject.c:empty_values_struct static PyDictValues
156119
Objects/listobject.c:indexerr static PyObject *indexerr
157-
Objects/longobject.c:small_ints static PyLongObject small_ints[NSMALLNEGINTS + NSMALLPOSINTS]
158-
Objects/setobject.c:emptyfrozenset static PyObject *emptyfrozenset
159120
Python/context.c:_token_missing static PyObject *_token_missing
160121
Python/hamt.c:_empty_hamt static PyHamtObject *_empty_hamt
161122

@@ -662,15 +623,6 @@ Modules/itertoolsmodule.c:takewhile_type static PyTypeOb
662623
Modules/itertoolsmodule.c:tee_type static PyTypeObject tee_type
663624
Modules/itertoolsmodule.c:teedataobject_type static PyTypeObject teedataobject_type
664625
Modules/itertoolsmodule.c:ziplongest_type static PyTypeObject ziplongest_type
665-
Modules/posixmodule.c:DirEntryType static PyTypeObject DirEntryType
666-
Modules/posixmodule.c:ScandirIteratorType static PyTypeObject ScandirIteratorType
667-
Modules/posixmodule.c:SchedParamType static PyTypeObject* SchedParamType
668-
Modules/posixmodule.c:StatResultType static PyTypeObject* StatResultType
669-
Modules/posixmodule.c:StatVFSResultType static PyTypeObject* StatVFSResultType
670-
Modules/posixmodule.c:TerminalSizeType static PyTypeObject* TerminalSizeType
671-
Modules/posixmodule.c:TimesResultType static PyTypeObject* TimesResultType
672-
Modules/posixmodule.c:UnameResultType static PyTypeObject* UnameResultType
673-
Modules/posixmodule.c:WaitidResultType static PyTypeObject* WaitidResultType
674626
Modules/signalmodule.c:SiginfoType static PyTypeObject SiginfoType
675627
Modules/timemodule.c:StructTimeType static PyTypeObject StructTimeType
676628
Modules/xxsubtype.c:spamdict_type static PyTypeObject spamdict_type

Tools/c-analyzer/c_parser/__init__.py

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,37 @@
1+
from c_common.fsutil import match_glob as _match_glob
12
from .parser import parse as _parse
23
from .preprocessor import get_preprocessor as _get_preprocessor
34

45

56
def parse_file(filename, *,
67
match_kind=None,
78
get_file_preprocessor=None,
9+
file_maxsizes=None,
810
):
911
if get_file_preprocessor is None:
1012
get_file_preprocessor = _get_preprocessor()
11-
yield from _parse_file(filename, match_kind, get_file_preprocessor)
13+
yield from _parse_file(
14+
filename, match_kind, get_file_preprocessor, file_maxsizes)
1215

1316

1417
def parse_files(filenames, *,
1518
match_kind=None,
1619
get_file_preprocessor=None,
20+
file_maxsizes=None,
1721
):
1822
if get_file_preprocessor is None:
1923
get_file_preprocessor = _get_preprocessor()
2024
for filename in filenames:
21-
yield from _parse_file(filename, match_kind, get_file_preprocessor)
25+
yield from _parse_file(
26+
filename, match_kind, get_file_preprocessor, file_maxsizes)
2227

2328

24-
def _parse_file(filename, match_kind, get_file_preprocessor):
29+
def _parse_file(filename, match_kind, get_file_preprocessor, maxsizes):
30+
srckwargs = {}
31+
maxsize = _resolve_max_size(filename, maxsizes)
32+
if maxsize:
33+
srckwargs['maxtext'], srckwargs['maxlines'] = maxsize
34+
2535
# Preprocess the file.
2636
preprocess = get_file_preprocessor(filename)
2737
preprocessed = preprocess()
@@ -30,14 +40,30 @@ def _parse_file(filename, match_kind, get_file_preprocessor):
3040

3141
# Parse the lines.
3242
srclines = ((l.file, l.data) for l in preprocessed if l.kind == 'source')
33-
for item in _parse(srclines):
43+
for item in _parse(srclines, **srckwargs):
3444
if match_kind is not None and not match_kind(item.kind):
3545
continue
3646
if not item.filename:
3747
raise NotImplementedError(repr(item))
3848
yield item
3949

4050

51+
def _resolve_max_size(filename, maxsizes):
52+
for pattern, maxsize in (maxsizes.items() if maxsizes else ()):
53+
if _match_glob(filename, pattern):
54+
break
55+
else:
56+
return None
57+
if not maxsize:
58+
return None, None
59+
maxtext, maxlines = maxsize
60+
if maxtext is not None:
61+
maxtext = int(maxtext)
62+
if maxlines is not None:
63+
maxlines = int(maxlines)
64+
return maxtext, maxlines
65+
66+
4167
def parse_signature(text):
4268
raise NotImplementedError
4369

Tools/c-analyzer/c_parser/parser/__init__.py

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -120,12 +120,12 @@
120120
from ._info import SourceInfo
121121

122122

123-
def parse(srclines):
123+
def parse(srclines, **srckwargs):
124124
if isinstance(srclines, str): # a filename
125125
raise NotImplementedError
126126

127127
anon_name = anonymous_names()
128-
for result in _parse(srclines, anon_name):
128+
for result in _parse(srclines, anon_name, **srckwargs):
129129
yield ParsedItem.from_raw(result)
130130

131131

@@ -152,17 +152,19 @@ def anon_name(prefix='anon-'):
152152
_logger = logging.getLogger(__name__)
153153

154154

155-
def _parse(srclines, anon_name):
155+
def _parse(srclines, anon_name, **srckwargs):
156156
from ._global import parse_globals
157157

158-
source = _iter_source(srclines)
159-
#source = _iter_source(srclines, showtext=True)
158+
source = _iter_source(srclines, **srckwargs)
160159
for result in parse_globals(source, anon_name):
161160
# XXX Handle blocks here instead of in parse_globals().
162161
yield result
163162

164163

165-
def _iter_source(lines, *, maxtext=20_000, maxlines=700, showtext=False):
164+
# We use defaults that cover most files. Files with bigger declarations
165+
# are covered elsewhere (MAX_SIZES in cpython/_parser.py).
166+
167+
def _iter_source(lines, *, maxtext=10_000, maxlines=200, showtext=False):
166168
maxtext = maxtext if maxtext and maxtext > 0 else None
167169
maxlines = maxlines if maxlines and maxlines > 0 else None
168170
filestack = []

Tools/c-analyzer/c_parser/preprocessor/__main__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ def add_common_cli(parser, *, get_preprocessor=_get_preprocessor):
4343
def process_args(args, *, argv):
4444
ns = vars(args)
4545

46-
process_fail_arg(args, argv)
46+
process_fail_arg(args, argv=argv)
4747
ignore_exc = ns.pop('ignore_exc')
4848
# We later pass ignore_exc to _get_preprocessor().
4949

Tools/c-analyzer/cpython/__main__.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ def cmd_parse(filenames=None, **kwargs):
111111
c_parser.cmd_parse(
112112
filenames,
113113
relroot=REPO_ROOT,
114+
file_maxsizes=_parser.MAX_SIZES,
114115
**kwargs
115116
)
116117

@@ -127,6 +128,7 @@ def cmd_check(filenames=None, **kwargs):
127128
relroot=REPO_ROOT,
128129
_analyze=_analyzer.analyze,
129130
_CHECKS=CHECKS,
131+
file_maxsizes=_parser.MAX_SIZES,
130132
**kwargs
131133
)
132134

@@ -141,6 +143,7 @@ def cmd_analyze(filenames=None, **kwargs):
141143
relroot=REPO_ROOT,
142144
_analyze=_analyzer.analyze,
143145
formats=formats,
146+
file_maxsizes=_parser.MAX_SIZES,
144147
**kwargs
145148
)
146149

0 commit comments

Comments
 (0)