Skip to content

test/cli/other_test.py: added more --showtime tests #8517

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
firewave wants to merge 4 commits into
base: main
Choose a base branch
from
+90 −41
Open

test/cli/other_test.py: added more --showtime tests #8517

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6df1fa1
test/cli/other_test.py: test `--showtime` with addons
firewave Apr 30, 2026
bd1cbe3
test/cli/other_test.py: test `--showtime` with clang-tidy
firewave Apr 30, 2026
d8a596d
test/cli/more-projects_test.py: added TODO
firewave Apr 30, 2026
360d961
test/cli/other_test.py: made global variable private
firewave Apr 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion test/cli/more-projects_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ def test_project_std(tmpdir):



@pytest.mark.skip() # clang-tidy is not available in all cases
@pytest.mark.skip() # clang-tidy is not available in all cases - TODO: enable conditionally
def test_clang_tidy(tmpdir):
test_file = os.path.join(tmpdir, 'test.cpp')
with open(test_file, 'wt') as f:
Expand Down
129 changes: 89 additions & 40 deletions test/cli/other_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,14 @@
from xml.etree import ElementTree


try:
# TODO: handle exitcode?
subprocess.call(['clang-tidy', '--version'])
__has_clang_tidy = True
except OSError:
__has_clang_tidy = False


def __remove_verbose_log(l : list):
l.remove('Defines:')
l.remove('Undefines:')
Expand Down Expand Up @@ -954,10 +962,8 @@ def test_unused_function_include(tmpdir):
__test_unused_function_include(tmpdir, [])


# TODO: test with clang-tidy
# TODO: test with --addon
# TODO: test with multiple files
def __test_showtime(tmp_path, showtime, exp_res, exp_last, use_compdb, extra_args=None):
def __test_showtime(tmp_path, showtime, exp_res, exp_last, use_compdb=False, use_addons=False, use_clang_tidy=False, extra_args=None):
test_file = tmp_path / 'test.cpp' # the use of C++ is intentional
with open(test_file, 'wt') as f:
f.write(
Expand All @@ -974,6 +980,17 @@ def __test_showtime(tmp_path, showtime, exp_res, exp_last, use_compdb, extra_arg
'--inline-suppr'
]

if use_addons:
args += ['--addon=misra', '--addon=misc']

if use_clang_tidy:
args += ['--clang-tidy']
args += ['--suppress=clang-tidy-misc-use-internal-linkage']
args += ['--suppress=clang-tidy-google-readability-casting']
args += ['--suppress=clang-tidy-modernize-avoid-c-style-cast']
args += ['--suppress=clang-tidy-hicpp-use-nullptr']
args += ['--suppress=clang-tidy-modernize-use-nullptr']

if use_compdb:
compdb_file = tmp_path / 'compile_commands.json'
create_compile_commands(compdb_file, [test_file])
Expand All @@ -991,6 +1008,9 @@ def __test_showtime(tmp_path, showtime, exp_res, exp_last, use_compdb, extra_arg
exp_len = exp_res
if 'cppcheck internal API usage' in stdout:
exp_len += 1
if use_addons:
exp_len += 1 # TODO: should have individual entries for each addon and whole program analysis
# TODO: add entry for clang-tidy analysis
exp_len += 1 # last line
assert len(lines) == exp_len
for i in range(1, exp_res):
Expand All @@ -1000,118 +1020,154 @@ def __test_showtime(tmp_path, showtime, exp_res, exp_last, use_compdb, extra_arg
assert stderr == ''


def __test_showtime_top5_file(tmp_path, use_compdb):
__test_showtime(tmp_path, 'top5_file', 5, 'Check time: ', use_compdb)
def __test_showtime_top5_file(tmp_path, use_compdb=False):
__test_showtime(tmp_path, 'top5_file', 5, 'Check time: ', use_compdb=use_compdb)


def test_showtime_top5_file(tmp_path):
__test_showtime_top5_file(tmp_path, False)
__test_showtime_top5_file(tmp_path)


def test_showtime_top5_file_compdb(tmp_path):
__test_showtime_top5_file(tmp_path, True)
__test_showtime_top5_file(tmp_path, use_compdb=True)


# TODO: remove extra args when --executor=process works
def __test_showtime_top5_summary(tmp_path, use_compdb):
__test_showtime(tmp_path, 'top5_summary', 5, 'Overall time: ', use_compdb, ['-j1'])
def __test_showtime_top5_summary(tmp_path, use_compdb=False):
__test_showtime(tmp_path, 'top5_summary', 5, 'Overall time: ', use_compdb=use_compdb, extra_args=['-j1'])


def test_showtime_top5_summary(tmp_path):
__test_showtime_top5_summary(tmp_path, False)
__test_showtime_top5_summary(tmp_path)


def test_showtime_top5_summary_compdb(tmp_path):
__test_showtime_top5_summary(tmp_path, True)
__test_showtime_top5_summary(tmp_path, use_compdb=True)


# TODO: remove when --executor=process works
def test_showtime_top5_summary_j_thread(tmp_path):
__test_showtime(tmp_path, 'top5_summary', 5, 'Overall time: ', False, ['-j2', '--executor=thread'])
__test_showtime(tmp_path, 'top5_summary', 5, 'Overall time: ', extra_args=['-j2', '--executor=thread'])


# TODO: remove when --executor=process works
def test_showtime_top5_summary_compdb_j_thread(tmp_path):
__test_showtime(tmp_path, 'top5_summary', 5, 'Overall time: ', True, ['-j2', '--executor=thread'])
__test_showtime(tmp_path, 'top5_summary', 5, 'Overall time: ', use_compdb=True, extra_args=['-j2', '--executor=thread'])


# TODO: remove override when fixed
@pytest.mark.skipif(sys.platform == 'win32', reason="requires ProcessExecutor")
@pytest.mark.xfail(strict=True) # TODO: need to transfer the timer results to parent process - see #4452
def test_showtime_top5_summary_j_process(tmp_path):
__test_showtime(tmp_path, 'top5_summary', 5, 'Overall time: ', False, ['-j2', '--executor=process'])
__test_showtime(tmp_path, 'top5_summary', 5, 'Overall time: ', extra_args=['-j2', '--executor=process'])


# TODO: remove override when fixed
@pytest.mark.skipif(sys.platform == 'win32', reason="requires ProcessExecutor")
@pytest.mark.xfail(strict=True) # TODO: need to transfer the timer results to parent process - see #4452
def test_showtime_top5_summary_compdb_j_process(tmp_path):
__test_showtime(tmp_path, 'top5_summary', 5, 'Overall time: ', True, ['-j2', '--executor=process'])
__test_showtime(tmp_path, 'top5_summary', 5, 'Overall time: ', use_compdb=True, extra_args=['-j2', '--executor=process'])


def __test_showtime_file(tmp_path, use_compdb):
def __test_showtime_file(tmp_path, use_compdb=False, use_addons=False, use_clang_tidy=False):
exp_res = 79
# project analysis does not call Preprocessor::getConfig()
if use_compdb:
exp_res -= 1
__test_showtime(tmp_path, 'file', exp_res, 'Check time: ', use_compdb)
__test_showtime(tmp_path, 'file', exp_res, 'Check time: ', use_compdb=use_compdb, use_addons=use_addons, use_clang_tidy=use_clang_tidy)


def test_showtime_file(tmp_path):
__test_showtime_file(tmp_path, False)
__test_showtime_file(tmp_path)


def test_showtime_file_compdb(tmp_path):
__test_showtime_file(tmp_path, True)
__test_showtime_file(tmp_path, use_compdb=True)


def test_showtime_file_addon(tmp_path):
__test_showtime_file(tmp_path, use_addons=True)


def test_showtime_file_addon_compdb(tmp_path):
__test_showtime_file(tmp_path, use_addons=True, use_compdb=True)


@pytest.mark.skipif(not __has_clang_tidy, reason='clang-tidy is not available')
def test_showtime_file_clang_tidy(tmp_path):
__test_showtime_file(tmp_path, use_clang_tidy=True)


@pytest.mark.skipif(not __has_clang_tidy, reason='clang-tidy is not available')
def test_showtime_file_clang_tidy_compdb(tmp_path):
__test_showtime_file(tmp_path, use_clang_tidy=True, use_compdb=True)


# TODO: remove extra args when --executor=process works
def __test_showtime_summary(tmp_path, use_compdb):
def __test_showtime_summary(tmp_path, use_compdb=False, use_addons=False, use_clang_tidy=False):
exp_res = 79
# project analysis does not call Preprocessor::getConfig()
if use_compdb:
exp_res -= 1
__test_showtime(tmp_path, 'summary', exp_res, 'Overall time: ', use_compdb, ['-j1'])
__test_showtime(tmp_path, 'summary', exp_res, 'Overall time: ', use_compdb=use_compdb, use_addons=use_addons, use_clang_tidy=use_clang_tidy, extra_args=['-j1'])


def test_showtime_summary(tmp_path):
__test_showtime_summary(tmp_path, False,)
__test_showtime_summary(tmp_path)


def test_showtime_summary_compdb(tmp_path):
__test_showtime_summary(tmp_path, True)
__test_showtime_summary(tmp_path, use_compdb=True)


def test_showtime_summary_addon(tmp_path):
__test_showtime_summary(tmp_path, use_addons=True)


def test_showtime_summary_addon_compdb(tmp_path):
__test_showtime_summary(tmp_path, use_addons=True, use_compdb=True)


@pytest.mark.skipif(not __has_clang_tidy, reason='clang-tidy is not available')
def test_showtime_summary_clang_tidy(tmp_path):
__test_showtime_summary(tmp_path, use_clang_tidy=True)


@pytest.mark.skipif(not __has_clang_tidy, reason='clang-tidy is not available')
def test_showtime_summary_clang_tidy_compdb(tmp_path):
__test_showtime_summary(tmp_path, use_clang_tidy=True, use_compdb=True)


# TODO: remove when --executor=process works
def test_showtime_summary_j_thread(tmp_path):
__test_showtime(tmp_path, 'summary', 79, 'Overall time: ', False, ['-j2', '--executor=thread'])
__test_showtime(tmp_path, 'summary', 79, 'Overall time: ', extra_args=['-j2', '--executor=thread'])


# TODO: remove when --executor=process works
def test_showtime_summary_compdb_j_thread(tmp_path):
__test_showtime(tmp_path, 'summary', 78, 'Overall time: ', True, ['-j2', '--executor=thread'])
__test_showtime(tmp_path, 'summary', 78, 'Overall time: ', use_compdb=True, extra_args=['-j2', '--executor=thread'])


# TODO: remove override when fixed
@pytest.mark.skipif(sys.platform == 'win32', reason="requires ProcessExecutor")
@pytest.mark.xfail(strict=True) # TODO: need to transfer the timer results to parent process - see #4452
def test_showtime_summary_j_process(tmp_path):
__test_showtime(tmp_path, 'summary', 79, 'Overall time: ', False, ['-j2', '--executor=process'])
__test_showtime(tmp_path, 'summary', 79, 'Overall time: ', extra_args=['-j2', '--executor=process'])


# TODO: remove override when fixed
@pytest.mark.skipif(sys.platform == 'win32', reason="requires ProcessExecutor")
@pytest.mark.xfail(strict=True) # TODO: need to transfer the timer results to parent process - see #4452
def test_showtime_summary_compdb_j_process(tmp_path):
__test_showtime(tmp_path, 'summary', 78, 'Overall time: ', True, ['-j2', '--executor=process'])
__test_showtime(tmp_path, 'summary', 78, 'Overall time: ', use_compdb=True, extra_args=['-j2', '--executor=process'])


def __test_showtime_file_total(tmp_path, use_compdb):
__test_showtime(tmp_path, 'file-total', 0, 'Check time: ', use_compdb)
def __test_showtime_file_total(tmp_path, use_compdb=False):
__test_showtime(tmp_path, 'file-total', 0, 'Check time: ', use_compdb=use_compdb)


def test_showtime_file_total(tmp_path):
__test_showtime_file_total(tmp_path, False)
__test_showtime_file_total(tmp_path)


def test_showtime_file_total_compdb(tmp_path):
Expand Down Expand Up @@ -3423,13 +3479,6 @@ def test_check_unused_templates_func(tmp_path): # #13714
assert stdout.splitlines() == []
assert stderr.splitlines() == [] # no error since the unused templates are not being checked

try:
# TODO: handle exitcode?
subprocess.call(['clang-tidy', '--version'])
has_clang_tidy = True
except OSError:
has_clang_tidy = False

def __test_clang_tidy(tmpdir, use_compdb):
test_file = os.path.join(tmpdir, 'test.cpp')
with open(test_file, 'wt') as f:
Expand Down Expand Up @@ -3460,18 +3509,18 @@ def __test_clang_tidy(tmpdir, use_compdb):
]


@pytest.mark.skipif(not has_clang_tidy, reason='clang-tidy is not available')
@pytest.mark.skipif(not __has_clang_tidy, reason='clang-tidy is not available')
@pytest.mark.xfail(strict=True) # TODO: clang-tidy is only invoked with FileSettings - see #12053
def test_clang_tidy(tmpdir): # #12053
__test_clang_tidy(tmpdir, False)


@pytest.mark.skipif(not has_clang_tidy, reason='clang-tidy is not available')
@pytest.mark.skipif(not __has_clang_tidy, reason='clang-tidy is not available')
def test_clang_tidy_project(tmpdir):
__test_clang_tidy(tmpdir, True)


@pytest.mark.skipif(not has_clang_tidy, reason='clang-tidy is not available')
@pytest.mark.skipif(not __has_clang_tidy, reason='clang-tidy is not available')
def test_clang_tidy_error_exit(tmp_path): # #13828 / #13829
test_file = tmp_path / 'test.cpp'
with open(test_file, 'wt') as f:
Expand Down
Loading