From e35e4748c7474d3c9cd6352e0391fbd2994e49e9 Mon Sep 17 00:00:00 2001 From: Praveen K Pandey Date: Tue, 28 Jul 2026 14:00:29 +0530 Subject: [PATCH 1/8] avocado-setup: add resume_job_dir parameter to run_test Add optional resume_job_dir parameter to run_test() to support resume mode. Defaults to None so all existing call sites are unaffected. Signed-off-by: Praveen K Pandey --- avocado-setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/avocado-setup.py b/avocado-setup.py index 6475da2..19e28b7 100644 --- a/avocado-setup.py +++ b/avocado-setup.py @@ -383,13 +383,14 @@ def bootstrap(enable_kvm=False, guest_os=None): helper.copy_dir_file(postscript, postscript_dir) -def run_test(testsuite, avocado_bin, runner, linux_src_path): +def run_test(testsuite, avocado_bin, runner, linux_src_path, resume_job_dir=None): """ To run given testsuite :param testsuite: Testsuite object which has details about the tests :param avocado_bin: Executable path of avocado :param runner: Whether to use --test-runner runner (True) or --max-parallel-tasks=1 (False) :param linux_src_path: Path to kernel source for gcov coverage (or None) + :param resume_job_dir: Prior avocado job dir for resume mode (or None) """ nrun = True if runner: From 03f2c427dd63079f7d89908104fa2917cdf60834 Mon Sep 17 00:00:00 2001 From: Praveen K Pandey Date: Tue, 28 Jul 2026 14:00:53 +0530 Subject: [PATCH 2/8] avocado-setup: restructure run_test normal path into else block Move the existing nrun/cmd build logic into an else branch to prepare for resume_job_dir handling in the if branch. No functional change to normal runs. Signed-off-by: Praveen K Pandey --- avocado-setup.py | 65 +++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/avocado-setup.py b/avocado-setup.py index 19e28b7..adf40c6 100644 --- a/avocado-setup.py +++ b/avocado-setup.py @@ -392,39 +392,42 @@ def run_test(testsuite, avocado_bin, runner, linux_src_path, resume_job_dir=None :param linux_src_path: Path to kernel source for gcov coverage (or None) :param resume_job_dir: Prior avocado job dir for resume mode (or None) """ - nrun = True - if runner: - runner = '--test-runner runner' - nrun = False + if resume_job_dir: + pass # placeholder — filled in next commit else: - runner = '--max-parallel-tasks=1' - - logger.info('') - if 'guest' in testsuite.type: - guest_args = TestSuite.guest_add_args - logger.info("Running Guest Tests Suite %s", testsuite.shortname) - if "sanity" in testsuite.shortname: - guest_args = " --vt-only-filter %s " % args.guest_os - cmd = "%s run --vt-type %s --vt-config %s \ - --force-job-id %s \ - --job-results-dir %s %s" % (avocado_bin, testsuite.vt_type, - testsuite.config(), - testsuite.jobid, - testsuite.resultdir, guest_args) - if 'host' in testsuite.type: - logger.info("Running Host Tests Suite %s", testsuite.shortname) - if nrun: - cmd = "%s run %s %s" % (avocado_bin, runner, os.path.join(TEST_DIR, testsuite.test)) + nrun = True + if runner: + runner = '--test-runner runner' + nrun = False else: - cmd = "%s run %s %s" % (avocado_bin, runner, testsuite.test) - if testsuite.mux: - cmd += " -m %s" % os.path.join(TEST_DIR, testsuite.mux) - cmd += " --force-job-id %s \ - --job-results-dir %s %s" % (testsuite.jobid, - testsuite.resultdir, - TestSuite.host_add_args) - if testsuite.args: - cmd += testsuite.args + runner = '--max-parallel-tasks=1' + + logger.info('') + if 'guest' in testsuite.type: + guest_args = TestSuite.guest_add_args + logger.info("Running Guest Tests Suite %s", testsuite.shortname) + if "sanity" in testsuite.shortname: + guest_args = " --vt-only-filter %s " % args.guest_os + cmd = "%s run --vt-type %s --vt-config %s \ + --force-job-id %s \ + --job-results-dir %s %s" % (avocado_bin, testsuite.vt_type, + testsuite.config(), + testsuite.jobid, + testsuite.resultdir, guest_args) + if 'host' in testsuite.type: + logger.info("Running Host Tests Suite %s", testsuite.shortname) + if nrun: + cmd = "%s run %s %s" % (avocado_bin, runner, os.path.join(TEST_DIR, testsuite.test)) + else: + cmd = "%s run %s %s" % (avocado_bin, runner, testsuite.test) + if testsuite.mux: + cmd += " -m %s" % os.path.join(TEST_DIR, testsuite.mux) + cmd += " --force-job-id %s \ + --job-results-dir %s %s" % (testsuite.jobid, + testsuite.resultdir, + TestSuite.host_add_args) + if testsuite.args: + cmd += testsuite.args input_file = args.inputfile try: From ccb8c7f08693826b8879845b8d6170f094fc0a95 Mon Sep 17 00:00:00 2001 From: Praveen K Pandey Date: Tue, 28 Jul 2026 14:01:19 +0530 Subject: [PATCH 3/8] avocado-setup: implement avocado replay for resume mode in run_test When resume_job_dir is set, extract the job ID from the job dir name and invoke 'avocado replay '. Modern avocado replay re-runs only the not-passed tests from the prior job automatically, works for both host and guest suites without extra flags. Signed-off-by: Praveen K Pandey --- avocado-setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/avocado-setup.py b/avocado-setup.py index adf40c6..ae5f283 100644 --- a/avocado-setup.py +++ b/avocado-setup.py @@ -393,7 +393,13 @@ def run_test(testsuite, avocado_bin, runner, linux_src_path, resume_job_dir=None :param resume_job_dir: Prior avocado job dir for resume mode (or None) """ if resume_job_dir: - pass # placeholder — filled in next commit + # avocado replay re-runs only not-passed tests from that job. + # The job_id is the trailing hex in the job dir name e.g. + # job-2026-07-28T04.29-07add70 → job_id = 07add70 + job_id = os.path.basename(resume_job_dir).rsplit('-', 1)[-1] + logger.info("Resuming suite %s via avocado replay %s", + testsuite.name, job_id) + cmd = "%s replay %s" % (avocado_bin, job_id) else: nrun = True if runner: From eac505665dcba961e357f166dcce0e2b09e3ddaa Mon Sep 17 00:00:00 2001 From: Praveen K Pandey Date: Tue, 28 Jul 2026 14:01:43 +0530 Subject: [PATCH 4/8] avocado-setup: add --resume CLI argument Add --resume flag to argparse. When set the wrapper uses the existing outputdir to scan prior job results and resume from where the run was interrupted. Usage: python avocado-setup.py --run-suite host_a,host_b,host_c --resume Signed-off-by: Praveen K Pandey --- avocado-setup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/avocado-setup.py b/avocado-setup.py index ae5f283..b538d44 100644 --- a/avocado-setup.py +++ b/avocado-setup.py @@ -732,6 +732,13 @@ def parse_test_config(test_config_file, avocado_bin, enable_kvm, runner): parser.add_argument('--config-norun', dest='NORUNTEST_PATH', action='store', default=NORUNTEST_PATH, help='Specify no run tests config path') + parser.add_argument('--resume', dest='resume', + action='store_true', default=False, + help='Resume an interrupted run using the same ' + '--output-dir results directory. Skips suites ' + 'that completed cleanly, replays the interrupted ' + 'suite via avocado replay, and runs remaining ' + 'suites fresh.') args = parser.parse_args() From 49dd389724c6b73f82633852eb853e5ae556e37e Mon Sep 17 00:00:00 2001 From: Praveen K Pandey Date: Tue, 28 Jul 2026 14:02:23 +0530 Subject: [PATCH 5/8] avocado-setup: scan prior job dirs to build suite_job_map on resume On --resume, scan all job-* subdirs in outputdir and build suite_job_map mapping suite_name to its prior job dir path. Host suites matched by test file basename in avocado test IDs. Guest suites matched by suite shortname in VT test IDs. Job dirs with no/unreadable results.json stored as __interrupted__ fallback (system crashed before results were written). Signed-off-by: Praveen K Pandey --- avocado-setup.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/avocado-setup.py b/avocado-setup.py index b538d44..a0b2df5 100644 --- a/avocado-setup.py +++ b/avocado-setup.py @@ -898,6 +898,55 @@ def parse_test_config(test_config_file, avocado_bin, enable_kvm, runner): "Config file not present") count_testsuites_status[Testsuite_status.Cant_Run.value] += 1 continue + # Build suite_job_map: scan outputdir job-* dirs and match each to a + # suite name so the run loop knows which suites completed, which was + # interrupted, and which never ran. + suite_job_map = {} + if args.resume: + if not os.path.isdir(outputdir): + logger.error("RESUME: results dir does not exist: %s", outputdir) + sys.exit(1) + # host suites: match test file basename in avocado test IDs + # guest suites: match suite shortname in avocado VT test IDs + basename_to_suite = {} + shortname_to_suite = {} + for ts_name in Testsuites_list: + ts = Testsuites[ts_name] + if ts.test: + basename_to_suite[os.path.basename(ts.test.rstrip('$'))] = ts_name + elif ts.type == 'guest': + shortname_to_suite[ts.shortname] = ts_name + for entry in sorted(os.listdir(outputdir)): + job_path = os.path.join(outputdir, entry) + if not entry.startswith("job-") or not os.path.isdir(job_path): + continue + rj = os.path.join(job_path, "results.json") + matched = None + if os.path.isfile(rj): + try: + with open(rj, encoding="utf-8") as fp: + rdata = json.load(fp) + for t in rdata.get("tests", []): + tid = t.get("id", "") + tbase = tid.split(":")[0].split("/")[-1] + if tbase in basename_to_suite: + matched = basename_to_suite[tbase] + break + for sname, ts_name in shortname_to_suite.items(): + if sname in tid: + matched = ts_name + break + if matched: + break + except (OSError, json.JSONDecodeError): + pass + if matched: + suite_job_map[matched] = job_path + else: + suite_job_map.setdefault("__interrupted__", job_path) + logger.info("RESUME: suite to job dir map from %s: %s", + outputdir, suite_job_map) + # Run Tests count_testsuites_status[Testsuite_status.Total.value] = len(Testsuites_list) for test_suite in Testsuites_list: From c428a0638fc032ededd207af85d940bfec9ddb1d Mon Sep 17 00:00:00 2001 From: Praveen K Pandey Date: Tue, 28 Jul 2026 14:02:44 +0530 Subject: [PATCH 6/8] avocado-setup: add _suite_completed and _suite_replay_dir helpers _suite_completed: returns True when a suite has a prior job dir with a clean results.json (total > 0, interrupt == 0). _suite_replay_dir: returns the prior job dir to replay for a suite, or None when the suite never ran (no job dir) so it runs fresh. Signed-off-by: Praveen K Pandey --- avocado-setup.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/avocado-setup.py b/avocado-setup.py index a0b2df5..02c45e6 100644 --- a/avocado-setup.py +++ b/avocado-setup.py @@ -947,6 +947,34 @@ def parse_test_config(test_config_file, avocado_bin, enable_kvm, runner): logger.info("RESUME: suite to job dir map from %s: %s", outputdir, suite_job_map) + def _suite_completed(suite_name): + """Return True if suite has a prior job with a clean results.json. + A suite is complete when total > 0 and interrupt == 0. + """ + if suite_name not in suite_job_map: + return False + rj = os.path.join(suite_job_map[suite_name], "results.json") + if not os.path.isfile(rj): + return False + try: + with open(rj, encoding="utf-8") as fp: + d = json.load(fp) + return (int(d.get("total", 0)) > 0 and + int(d.get("interrupt", 0)) == 0) + except (OSError, json.JSONDecodeError): + return False + + def _suite_replay_dir(suite_name): + """Return the prior job dir to replay for this suite, or None. + Returns None when suite has no prior job dir (never ran) + so it gets a normal fresh run instead of a replay. + """ + if suite_name in suite_job_map and not _suite_completed(suite_name): + return suite_job_map[suite_name] + if "__interrupted__" in suite_job_map and suite_name in suite_job_map: + return suite_job_map.pop("__interrupted__") + return None + # Run Tests count_testsuites_status[Testsuite_status.Total.value] = len(Testsuites_list) for test_suite in Testsuites_list: From 982dc5bc0e854bc3bf5b6d527a04fc793cf72885 Mon Sep 17 00:00:00 2001 From: Praveen K Pandey Date: Tue, 28 Jul 2026 14:03:12 +0530 Subject: [PATCH 7/8] avocado-setup: integrate resume logic into main run loop On --resume, for each suite in order: - completed (clean results.json) -> skip with logged message - interrupted (no/partial results) -> avocado replay - never ran (no job dir at all) -> normal fresh run Suites that never ran (e.g. system rebooted before they started) receive None from _suite_replay_dir and run fully fresh. Signed-off-by: Praveen K Pandey --- avocado-setup.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/avocado-setup.py b/avocado-setup.py index 02c45e6..ae09f76 100644 --- a/avocado-setup.py +++ b/avocado-setup.py @@ -978,10 +978,28 @@ def _suite_replay_dir(suite_name): # Run Tests count_testsuites_status[Testsuite_status.Total.value] = len(Testsuites_list) for test_suite in Testsuites_list: - if not Testsuites[test_suite].run == Testsuite_status.Cant_Run.value: - run_test(Testsuites[test_suite], avocado_bin, args.runner, args.linux_src_path) - if args.interval: - time.sleep(int(args.interval)) + if Testsuites[test_suite].run == Testsuite_status.Cant_Run.value: + continue + if args.resume: + if _suite_completed(test_suite): + logger.info("RESUME: skipping '%s' (completed in prior run)", + test_suite) + Testsuites[test_suite].runstatus( + Testsuite_status.Run.value, + "Skipped (completed in prior run)") + count_testsuites_status[Testsuite_status.Run.value] += 1 + else: + replay_dir = _suite_replay_dir(test_suite) + if replay_dir: + logger.info("RESUME: replaying '%s' from %s", + test_suite, replay_dir) + run_test(Testsuites[test_suite], avocado_bin, args.runner, + args.linux_src_path, replay_dir) + else: + run_test(Testsuites[test_suite], avocado_bin, args.runner, + args.linux_src_path) + if args.interval: + time.sleep(int(args.interval)) # Finding the space needed for formatting result summary test_name_list = [] From 98873351a9ebdd6078ddf71f6903b4f9f6062067 Mon Sep 17 00:00:00 2001 From: Praveen K Pandey Date: Tue, 28 Jul 2026 14:13:29 +0530 Subject: [PATCH 8/8] README: document --resume option for interrupted run recovery Add --resume to the script help output block and add a new argument details entry (item 20) explaining: - What --resume does (scans results dir, classifies suites) - The three states: completed/skip, interrupted/replay, never ran/fresh - Usage examples with and without --output-dir - Note that --run-suite must match the original run - Works for both host and guest test suites Signed-off-by: Praveen K Pandey --- README.md | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b9b12a..dcd20fb 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ $ ./avocado-setup.py -h [--additional-args ADD_ARGS] [--guest-os GUEST_OS] [--vt {qemu,libvirt}] [--install] [--no-download] [--no-deps-check] [--install-deps] [--clean] [--enable-kvm] [--nrunner] [--run-tests RUN_TESTS] [--config-env CONFIG_PATH] - [--config-norun NORUNTEST_PATH] + [--config-norun NORUNTEST_PATH] [--resume] options: -h, --help show this help message and exit @@ -103,6 +103,9 @@ $ ./avocado-setup.py -h Specify env config path --config-norun NORUNTEST_PATH Specify no run tests config path + --resume Resume an interrupted run using the same --output-dir results + directory. Skips suites that completed cleanly, replays the + interrupted suite via avocado replay, and runs remaining suites fresh. ``` @@ -262,6 +265,36 @@ $ ./avocado-setup.py -h 19. `--config-norun`: > Path to a custom NORUNTEST File path +20. `--resume`: + > Use this option to resume a test run that was interrupted due to a system crash or reboot. + > The wrapper scans the results directory (from `--output-dir` or the default `results/` path) + > and automatically determines the state of each suite: + > + > | Suite state in prior run | Action on resume | + > |---|---| + > | Completed cleanly | Skipped — not re-run | + > | Was running at time of crash/reboot | Replayed via `avocado replay ` — only not-passed tests re-run | + > | Never started (system crashed before it was reached) | Normal fresh run | + > + > **Usage — first run:** + > ``` + > ./avocado-setup.py --run-suite host_a,host_b,host_c + > ``` + > + > **After system reboot/crash — same command with `--resume`:** + > ``` + > ./avocado-setup.py --run-suite host_a,host_b,host_c --resume + > ``` + > + > If a custom `--output-dir` was used in the original run, pass it again with `--resume`: + > ``` + > ./avocado-setup.py --run-suite host_a,host_b,host_c --output-dir /path/to/output --resume + > ``` + > + > NOTE: The `--run-suite` list must be identical to the original run so the wrapper can + > correctly identify which suites ran, which was interrupted, and which never started. + > Works for both host and guest test suites. + ### Customizing Test Suites: The Host and Guest sanity suites were created to include a varied collection of tests to validate new Host OS installations.