[2025-07-26 01:19:36,764][ INFO][PID:2676695] Marking build as starting [2025-07-26 01:19:37,209][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:19:37,210][ INFO][PID:2676695] VM allocation process starts [2025-07-26 01:19:37,221][ INFO][PID:2676695] Trying to allocate VM: ResallocHost, ticket_id=4169077, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-07-26 01:19:42,222][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:19:47,223][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:19:52,224][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:19:57,225][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:20:02,226][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:20:07,228][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:20:12,229][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:20:17,230][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:20:22,231][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:20:27,232][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:20:32,694][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:20:37,825][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:20:42,826][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:20:45,286][ INFO][PID:2676695] Allocated host ResallocHost, ticket_id=4169077, hostname=140.211.168.48, name=copr_osuosl_p09_prod_05888550_20250726_011701, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-07-26 01:20:45,287][ INFO][PID:2676695] Allocating ssh connection to builder [2025-07-26 01:20:45,287][ INFO][PID:2676695] Checking that builder machine is OK [2025-07-26 01:20:46,748][ INFO][PID:2676695] Running remote command: copr-builder-ready fedora-41-ppc64le [2025-07-26 01:20:47,816][ INFO][PID:2676695] Red Hat subscription not needed for fedora-41-ppc64le Builder is ready to be used [2025-07-26 01:20:47,817][ INFO][PID:2676695] Filling build.info file with builder info [2025-07-26 01:20:47,817][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:20:47,818][ INFO][PID:2676695] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "rpmsoftwaremanagement", "project_name": "dnf5-testing-nightly", "project_dirname": "dnf5-testing-nightly", "submitter": "rpmsoftwaremanagement", "ended_on": null, "started_on": 1753492847.8186085, "submitted_on": null, "status": 3, "chroot": "fedora-41-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "9339900-fedora-41-ppc64le", "build_id": 9339900, "package_name": "libdnf", "package_version": "0.75.0-0.20250726011806.0.74.0+27.g96de9331", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/rpmsoftwaremanagement/dnf5-testing-nightly/libdnf", "git_hash": "ed0eb074385d454d26110f08e8ed566d2067ac20", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "rpmsoftwaremanagement/dnf5-testing-nightly--rpmsoftwaremanagement", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/rpmsoftwaremanagement/dnf5-testing-nightly/fedora-41-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": 50 } ], "background": false, "bootstrap": "off", "fedora_review": false, "isolation": "default", "repo_priority": 50, "tags": [ "arch_ppc64le" ], "with_opts": [ "dnf5_obsoletes_dnf" ], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf5-testing-nightly", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/rpmsoftwaremanagement/dnf5-testing-nightly", "result_dir": "09339900-libdnf", "built_packages": "", "id": 9339900, "mockchain_macros": { "copr_username": "rpmsoftwaremanagement", "copr_projectname": "dnf5-testing-nightly", "vendor": "Fedora Project COPR (rpmsoftwaremanagement/dnf5-testing-nightly)" } } ] } [2025-07-26 01:20:47,894][ INFO][PID:2676695] Sending fedora-messaging bus message in build.start [2025-07-26 01:20:48,581][ INFO][PID:2676695] Sending fedora-messaging bus message in chroot.start [2025-07-26 01:20:48,607][ INFO][PID:2676695] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/9339900-fedora-41-ppc64le --chroot fedora-41-ppc64le --detached [2025-07-26 01:20:49,885][ INFO][PID:2676695] Downloading the builder-live.log file, attempt 1 [2025-07-26 01:20:49,887][ INFO][PID:2676695] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.48 copr-rpmbuild-log [2025-07-26 01:20:54,896][ INFO][PID:2676695] Periodic builder liveness probe: alive [2025-07-26 01:20:54,897][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:20:59,899][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:21:04,900][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:21:09,902][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:21:14,904][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:21:19,910][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:21:24,913][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:21:29,916][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:21:34,918][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:21:39,921][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:21:44,923][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:21:49,924][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:21:54,925][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:21:59,926][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:22:04,927][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:22:09,928][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:22:14,930][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:22:19,932][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:22:24,933][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:22:29,934][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:22:34,935][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:22:39,939][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:22:44,941][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:22:49,942][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:22:54,943][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:22:59,946][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:23:04,950][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:23:09,955][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:23:14,956][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:23:19,958][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:23:24,960][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:23:29,962][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:23:34,964][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:23:39,965][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:23:44,966][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:23:49,967][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:23:54,969][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:23:59,970][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:24:04,971][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:24:09,972][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:24:14,974][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:24:19,976][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:24:24,978][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:24:28,141][ ERROR][PID:2676695] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-26 01:24:29,982][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:24:34,984][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:24:38,146][WARNING][PID:2676695] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-26 01:24:39,985][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:24:44,988][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:24:48,147][ INFO][PID:2676695] Downloading the builder-live.log file, attempt 2 [2025-07-26 01:24:48,149][ INFO][PID:2676695] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.48 copr-rpmbuild-log [2025-07-26 01:24:49,989][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:24:54,991][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:24:59,992][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:25:04,993][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:25:09,994][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:25:14,996][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:25:19,999][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:25:25,000][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:25:30,002][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:25:35,004][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:25:40,006][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:25:45,007][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:25:48,288][ ERROR][PID:2676695] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-26 01:25:50,009][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:25:55,012][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:25:58,295][WARNING][PID:2676695] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-26 01:26:00,016][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:26:05,017][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:26:08,296][ INFO][PID:2676695] Downloading the builder-live.log file, attempt 3 [2025-07-26 01:26:08,301][ INFO][PID:2676695] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.48 copr-rpmbuild-log [2025-07-26 01:26:10,031][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:26:15,033][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:26:20,034][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:26:25,038][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:26:30,045][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:26:35,046][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:26:40,047][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:26:45,048][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:26:50,049][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:26:55,051][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:27:00,053][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:27:05,054][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:27:08,442][ ERROR][PID:2676695] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-26 01:27:10,055][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:27:15,056][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:27:18,447][WARNING][PID:2676695] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-26 01:27:20,057][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:27:25,058][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:27:28,448][ INFO][PID:2676695] Downloading the builder-live.log file, attempt 4 [2025-07-26 01:27:28,450][ INFO][PID:2676695] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.48 copr-rpmbuild-log [2025-07-26 01:27:30,059][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:27:35,061][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:27:40,062][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:27:45,063][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:27:50,064][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:27:55,065][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:28:00,066][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:28:05,067][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:28:10,068][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:28:15,069][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:28:20,070][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:28:25,071][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:28:28,588][ ERROR][PID:2676695] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-26 01:28:30,072][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:28:35,073][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:28:38,593][ ERROR][PID:2676695] Giving up for unstable SSH, failures: 1753493078.1461155, 1753493158.2958627, 1753493238.4477656, 1753493318.593559 [2025-07-26 01:28:38,595][ ERROR][PID:2676695] Re-try request for task on 'ResallocHost, ticket_id=4169077, hostname=140.211.168.48, name=copr_osuosl_p09_prod_05888550_20250726_011701, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-07-26 01:28:38,595][ INFO][PID:2676695] Releasing VM back to pool [2025-07-26 01:28:38,607][ INFO][PID:2676695] Retry #1 (on other host) [2025-07-26 01:28:38,608][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:28:38,609][ INFO][PID:2676695] VM allocation process starts [2025-07-26 01:28:38,621][ INFO][PID:2676695] Trying to allocate VM: ResallocHost, ticket_id=4169261, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-07-26 01:28:43,623][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:28:48,624][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:28:53,626][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:28:56,654][ INFO][PID:2676695] Allocated host ResallocHost, ticket_id=4169261, hostname=140.211.168.107, name=copr_osuosl_p09_prod_05888607_20250726_012316, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-07-26 01:28:56,655][ INFO][PID:2676695] Allocating ssh connection to builder [2025-07-26 01:28:56,655][ INFO][PID:2676695] Checking that builder machine is OK [2025-07-26 01:28:57,610][ INFO][PID:2676695] Running remote command: copr-builder-ready fedora-41-ppc64le [2025-07-26 01:28:58,607][ INFO][PID:2676695] Red Hat subscription not needed for fedora-41-ppc64le Builder is ready to be used [2025-07-26 01:28:58,607][ INFO][PID:2676695] Filling build.info file with builder info [2025-07-26 01:28:58,611][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:28:58,612][ INFO][PID:2676695] Not re-notifying FE and msg buses for the new host. [2025-07-26 01:28:58,613][ INFO][PID:2676695] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/9339900-fedora-41-ppc64le --chroot fedora-41-ppc64le --detached [2025-07-26 01:28:59,847][ INFO][PID:2676695] Downloading the builder-live.log file, attempt 1 [2025-07-26 01:28:59,848][ INFO][PID:2676695] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.107 copr-rpmbuild-log [2025-07-26 01:29:04,847][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:29:09,849][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:29:14,850][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:29:19,851][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:29:24,852][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:29:29,853][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:29:34,854][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:29:39,856][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:29:44,857][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:29:49,858][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:29:54,859][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:29:59,860][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:30:04,861][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:30:09,862][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:30:14,863][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:30:19,865][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:30:24,866][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:30:29,867][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:30:34,868][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:30:39,869][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:30:44,870][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:30:49,871][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:30:54,873][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:30:59,881][ INFO][PID:2676695] Periodic builder liveness probe: alive [2025-07-26 01:30:59,881][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:31:04,882][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:31:09,883][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:31:14,885][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:31:19,886][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:31:24,887][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:31:29,889][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:31:34,890][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:31:39,891][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:31:44,892][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:31:49,893][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:31:54,894][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:31:59,895][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:32:04,896][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:32:09,897][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:32:14,898][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:32:19,899][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:32:24,900][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:32:29,901][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:32:34,902][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:32:39,904][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:32:44,905][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:32:49,906][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:32:54,907][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:32:59,908][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:33:04,909][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:33:09,910][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:33:14,911][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:33:19,912][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:33:24,913][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:33:29,914][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:33:34,915][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:33:39,916][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:33:44,918][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:33:49,919][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:33:54,920][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:33:59,921][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:34:04,922][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:34:09,923][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:34:14,924][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:34:19,925][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:34:24,926][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:34:29,929][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:34:34,930][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:34:39,933][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:34:44,934][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:34:49,935][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:34:54,936][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:34:59,938][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:35:04,940][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:35:09,941][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:35:14,942][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:35:19,943][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:35:24,944][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:35:29,945][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:35:34,946][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:35:39,948][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:35:44,949][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:35:49,950][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:35:54,951][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:35:59,952][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:36:04,953][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:36:09,954][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:36:14,955][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:36:19,956][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:36:24,958][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:36:29,959][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:36:34,960][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:36:39,961][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:36:44,962][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:36:49,963][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:36:54,964][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:36:59,966][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:37:04,967][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:37:09,968][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:37:14,969][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:37:19,970][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:37:24,971][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:37:29,972][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:37:34,974][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:37:39,975][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:37:44,976][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:37:49,977][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:37:54,979][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:37:59,980][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:38:04,981][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:38:09,982][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:38:14,983][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:38:19,985][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:38:24,986][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:38:29,987][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:38:34,988][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:38:39,989][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:38:44,990][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:38:49,992][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:38:54,993][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:38:59,994][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:39:04,994][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:39:09,996][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:39:14,997][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:39:19,998][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:39:24,999][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:39:30,000][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:39:35,001][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:39:40,002][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:39:45,003][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:39:50,004][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:39:55,005][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:40:00,006][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:40:05,007][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:40:10,008][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:40:15,009][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:40:20,010][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:40:25,011][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:40:30,012][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:40:35,013][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:40:40,014][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:40:45,015][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:40:50,016][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:40:55,018][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:41:00,024][ INFO][PID:2676695] Periodic builder liveness probe: alive [2025-07-26 01:41:00,025][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:41:05,025][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:41:10,026][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:41:15,027][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:41:20,028][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:41:25,030][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:41:30,031][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:41:35,032][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:41:40,033][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:41:45,034][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:41:50,036][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:41:55,037][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:42:00,038][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:42:05,039][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:42:10,040][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:42:15,041][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:42:20,042][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:42:25,043][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:42:30,044][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:42:35,045][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:42:40,046][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:42:45,047][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:42:50,048][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:42:55,050][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:43:00,051][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:43:05,052][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:43:10,053][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:43:15,054][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:43:20,055][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:43:25,056][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:43:30,057][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:43:35,058][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:43:40,059][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:43:45,060][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:43:50,061][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:43:55,062][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:44:00,063][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:44:05,064][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:44:10,065][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:44:15,066][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:44:20,067][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:44:25,068][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:44:30,069][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:44:35,070][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:44:40,071][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:44:45,073][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:44:50,074][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:44:55,074][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:45:00,075][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:45:05,076][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:45:10,077][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:45:15,078][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:45:20,079][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:45:25,080][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:45:30,081][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:45:35,082][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:45:40,083][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:45:45,084][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:45:50,085][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:45:55,086][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:46:00,087][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:46:05,088][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:46:10,089][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:46:15,090][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:46:20,091][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:46:25,092][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:46:30,093][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:46:35,094][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:46:40,096][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:46:45,096][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:46:50,098][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:46:55,098][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:47:00,100][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:47:05,101][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:47:10,102][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:47:15,103][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:47:20,103][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:47:25,104][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:47:30,106][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:47:35,107][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:47:40,108][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:47:45,109][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:47:50,110][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:47:55,111][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:48:00,112][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:48:05,114][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:48:10,116][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:48:15,117][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:48:20,118][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:48:25,119][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:48:30,120][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:48:35,121][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:48:40,122][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:48:45,123][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:48:50,124][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:48:55,125][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:49:00,127][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:49:05,128][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:49:10,129][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:49:15,130][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:49:20,131][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:49:25,132][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:49:30,133][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:49:35,134][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:49:40,135][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:49:45,136][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:49:50,137][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:49:55,138][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:50:00,139][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:50:05,140][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:50:10,142][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:50:15,143][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:50:20,144][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:50:25,145][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:50:30,146][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:50:35,147][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:50:40,148][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:50:45,149][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:50:50,150][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:50:55,151][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:51:00,157][ INFO][PID:2676695] Periodic builder liveness probe: alive [2025-07-26 01:51:00,157][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:51:05,158][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:51:10,159][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:51:15,160][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:51:20,161][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:51:25,162][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:51:30,163][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:51:35,165][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:51:40,167][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:51:45,170][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:51:50,172][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:51:55,173][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:52:00,175][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:52:05,178][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:52:10,184][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:52:15,186][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:52:20,190][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:52:25,193][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:52:30,199][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:52:35,202][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:52:40,203][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:52:45,205][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:52:50,210][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:52:55,212][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:53:00,213][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:53:05,214][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:53:10,215][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:53:15,216][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:53:20,217][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:53:25,218][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:53:30,219][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:53:35,226][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:53:40,228][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:53:45,230][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:53:50,232][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:53:55,234][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:54:00,235][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:54:05,237][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:54:10,242][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:54:13,139][ ERROR][PID:2676695] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-26 01:54:15,244][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:54:20,245][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:54:23,139][WARNING][PID:2676695] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-26 01:54:25,247][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:54:30,253][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:54:33,140][ INFO][PID:2676695] Downloading the builder-live.log file, attempt 2 [2025-07-26 01:54:33,147][ INFO][PID:2676695] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.107 copr-rpmbuild-log [2025-07-26 01:54:35,256][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:54:40,260][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:54:45,262][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:54:50,264][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:54:55,266][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:55:00,267][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:55:05,268][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:55:10,273][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:55:15,274][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:55:20,276][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:55:25,277][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:55:30,278][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:55:33,206][ ERROR][PID:2676695] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-26 01:55:35,279][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:55:40,281][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:55:43,211][WARNING][PID:2676695] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-26 01:55:45,282][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:55:50,283][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:55:53,212][ INFO][PID:2676695] Downloading the builder-live.log file, attempt 3 [2025-07-26 01:55:53,213][ INFO][PID:2676695] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.107 copr-rpmbuild-log [2025-07-26 01:55:55,289][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:56:00,291][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:56:05,296][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:56:10,303][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:56:15,308][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:56:20,315][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:56:25,324][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:56:30,340][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:56:35,354][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:56:40,356][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:56:45,357][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:56:50,358][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:56:53,281][ ERROR][PID:2676695] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-26 01:56:55,361][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:57:00,365][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:57:03,285][WARNING][PID:2676695] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-26 01:57:05,366][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:57:10,367][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:57:13,286][ INFO][PID:2676695] Downloading the builder-live.log file, attempt 4 [2025-07-26 01:57:13,288][ INFO][PID:2676695] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.107 copr-rpmbuild-log [2025-07-26 01:57:15,368][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:57:20,369][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:57:25,371][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:57:30,373][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:57:35,426][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:57:40,427][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:57:45,432][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:57:50,445][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:57:55,484][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:58:00,488][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:58:05,491][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:58:10,493][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:58:13,357][ ERROR][PID:2676695] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-26 01:58:15,494][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:58:20,495][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:58:23,362][ ERROR][PID:2676695] Giving up for unstable SSH, failures: 1753494863.1398764, 1753494943.2112718, 1753495023.2859218, 1753495103.3619702 [2025-07-26 01:58:23,363][ ERROR][PID:2676695] Re-try request for task on 'ResallocHost, ticket_id=4169261, hostname=140.211.168.107, name=copr_osuosl_p09_prod_05888607_20250726_012316, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-07-26 01:58:23,364][ INFO][PID:2676695] Releasing VM back to pool [2025-07-26 01:58:23,418][ INFO][PID:2676695] Retry #2 (on other host) [2025-07-26 01:58:23,418][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:58:23,419][ INFO][PID:2676695] VM allocation process starts [2025-07-26 01:58:23,439][ INFO][PID:2676695] Trying to allocate VM: ResallocHost, ticket_id=4169501, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-07-26 01:58:26,451][ INFO][PID:2676695] Allocated host ResallocHost, ticket_id=4169501, hostname=140.211.168.94, name=copr_osuosl_p09_prod_05888882_20250726_015028, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-07-26 01:58:26,452][ INFO][PID:2676695] Allocating ssh connection to builder [2025-07-26 01:58:26,452][ INFO][PID:2676695] Checking that builder machine is OK [2025-07-26 01:58:27,703][ INFO][PID:2676695] Running remote command: copr-builder-ready fedora-41-ppc64le [2025-07-26 01:58:28,745][ INFO][PID:2676695] Red Hat subscription not needed for fedora-41-ppc64le Builder is ready to be used [2025-07-26 01:58:28,745][ INFO][PID:2676695] Filling build.info file with builder info [2025-07-26 01:58:28,805][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:58:28,806][ INFO][PID:2676695] Not re-notifying FE and msg buses for the new host. [2025-07-26 01:58:28,806][ INFO][PID:2676695] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/9339900-fedora-41-ppc64le --chroot fedora-41-ppc64le --detached [2025-07-26 01:58:30,101][ INFO][PID:2676695] Downloading the builder-live.log file, attempt 1 [2025-07-26 01:58:30,102][ INFO][PID:2676695] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.94 copr-rpmbuild-log [2025-07-26 01:58:35,102][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:58:40,103][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:58:45,104][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:58:50,105][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:58:55,106][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:59:00,107][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:59:05,108][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:59:10,109][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:59:15,110][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:59:20,111][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:59:25,112][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:59:30,113][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:59:35,116][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:59:40,117][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:59:45,118][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:59:50,119][ INFO][PID:2676695] Checking for cancel request [2025-07-26 01:59:55,120][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:00:00,121][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:00:05,122][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:00:10,123][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:00:15,124][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:00:20,125][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:00:25,126][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:00:30,127][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:00:35,129][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:00:40,130][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:00:45,131][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:00:50,132][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:00:55,134][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:01:00,135][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:01:05,148][ INFO][PID:2676695] Periodic builder liveness probe: alive [2025-07-26 02:01:05,149][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:01:10,151][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:01:15,152][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:01:20,154][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:01:25,155][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:01:30,156][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:01:35,157][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:01:40,158][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:01:45,159][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:01:50,160][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:01:55,163][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:02:00,164][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:02:05,165][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:02:10,166][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:02:15,167][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:02:20,168][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:02:25,169][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:02:30,170][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:02:35,171][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:02:40,172][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:02:45,173][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:02:50,174][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:02:55,175][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:03:00,177][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:03:05,179][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:03:10,183][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:03:15,184][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:03:20,185][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:03:25,186][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:03:30,187][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:03:35,188][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:03:40,189][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:03:45,190][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:03:50,191][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:03:55,200][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:04:00,201][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:04:05,202][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:04:10,203][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:04:15,204][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:04:20,205][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:04:25,206][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:04:30,207][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:04:35,208][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:04:40,209][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:04:45,210][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:04:50,213][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:04:55,216][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:05:00,217][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:05:05,218][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:05:10,219][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:05:15,220][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:05:20,221][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:05:25,222][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:05:30,224][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:05:35,225][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:05:40,226][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:05:45,227][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:05:50,228][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:05:55,236][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:06:00,245][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:06:05,246][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:06:10,249][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:06:15,251][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:06:20,253][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:06:25,254][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:06:30,257][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:06:35,258][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:06:40,260][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:06:45,261][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:06:50,262][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:06:55,265][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:07:00,267][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:07:05,271][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:07:10,274][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:07:15,275][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:07:20,276][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:07:25,277][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:07:30,279][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:07:35,280][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:07:40,281][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:07:45,283][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:07:50,284][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:07:55,285][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:08:00,289][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:08:05,290][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:08:10,292][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:08:15,296][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:08:20,297][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:08:25,298][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:08:30,300][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:08:35,301][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:08:40,303][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:08:45,306][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:08:50,307][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:08:55,309][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:09:00,311][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:09:05,312][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:09:10,316][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:09:15,321][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:09:20,324][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:09:25,325][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:09:30,327][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:09:35,328][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:09:40,330][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:09:45,332][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:09:50,334][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:09:55,336][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:10:00,337][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:10:05,338][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:10:10,341][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:10:15,345][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:10:20,346][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:10:25,352][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:10:30,354][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:10:35,356][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:10:40,358][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:10:45,363][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:10:50,364][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:10:55,365][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:11:00,367][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:11:05,379][ INFO][PID:2676695] Periodic builder liveness probe: alive [2025-07-26 02:11:05,380][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:11:10,382][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:11:15,384][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:11:20,385][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:11:25,387][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:11:30,389][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:11:35,391][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:11:40,392][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:11:45,393][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:11:50,395][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:11:55,403][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:12:00,411][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:12:05,413][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:12:10,415][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:12:15,416][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:12:20,417][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:12:25,419][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:12:30,420][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:12:35,423][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:12:40,427][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:12:45,428][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:12:50,429][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:12:55,431][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:13:00,432][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:13:05,435][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:13:10,436][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:13:15,437][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:13:20,439][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:13:25,440][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:13:30,441][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:13:35,443][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:13:40,445][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:13:45,447][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:13:50,449][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:13:55,450][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:14:00,453][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:14:05,455][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:14:10,456][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:14:15,457][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:14:20,459][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:14:25,461][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:14:30,462][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:14:35,463][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:14:40,464][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:14:45,465][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:14:50,466][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:14:55,468][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:15:00,472][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:15:05,473][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:15:10,477][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:15:15,479][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:15:20,482][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:15:25,485][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:15:30,487][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:15:35,493][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:15:40,499][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:15:45,503][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:15:50,509][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:15:55,510][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:16:00,518][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:16:05,522][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:16:10,526][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:16:15,531][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:16:20,532][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:16:25,534][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:16:30,540][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:16:35,542][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:16:40,545][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:16:45,549][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:16:50,552][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:16:55,557][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:17:00,558][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:17:05,559][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:17:10,561][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:17:15,563][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:17:20,566][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:17:25,567][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:17:30,569][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:17:35,572][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:17:40,574][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:17:45,576][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:17:50,578][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:17:55,583][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:18:00,586][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:18:05,589][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:18:10,591][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:18:15,973][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:18:21,134][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:18:26,448][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:18:31,892][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:18:37,460][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:18:42,481][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:18:47,496][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:18:52,503][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:18:57,520][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:19:02,526][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:19:07,531][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:19:12,538][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:19:17,542][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:19:22,544][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:19:27,545][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:19:32,547][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:19:37,548][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:19:42,549][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:19:47,550][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:19:52,551][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:19:57,553][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:20:02,554][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:20:07,556][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:20:12,557][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:20:17,560][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:20:22,561][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:20:27,563][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:20:32,568][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:20:37,570][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:20:42,572][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:20:47,577][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:20:52,578][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:20:57,579][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:21:02,580][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:21:07,593][ INFO][PID:2676695] Periodic builder liveness probe: alive [2025-07-26 02:21:07,594][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:21:12,597][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:21:17,598][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:21:22,600][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:21:27,603][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:21:32,607][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:21:37,609][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:21:42,614][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:21:47,617][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:21:52,622][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:21:57,624][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:22:02,629][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:22:07,630][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:22:12,637][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:22:17,638][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:22:22,639][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:22:27,640][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:22:32,644][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:22:37,648][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:22:42,653][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:22:47,654][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:22:52,655][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:22:57,657][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:23:02,660][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:23:07,663][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:23:12,668][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:23:17,670][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:23:22,672][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:23:27,673][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:23:32,675][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:23:37,677][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:23:42,679][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:23:47,680][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:23:52,683][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:23:57,685][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:24:02,686][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:24:07,690][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:24:12,691][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:24:17,692][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:24:22,693][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:24:27,694][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:24:32,695][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:24:37,696][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:24:42,697][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:24:47,698][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:24:52,699][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:24:57,700][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:25:02,701][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:25:07,702][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:25:12,703][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:25:17,704][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:25:22,705][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:25:27,707][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:25:32,708][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:25:37,709][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:25:42,710][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:25:47,711][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:25:52,712][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:25:57,713][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:26:02,714][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:26:07,714][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:26:12,716][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:26:17,716][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:26:22,717][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:26:27,718][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:26:32,719][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:26:37,720][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:26:42,722][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:26:47,722][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:26:52,724][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:26:57,725][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:27:02,726][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:27:07,727][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:27:12,728][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:27:17,729][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:27:22,730][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:27:27,731][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:27:32,732][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:27:37,733][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:27:42,735][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:27:47,736][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:27:52,737][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:27:57,738][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:28:02,739][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:28:07,741][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:28:12,742][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:28:17,743][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:28:22,744][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:28:27,745][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:28:32,748][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:28:37,749][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:28:42,750][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:28:47,751][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:28:52,752][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:28:57,753][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:29:02,754][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:29:07,755][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:29:12,756][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:29:17,757][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:29:22,758][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:29:27,759][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:29:32,760][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:29:37,761][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:29:42,762][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:29:47,763][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:29:52,764][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:29:57,765][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:30:02,766][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:30:07,767][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:30:12,768][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:30:17,769][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:30:22,770][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:30:27,771][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:30:32,772][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:30:37,773][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:30:42,774][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:30:47,775][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:30:48,017][ ERROR][PID:2676695] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-26 02:30:52,776][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:30:57,777][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:30:58,017][WARNING][PID:2676695] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-26 02:31:02,779][ INFO][PID:2676695] Checking for cancel request [2025-07-26 02:31:07,785][ INFO][PID:2676695] Periodic builder liveness probe: dead [2025-07-26 02:31:08,018][ INFO][PID:2676695] Downloading the builder-live.log file, attempt 2 [2025-07-26 02:31:08,020][ INFO][PID:2676695] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.94 copr-rpmbuild-log [2025-07-26 02:31:09,797][ INFO][PID:2676695] Cancel request succeeded out: err: Warning: Permanently added '140.211.168.94' (ED25519) to the list of known hosts. Unexpected exception Traceback (most recent call last): File "/usr/bin/copr-rpmbuild-cancel", line 86, in _main() ~~~~~^^ File "/usr/bin/copr-rpmbuild-cancel", line 69, in _main with open(pidfile, "r") as pidfd: ~~~~^^^^^^^^^^^^^^ FileNotFoundError: [Errno 2] No such file or directory: '/var/lib/copr-rpmbuild/pid' [2025-07-26 02:33:10,132][ ERROR][PID:2676695] Re-try request for task on 'ResallocHost, ticket_id=4169501, hostname=140.211.168.94, name=copr_osuosl_p09_prod_05888882_20250726_015028, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Resalloc ticket FAILED [2025-07-26 02:33:10,133][ INFO][PID:2676695] Releasing VM back to pool [2025-07-26 02:33:10,142][ ERROR][PID:2676695] Backend process error: Three host tried without success: {'140.211.168.107', '140.211.168.48', '140.211.168.94'} [2025-07-26 02:33:10,143][WARNING][PID:2676695] Switching not-finished job state to 'failed' [2025-07-26 02:33:10,143][ INFO][PID:2676695] Worker failed build, took 2081.337019920349 [2025-07-26 02:33:10,144][ INFO][PID:2676695] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "rpmsoftwaremanagement", "project_name": "dnf5-testing-nightly", "project_dirname": "dnf5-testing-nightly", "submitter": "rpmsoftwaremanagement", "ended_on": 1753497190.1430836, "started_on": 1753495108.8060637, "submitted_on": null, "status": 0, "chroot": "fedora-41-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "9339900-fedora-41-ppc64le", "build_id": 9339900, "package_name": "libdnf", "package_version": "0.75.0-0.20250726011806.0.74.0+27.g96de9331", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/rpmsoftwaremanagement/dnf5-testing-nightly/libdnf", "git_hash": "ed0eb074385d454d26110f08e8ed566d2067ac20", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "rpmsoftwaremanagement/dnf5-testing-nightly--rpmsoftwaremanagement", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/rpmsoftwaremanagement/dnf5-testing-nightly/fedora-41-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": 50 } ], "background": false, "bootstrap": "off", "fedora_review": false, "isolation": "default", "repo_priority": 50, "tags": [ "arch_ppc64le" ], "with_opts": [ "dnf5_obsoletes_dnf" ], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf5-testing-nightly", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/rpmsoftwaremanagement/dnf5-testing-nightly", "result_dir": "09339900-libdnf", "built_packages": "", "id": 9339900, "mockchain_macros": { "copr_username": "rpmsoftwaremanagement", "copr_projectname": "dnf5-testing-nightly", "vendor": "Fedora Project COPR (rpmsoftwaremanagement/dnf5-testing-nightly)" } } ] } [2025-07-26 02:33:10,197][ INFO][PID:2676695] Sending fedora-messaging bus message in build.end [2025-07-26 02:33:10,225][ INFO][PID:2676695] Compressing /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf5-testing-nightly/fedora-41-ppc64le/09339900-libdnf/builder-live.log by gzip [2025-07-26 02:33:10,227][ INFO][PID:2676695] Running command 'gzip /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf5-testing-nightly/fedora-41-ppc64le/09339900-libdnf/builder-live.log' as PID 3046108 [2025-07-26 02:33:10,234][ INFO][PID:2676695] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf5-testing-nightly/fedora-41-ppc64le/09339900-libdnf/builder-live.log) [2025-07-26 02:33:10,235][ INFO][PID:2676695] Compressing /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf5-testing-nightly/fedora-41-ppc64le/09339900-libdnf/backend.log by gzip [2025-07-26 02:33:10,236][ INFO][PID:2676695] Running command 'gzip /var/lib/copr/public_html/results/rpmsoftwaremanagement/dnf5-testing-nightly/fedora-41-ppc64le/09339900-libdnf/backend.log' as PID 3046109