[2026-01-05 11:26:41,965][ INFO][PID:348249] Marking build as starting [2026-01-05 11:26:42,034][ INFO][PID:348249] Checking that /var/lib/copr/public_html/results/@osbuild/image-builder/fedora-rawhide-ppc64le/repodata/repomd.xml exists [2026-01-05 11:26:42,056][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:26:42,057][ INFO][PID:348249] VM allocation process starts [2026-01-05 11:26:42,069][ INFO][PID:348249] Trying to allocate VM: ResallocHost, ticket_id=659019, requested_tags=['copr_builder', 'arch_ppc64le'] [2026-01-05 11:26:45,093][ INFO][PID:348249] Allocated host ResallocHost, ticket_id=659019, hostname=2620:52:6:1161:dead:beef:cafe:c152, name=vmhost_p09_03_prod_00921170_20260105_112042, requested_tags=['copr_builder', 'arch_ppc64le'] [2026-01-05 11:26:45,095][ INFO][PID:348249] Allocating ssh connection to builder [2026-01-05 11:26:45,096][ INFO][PID:348249] Checking that builder machine is OK [2026-01-05 11:26:46,066][ INFO][PID:348249] Running remote command: copr-builder-ready fedora-rawhide-ppc64le [2026-01-05 11:26:46,540][ INFO][PID:348249] Red Hat subscription not needed for fedora-rawhide-ppc64le Builder is ready to be used [2026-01-05 11:26:46,540][ INFO][PID:348249] Filling build.info file with builder info [2026-01-05 11:26:46,677][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:26:46,678][ INFO][PID:348249] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "@osbuild", "project_name": "image-builder", "project_dirname": "image-builder", "submitter": "packit", "ended_on": null, "started_on": 1767612406.6781714, "submitted_on": null, "status": 3, "chroot": "fedora-rawhide-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "9975470-fedora-rawhide-ppc64le", "build_id": 9975470, "package_name": "image-builder", "package_version": "45-1.20260105112612118237.main.1.ge2a9768", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/@osbuild/image-builder/image-builder", "git_hash": "8303873e9f33ae8571d5aa3282a7ff6bbd24adc4", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "@osbuild/image-builder--packit", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/@osbuild/image-builder/fedora-rawhide-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "tags": [ "arch_ppc64le" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/@osbuild/image-builder", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/@osbuild/image-builder", "result_dir": "09975470-image-builder", "built_packages": "", "id": 9975470, "mockchain_macros": { "copr_username": "@osbuild", "copr_projectname": "image-builder", "vendor": "Fedora Project COPR (@osbuild/image-builder)" } } ] } [2026-01-05 11:26:46,774][ INFO][PID:348249] Sending fedora-messaging bus message in build.start [2026-01-05 11:26:47,397][ INFO][PID:348249] Sending fedora-messaging bus message in chroot.start [2026-01-05 11:26:47,428][ INFO][PID:348249] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/9975470-fedora-rawhide-ppc64le --chroot fedora-rawhide-ppc64le --detached [2026-01-05 11:26:48,381][ INFO][PID:348249] The copr-rpmbuild seems started, per: stdout: stderr: Warning: Permanently added '2620:52:6:1161:dead:beef:cafe:c152' (ED25519) to the list of known hosts. [2026-01-05 11:26:48,382][ INFO][PID:348249] Downloading the builder-live.log file, attempt 1 [2026-01-05 11:26:48,384][ INFO][PID:348249] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:6:1161:dead:beef:cafe:c152 copr-rpmbuild-log [2026-01-05 11:26:53,388][ INFO][PID:348249] Periodic builder liveness probe: alive [2026-01-05 11:26:53,389][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:26:58,390][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:27:03,391][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:27:08,392][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:27:13,401][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:27:18,402][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:27:23,403][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:27:28,404][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:27:33,406][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:27:38,407][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:27:43,408][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:27:48,409][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:27:53,411][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:27:58,412][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:28:03,413][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:28:08,414][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:28:13,415][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:28:18,416][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:28:23,417][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:28:28,419][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:28:33,420][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:28:38,421][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:28:43,422][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:28:48,429][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:28:53,611][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:28:58,612][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:29:03,699][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:29:08,700][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:29:13,701][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:29:18,702][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:29:23,704][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:29:28,708][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:29:33,711][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:29:38,712][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:29:43,720][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:29:48,723][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:29:53,724][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:29:58,757][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:30:03,759][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:30:08,760][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:30:13,761][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:30:18,762][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:30:23,763][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:30:28,783][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:30:33,785][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:30:38,839][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:30:43,840][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:30:48,841][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:30:53,842][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:30:58,892][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:31:03,893][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:31:08,369][ ERROR][PID:348249] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2026-01-05 11:31:08,894][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:31:13,907][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:31:18,370][WARNING][PID:348249] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2026-01-05 11:31:18,969][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:31:23,971][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:31:28,371][ INFO][PID:348249] Downloading the builder-live.log file, attempt 2 [2026-01-05 11:31:28,372][ INFO][PID:348249] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:6:1161:dead:beef:cafe:c152 copr-rpmbuild-log [2026-01-05 11:31:28,973][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:31:34,000][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:31:39,096][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:31:44,188][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:31:49,190][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:31:54,191][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:31:59,193][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:32:04,194][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:32:09,425][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:32:14,426][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:32:19,427][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:32:24,428][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:32:28,443][ ERROR][PID:348249] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2026-01-05 11:32:29,431][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:32:34,448][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:32:38,453][WARNING][PID:348249] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2026-01-05 11:32:39,448][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:32:44,450][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:32:48,454][ INFO][PID:348249] Downloading the builder-live.log file, attempt 3 [2026-01-05 11:32:48,464][ INFO][PID:348249] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:6:1161:dead:beef:cafe:c152 copr-rpmbuild-log [2026-01-05 11:32:49,452][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:32:54,465][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:32:59,467][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:33:04,468][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:33:09,469][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:33:14,472][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:33:19,473][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:33:24,474][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:33:29,476][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:33:34,477][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:33:39,478][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:33:44,479][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:33:48,554][ ERROR][PID:348249] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2026-01-05 11:33:49,481][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:33:54,486][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:33:58,560][WARNING][PID:348249] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2026-01-05 11:33:59,487][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:34:04,490][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:34:08,562][ INFO][PID:348249] Downloading the builder-live.log file, attempt 4 [2026-01-05 11:34:08,564][ INFO][PID:348249] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:6:1161:dead:beef:cafe:c152 copr-rpmbuild-log [2026-01-05 11:34:09,491][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:34:14,492][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:34:19,493][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:34:24,495][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:34:29,496][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:34:34,497][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:34:39,498][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:34:44,499][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:34:49,500][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:34:54,501][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:34:59,502][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:35:04,504][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:35:08,634][ ERROR][PID:348249] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2026-01-05 11:35:09,505][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:35:14,506][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:35:18,639][ ERROR][PID:348249] Giving up for unstable SSH, failures: 1767612678.370675, 1767612758.4536235, 1767612838.560828, 1767612918.6395166 [2026-01-05 11:35:18,640][ ERROR][PID:348249] Re-try request for task on 'ResallocHost, ticket_id=659019, hostname=2620:52:6:1161:dead:beef:cafe:c152, name=vmhost_p09_03_prod_00921170_20260105_112042, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2026-01-05 11:35:18,641][ INFO][PID:348249] Releasing VM back to pool [2026-01-05 11:35:18,654][ INFO][PID:348249] Retry #1 (on other host) [2026-01-05 11:35:18,655][ INFO][PID:348249] Checking that /var/lib/copr/public_html/results/@osbuild/image-builder/fedora-rawhide-ppc64le/repodata/repomd.xml exists [2026-01-05 11:35:18,656][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:35:18,656][ INFO][PID:348249] VM allocation process starts [2026-01-05 11:35:18,667][ INFO][PID:348249] Trying to allocate VM: ResallocHost, ticket_id=659148, requested_tags=['copr_builder', 'arch_ppc64le'] [2026-01-05 11:35:21,684][ INFO][PID:348249] Allocated host ResallocHost, ticket_id=659148, hostname=2620:52:6:1161:dead:beef:cafe:c155, name=vmhost_p09_03_prod_00921284_20260105_112533, requested_tags=['copr_builder', 'arch_ppc64le'] [2026-01-05 11:35:21,686][ INFO][PID:348249] Allocating ssh connection to builder [2026-01-05 11:35:21,686][ INFO][PID:348249] Checking that builder machine is OK [2026-01-05 11:36:21,756][ ERROR][PID:348249] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 2620:52:6:1161:dead:beef:cafe:c155 port 22: Connection timed out [2026-01-05 11:36:31,757][ ERROR][PID:348249] Re-try request for task on 'ResallocHost, ticket_id=659148, hostname=2620:52:6:1161:dead:beef:cafe:c155, name=vmhost_p09_03_prod_00921284_20260105_112533, requested_tags=['copr_builder', 'arch_ppc64le']': Unable to finish after 1 SSH attempts [2026-01-05 11:36:31,758][ INFO][PID:348249] Releasing VM back to pool [2026-01-05 11:36:31,766][ INFO][PID:348249] Retry #2 (on other host) [2026-01-05 11:36:31,766][ INFO][PID:348249] Checking that /var/lib/copr/public_html/results/@osbuild/image-builder/fedora-rawhide-ppc64le/repodata/repomd.xml exists [2026-01-05 11:36:31,767][ INFO][PID:348249] Checking for cancel request [2026-01-05 11:36:31,767][ INFO][PID:348249] VM allocation process starts [2026-01-05 11:36:31,826][ INFO][PID:348249] Trying to allocate VM: ResallocHost, ticket_id=659157, requested_tags=['copr_builder', 'arch_ppc64le'] [2026-01-05 11:36:34,867][ INFO][PID:348249] Allocated host ResallocHost, ticket_id=659157, hostname=2620:52:6:1161:dead:beef:cafe:c18c, name=vmhost_p09_04_prod_00921306_20260105_112657, requested_tags=['copr_builder', 'arch_ppc64le'] [2026-01-05 11:36:34,867][ INFO][PID:348249] Allocating ssh connection to builder [2026-01-05 11:36:34,868][ INFO][PID:348249] Checking that builder machine is OK [2026-01-05 11:37:34,938][ ERROR][PID:348249] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 2620:52:6:1161:dead:beef:cafe:c18c port 22: Connection timed out [2026-01-05 11:37:44,941][ ERROR][PID:348249] Re-try request for task on 'ResallocHost, ticket_id=659157, hostname=2620:52:6:1161:dead:beef:cafe:c18c, name=vmhost_p09_04_prod_00921306_20260105_112657, requested_tags=['copr_builder', 'arch_ppc64le']': Unable to finish after 1 SSH attempts [2026-01-05 11:37:44,941][ INFO][PID:348249] Releasing VM back to pool [2026-01-05 11:37:44,950][ ERROR][PID:348249] Backend process error: Three host tried without success: {'2620:52:6:1161:dead:beef:cafe:c152', '2620:52:6:1161:dead:beef:cafe:c155', '2620:52:6:1161:dead:beef:cafe:c18c'} [2026-01-05 11:37:44,951][WARNING][PID:348249] Switching not-finished job state to 'failed' [2026-01-05 11:37:44,952][ INFO][PID:348249] Worker failed build, took 658.2732272148132 [2026-01-05 11:37:44,952][ INFO][PID:348249] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "@osbuild", "project_name": "image-builder", "project_dirname": "image-builder", "submitter": "packit", "ended_on": 1767613064.9513986, "started_on": 1767612406.6781714, "submitted_on": null, "status": 0, "chroot": "fedora-rawhide-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "9975470-fedora-rawhide-ppc64le", "build_id": 9975470, "package_name": "image-builder", "package_version": "45-1.20260105112612118237.main.1.ge2a9768", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/@osbuild/image-builder/image-builder", "git_hash": "8303873e9f33ae8571d5aa3282a7ff6bbd24adc4", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "@osbuild/image-builder--packit", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/@osbuild/image-builder/fedora-rawhide-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "tags": [ "arch_ppc64le" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/@osbuild/image-builder", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/@osbuild/image-builder", "result_dir": "09975470-image-builder", "built_packages": "", "id": 9975470, "mockchain_macros": { "copr_username": "@osbuild", "copr_projectname": "image-builder", "vendor": "Fedora Project COPR (@osbuild/image-builder)" } } ] } [2026-01-05 11:37:45,019][ INFO][PID:348249] Sending fedora-messaging bus message in build.end [2026-01-05 11:37:45,050][ INFO][PID:348249] Compressing /var/lib/copr/public_html/results/@osbuild/image-builder/fedora-rawhide-ppc64le/09975470-image-builder/builder-live.log by gzip [2026-01-05 11:37:45,051][ INFO][PID:348249] Running command 'gzip /var/lib/copr/public_html/results/@osbuild/image-builder/fedora-rawhide-ppc64le/09975470-image-builder/builder-live.log' as PID 427233 [2026-01-05 11:37:45,053][ INFO][PID:348249] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/@osbuild/image-builder/fedora-rawhide-ppc64le/09975470-image-builder/builder-live.log) [2026-01-05 11:37:45,053][ INFO][PID:348249] Compressing /var/lib/copr/public_html/results/@osbuild/image-builder/fedora-rawhide-ppc64le/09975470-image-builder/backend.log by gzip [2026-01-05 11:37:45,054][ INFO][PID:348249] Running command 'gzip /var/lib/copr/public_html/results/@osbuild/image-builder/fedora-rawhide-ppc64le/09975470-image-builder/backend.log' as PID 427238