[2025-04-28 02:27:52,434][ INFO][PID:1079746] Marking build as starting [2025-04-28 02:27:52,607][ INFO][PID:1079746] Trying to allocate VM: ResallocHost, ticket_id=2687588, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 02:35:10,809][ INFO][PID:1079746] Allocated host ResallocHost, ticket_id=2687588, hostname=140.211.168.248, name=copr_osuosl_p09_prod_03441687_20250428_023059, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 02:35:10,810][ INFO][PID:1079746] Allocating ssh connection to builder [2025-04-28 02:35:10,811][ INFO][PID:1079746] Checking that builder machine is OK [2025-04-28 02:35:12,018][ INFO][PID:1079746] Running remote command: copr-builder-ready fedora-rawhide-ppc64le [2025-04-28 02:35:13,011][ INFO][PID:1079746] Red Hat subscription not needed for fedora-rawhide-ppc64le Builder is ready to be used [2025-04-28 02:35:13,012][ INFO][PID:1079746] Filling build.info file with builder info [2025-04-28 02:35:13,014][ INFO][PID:1079746] Sending build state back to frontend: { "builds": [ { "timeout": 108000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": true, "project_owner": "@fedora-llvm-team", "project_name": "llvm-snapshots-big-merge-20250428", "project_dirname": "llvm-snapshots-big-merge-20250428", "submitter": "kkleine", "ended_on": null, "started_on": 1745807713.0138009, "submitted_on": null, "status": 3, "chroot": "fedora-rawhide-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "8971714-fedora-rawhide-ppc64le", "build_id": 8971714, "package_name": "llvm", "package_version": "20.1.3-1", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/@fedora-llvm-team/llvm-snapshots-big-merge-20250428/llvm", "git_hash": "43884570c5d3a883351308d3cb8d971a8d2756d7", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "@fedora-llvm-team/llvm-snapshots-big-merge-20250428--kkleine", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250428/fedora-rawhide-ppc64le/", "id": "copr_base", "module_hotfixes": true, "name": "Copr repository", "priority": null } ], "tags": [ "arch_ppc64le" ], "with_opts": [ "snapshot_build" ], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250428", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250428", "result_dir": "08971714-llvm", "built_packages": "", "id": 8971714, "mockchain_macros": { "copr_username": "@fedora-llvm-team", "copr_projectname": "llvm-snapshots-big-merge-20250428", "vendor": "Fedora Project COPR (@fedora-llvm-team/llvm-snapshots-big-merge-20250428)" } } ] } [2025-04-28 02:35:13,039][ INFO][PID:1079746] Sending fedora-messaging bus message in build.start [2025-04-28 02:35:13,725][ INFO][PID:1079746] Sending fedora-messaging bus message in chroot.start [2025-04-28 02:35:13,741][ INFO][PID:1079746] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8971714-fedora-rawhide-ppc64le --chroot fedora-rawhide-ppc64le --detached [2025-04-28 02:35:15,033][ INFO][PID:1079746] Downloading the builder-live.log file, attempt 1 [2025-04-28 02:35:15,035][ INFO][PID:1079746] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.248 copr-rpmbuild-log [2025-04-28 02:35:20,040][ INFO][PID:1079746] Periodic builder liveness probe: alive [2025-04-28 02:45:20,185][ INFO][PID:1079746] Periodic builder liveness probe: alive [2025-04-28 02:55:20,302][ INFO][PID:1079746] Periodic builder liveness probe: dead [2025-04-28 02:56:20,441][ ERROR][PID:1079746] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: Connection timed out during banner exchange Connection to 140.211.168.248 port 22 timed out [2025-04-28 02:57:30,580][ ERROR][PID:1079746] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: Connection timed out during banner exchange Connection to 140.211.168.248 port 22 timed out [2025-04-28 02:57:40,752][ ERROR][PID:1079746] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: kex_exchange_identification: read: Connection reset by peer Connection reset by 140.211.168.248 port 22 [2025-04-28 02:57:50,900][ ERROR][PID:1079746] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: kex_exchange_identification: read: Connection reset by peer Connection reset by 140.211.168.248 port 22 [2025-04-28 02:58:00,901][ ERROR][PID:1079746] Can't ssh to cancel build. [2025-04-28 03:00:28,134][ ERROR][PID:1079746] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 03:00:38,135][WARNING][PID:1079746] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 03:00:48,136][ INFO][PID:1079746] Downloading the builder-live.log file, attempt 2 [2025-04-28 03:00:48,138][ INFO][PID:1079746] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.248 copr-rpmbuild-log [2025-04-28 03:01:48,207][ ERROR][PID:1079746] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 03:01:58,212][WARNING][PID:1079746] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 03:02:08,214][ INFO][PID:1079746] Downloading the builder-live.log file, attempt 3 [2025-04-28 03:02:08,215][ INFO][PID:1079746] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.248 copr-rpmbuild-log [2025-04-28 03:03:08,283][ ERROR][PID:1079746] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 03:03:18,286][WARNING][PID:1079746] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 03:03:28,287][ INFO][PID:1079746] Downloading the builder-live.log file, attempt 4 [2025-04-28 03:03:28,289][ INFO][PID:1079746] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.248 copr-rpmbuild-log [2025-04-28 03:04:28,357][ ERROR][PID:1079746] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 03:04:38,360][ ERROR][PID:1079746] Giving up for unstable SSH, failures: 1745809238.1353555, 1745809318.212537, 1745809398.2868416, 1745809478.360391 [2025-04-28 03:04:38,361][ ERROR][PID:1079746] Re-try request for task on 'ResallocHost, ticket_id=2687588, hostname=140.211.168.248, name=copr_osuosl_p09_prod_03441687_20250428_023059, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Resalloc ticket FAILED [2025-04-28 03:04:38,363][ INFO][PID:1079746] Releasing VM back to pool [2025-04-28 03:04:38,375][ INFO][PID:1079746] Retry #1 (on other host) [2025-04-28 03:04:38,387][ INFO][PID:1079746] Trying to allocate VM: ResallocHost, ticket_id=2687776, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 03:08:56,508][ INFO][PID:1079746] Allocated host ResallocHost, ticket_id=2687776, hostname=140.211.168.171, name=copr_osuosl_p09_prod_03442465_20250428_030403, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 03:08:56,509][ INFO][PID:1079746] Allocating ssh connection to builder [2025-04-28 03:08:56,510][ INFO][PID:1079746] Checking that builder machine is OK [2025-04-28 03:08:57,801][ INFO][PID:1079746] Running remote command: copr-builder-ready fedora-rawhide-ppc64le [2025-04-28 03:08:58,832][ INFO][PID:1079746] Red Hat subscription not needed for fedora-rawhide-ppc64le Builder is ready to be used [2025-04-28 03:08:58,833][ INFO][PID:1079746] Filling build.info file with builder info [2025-04-28 03:08:58,841][ INFO][PID:1079746] Not re-notifying FE and msg buses for the new host. [2025-04-28 03:08:58,842][ INFO][PID:1079746] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8971714-fedora-rawhide-ppc64le --chroot fedora-rawhide-ppc64le --detached [2025-04-28 03:09:00,145][ INFO][PID:1079746] Downloading the builder-live.log file, attempt 1 [2025-04-28 03:09:00,153][ INFO][PID:1079746] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.171 copr-rpmbuild-log [2025-04-28 03:09:05,161][ INFO][PID:1079746] Periodic builder liveness probe: alive [2025-04-28 03:19:05,347][ INFO][PID:1079746] Periodic builder liveness probe: alive [2025-04-28 03:29:05,793][ INFO][PID:1079746] Periodic builder liveness probe: alive [2025-04-28 03:35:57,146][ ERROR][PID:1079746] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 03:36:07,152][WARNING][PID:1079746] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 03:36:17,163][ INFO][PID:1079746] Downloading the builder-live.log file, attempt 2 [2025-04-28 03:36:17,165][ INFO][PID:1079746] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.171 copr-rpmbuild-log [2025-04-28 03:37:17,262][ ERROR][PID:1079746] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 03:37:27,268][WARNING][PID:1079746] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 03:37:37,269][ INFO][PID:1079746] Downloading the builder-live.log file, attempt 3 [2025-04-28 03:37:37,328][ INFO][PID:1079746] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.171 copr-rpmbuild-log [2025-04-28 03:38:37,631][ ERROR][PID:1079746] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 03:38:47,632][WARNING][PID:1079746] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 03:38:57,634][ INFO][PID:1079746] Downloading the builder-live.log file, attempt 4 [2025-04-28 03:38:57,639][ INFO][PID:1079746] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.171 copr-rpmbuild-log [2025-04-28 03:39:06,135][ INFO][PID:1079746] Periodic builder liveness probe: dead [2025-04-28 03:39:57,743][ ERROR][PID:1079746] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 03:40:06,210][ ERROR][PID:1079746] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.171 port 22: Connection timed out [2025-04-28 03:40:07,750][ ERROR][PID:1079746] Giving up for unstable SSH, failures: 1745811367.1524959, 1745811447.2681804, 1745811527.632823, 1745811607.750762 [2025-04-28 03:41:16,286][ ERROR][PID:1079746] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.171 port 22: Connection timed out [2025-04-28 03:42:26,356][ ERROR][PID:1079746] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.171 port 22: Connection timed out [2025-04-28 03:43:36,426][ ERROR][PID:1079746] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.171 port 22: Connection timed out [2025-04-28 03:43:46,427][ ERROR][PID:1079746] Can't ssh to cancel build. [2025-04-28 03:43:46,428][ ERROR][PID:1079746] Re-try request for task on 'ResallocHost, ticket_id=2687776, hostname=140.211.168.171, name=copr_osuosl_p09_prod_03442465_20250428_030403, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Resalloc ticket FAILED [2025-04-28 03:43:46,429][ INFO][PID:1079746] Releasing VM back to pool [2025-04-28 03:43:46,440][ INFO][PID:1079746] Retry #2 (on other host) [2025-04-28 03:43:46,589][ INFO][PID:1079746] Trying to allocate VM: ResallocHost, ticket_id=2688186, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 03:43:49,607][ INFO][PID:1079746] Allocated host ResallocHost, ticket_id=2688186, hostname=140.211.168.190, name=copr_osuosl_p09_prod_03443299_20250428_033241, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 03:43:49,608][ INFO][PID:1079746] Allocating ssh connection to builder [2025-04-28 03:43:49,609][ INFO][PID:1079746] Checking that builder machine is OK [2025-04-28 03:43:50,909][ INFO][PID:1079746] Running remote command: copr-builder-ready fedora-rawhide-ppc64le [2025-04-28 03:43:51,879][ INFO][PID:1079746] Red Hat subscription not needed for fedora-rawhide-ppc64le Builder is ready to be used [2025-04-28 03:43:51,880][ INFO][PID:1079746] Filling build.info file with builder info [2025-04-28 03:43:51,889][ INFO][PID:1079746] Not re-notifying FE and msg buses for the new host. [2025-04-28 03:43:51,890][ INFO][PID:1079746] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8971714-fedora-rawhide-ppc64le --chroot fedora-rawhide-ppc64le --detached [2025-04-28 03:43:53,209][ INFO][PID:1079746] Downloading the builder-live.log file, attempt 1 [2025-04-28 03:43:53,210][ INFO][PID:1079746] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.190 copr-rpmbuild-log [2025-04-28 03:44:02,425][ INFO][PID:1079746] Cancel request succeeded out: success err: Warning: Permanently added '140.211.168.190' (ED25519) to the list of known hosts. [2025-04-28 03:44:02,427][ INFO][PID:1079746] Downloading results from builder [2025-04-28 03:44:02,428][ INFO][PID:1079746] rsyncing of mockbuilder@140.211.168.190:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250428/fedora-rawhide-ppc64le/08971714-llvm started [2025-04-28 03:44:02,429][ INFO][PID:1079746] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@140.211.168.190:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250428/fedora-rawhide-ppc64le/08971714-llvm/ &> /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250428/fedora-rawhide-ppc64le/08971714-llvm/build-08971714.rsync.log [2025-04-28 03:44:03,849][ INFO][PID:1079746] rsyncing finished. [2025-04-28 03:44:03,850][ INFO][PID:1079746] Releasing VM back to pool [2025-04-28 03:44:03,862][ INFO][PID:1079746] Searching for 'success' file in resultdir [2025-04-28 03:44:03,863][ ERROR][PID:1079746] Build failed: Backend process error: No success file => build failure [2025-04-28 03:44:03,864][ INFO][PID:1079746] Finished build: id=8971714 failed=True timeout=108000 destdir=/var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250428 chroot=fedora-rawhide-ppc64le [2025-04-28 03:44:03,868][ ERROR][PID:1079746] Backend process error: No success file => build failure [2025-04-28 03:44:03,869][ INFO][PID:1079746] Worker failed build, took 11.980106115341187 [2025-04-28 03:44:03,870][ INFO][PID:1079746] Sending build state back to frontend: { "builds": [ { "timeout": 108000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": true, "project_owner": "@fedora-llvm-team", "project_name": "llvm-snapshots-big-merge-20250428", "project_dirname": "llvm-snapshots-big-merge-20250428", "submitter": "kkleine", "ended_on": 1745811843.8698263, "started_on": 1745811831.8897202, "submitted_on": null, "status": 0, "chroot": "fedora-rawhide-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "8971714-fedora-rawhide-ppc64le", "build_id": 8971714, "package_name": "llvm", "package_version": "20.1.3-1", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/@fedora-llvm-team/llvm-snapshots-big-merge-20250428/llvm", "git_hash": "43884570c5d3a883351308d3cb8d971a8d2756d7", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "@fedora-llvm-team/llvm-snapshots-big-merge-20250428--kkleine", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250428/fedora-rawhide-ppc64le/", "id": "copr_base", "module_hotfixes": true, "name": "Copr repository", "priority": null } ], "tags": [ "arch_ppc64le" ], "with_opts": [ "snapshot_build" ], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250428", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250428", "result_dir": "08971714-llvm", "built_packages": "", "id": 8971714, "mockchain_macros": { "copr_username": "@fedora-llvm-team", "copr_projectname": "llvm-snapshots-big-merge-20250428", "vendor": "Fedora Project COPR (@fedora-llvm-team/llvm-snapshots-big-merge-20250428)" } } ] } [2025-04-28 03:44:03,892][ INFO][PID:1079746] Sending fedora-messaging bus message in build.end [2025-04-28 03:44:03,913][ INFO][PID:1079746] Compressing /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250428/fedora-rawhide-ppc64le/08971714-llvm/builder-live.log by gzip [2025-04-28 03:44:03,915][ INFO][PID:1079746] Running command 'gzip /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250428/fedora-rawhide-ppc64le/08971714-llvm/builder-live.log' as PID 1548150 [2025-04-28 03:44:03,916][ INFO][PID:1079746] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250428/fedora-rawhide-ppc64le/08971714-llvm/builder-live.log) [2025-04-28 03:44:03,917][ INFO][PID:1079746] Compressing /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250428/fedora-rawhide-ppc64le/08971714-llvm/backend.log by gzip [2025-04-28 03:44:03,919][ INFO][PID:1079746] Running command 'gzip /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250428/fedora-rawhide-ppc64le/08971714-llvm/backend.log' as PID 1548152