[2025-04-26 02:18:48,168][ INFO][PID:4042093] Marking build as starting [2025-04-26 02:18:48,232][ INFO][PID:4042093] Trying to allocate VM: ResallocHost, ticket_id=2670551, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-26 02:34:06,446][ INFO][PID:4042093] Allocated host ResallocHost, ticket_id=2670551, hostname=140.211.168.89, name=copr_osuosl_p09_prod_03368791_20250426_022915, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-26 02:34:06,447][ INFO][PID:4042093] Allocating ssh connection to builder [2025-04-26 02:34:06,447][ INFO][PID:4042093] Checking that builder machine is OK [2025-04-26 02:34:07,692][ INFO][PID:4042093] Running remote command: copr-builder-ready fedora-40-ppc64le [2025-04-26 02:34:08,684][ INFO][PID:4042093] Red Hat subscription not needed for fedora-40-ppc64le Builder is ready to be used [2025-04-26 02:34:08,684][ INFO][PID:4042093] Filling build.info file with builder info [2025-04-26 02:34:08,686][ INFO][PID:4042093] Sending build state back to frontend: { "builds": [ { "timeout": 108000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": true, "project_owner": "@fedora-llvm-team", "project_name": "llvm-snapshots-big-merge-20250426", "project_dirname": "llvm-snapshots-big-merge-20250426", "submitter": "kkleine", "ended_on": null, "started_on": 1745634848.6858954, "submitted_on": null, "status": 3, "chroot": "fedora-40-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "8968517-fedora-40-ppc64le", "build_id": 8968517, "package_name": "llvm", "package_version": "20.1.3-1", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/@fedora-llvm-team/llvm-snapshots-big-merge-20250426/llvm", "git_hash": "11c1bdb37c5f58c654be4a4e65a20bcd18390763", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "@fedora-llvm-team/llvm-snapshots-big-merge-20250426--kkleine", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250426/fedora-40-ppc64le/", "id": "copr_base", "module_hotfixes": true, "name": "Copr repository", "priority": null } ], "tags": [ "arch_ppc64le" ], "with_opts": [ "snapshot_build" ], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250426", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250426", "result_dir": "08968517-llvm", "built_packages": "", "id": 8968517, "mockchain_macros": { "copr_username": "@fedora-llvm-team", "copr_projectname": "llvm-snapshots-big-merge-20250426", "vendor": "Fedora Project COPR (@fedora-llvm-team/llvm-snapshots-big-merge-20250426)" } } ] } [2025-04-26 02:34:08,707][ INFO][PID:4042093] Sending fedora-messaging bus message in build.start [2025-04-26 02:34:09,257][ INFO][PID:4042093] Sending fedora-messaging bus message in chroot.start [2025-04-26 02:34:09,273][ INFO][PID:4042093] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8968517-fedora-40-ppc64le --chroot fedora-40-ppc64le --detached [2025-04-26 02:34:10,535][ INFO][PID:4042093] Downloading the builder-live.log file, attempt 1 [2025-04-26 02:34:10,537][ INFO][PID:4042093] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.89 copr-rpmbuild-log [2025-04-26 02:34:15,541][ INFO][PID:4042093] Periodic builder liveness probe: alive [2025-04-26 02:44:15,629][ INFO][PID:4042093] Periodic builder liveness probe: alive [2025-04-26 02:54:15,716][ INFO][PID:4042093] Periodic builder liveness probe: dead [2025-04-26 02:54:39,145][ ERROR][PID:4042093] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-26 02:54:49,146][WARNING][PID:4042093] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-26 02:54:59,147][ INFO][PID:4042093] Downloading the builder-live.log file, attempt 2 [2025-04-26 02:54:59,149][ INFO][PID:4042093] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.89 copr-rpmbuild-log [2025-04-26 02:55:15,792][ ERROR][PID:4042093] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.89 port 22: Connection timed out [2025-04-26 02:55:59,220][ ERROR][PID:4042093] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-26 02:56:09,233][WARNING][PID:4042093] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-26 02:56:19,234][ INFO][PID:4042093] Downloading the builder-live.log file, attempt 3 [2025-04-26 02:56:19,236][ INFO][PID:4042093] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.89 copr-rpmbuild-log [2025-04-26 02:56:25,863][ ERROR][PID:4042093] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.89 port 22: Connection timed out [2025-04-26 02:57:19,310][ ERROR][PID:4042093] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-26 02:57:29,352][WARNING][PID:4042093] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-26 02:57:35,938][ ERROR][PID:4042093] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.89 port 22: Connection timed out [2025-04-26 02:57:39,354][ INFO][PID:4042093] Downloading the builder-live.log file, attempt 4 [2025-04-26 02:57:39,356][ INFO][PID:4042093] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.89 copr-rpmbuild-log [2025-04-26 02:58:39,425][ ERROR][PID:4042093] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-26 02:58:46,007][ ERROR][PID:4042093] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.89 port 22: Connection timed out [2025-04-26 02:58:49,431][ ERROR][PID:4042093] Giving up for unstable SSH, failures: 1745636089.1463172, 1745636169.2338152, 1745636249.3524022, 1745636329.430905 [2025-04-26 02:58:56,008][ ERROR][PID:4042093] Can't ssh to cancel build. [2025-04-26 02:58:56,009][ ERROR][PID:4042093] Re-try request for task on 'ResallocHost, ticket_id=2670551, hostname=140.211.168.89, name=copr_osuosl_p09_prod_03368791_20250426_022915, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Resalloc ticket FAILED [2025-04-26 02:58:56,010][ INFO][PID:4042093] Releasing VM back to pool [2025-04-26 02:58:56,029][ INFO][PID:4042093] Retry #1 (on other host) [2025-04-26 02:58:56,047][ INFO][PID:4042093] Trying to allocate VM: ResallocHost, ticket_id=2670763, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-26 03:24:14,380][ INFO][PID:4042093] Allocated host ResallocHost, ticket_id=2670763, hostname=140.211.168.5, name=copr_osuosl_p09_prod_03369908_20250426_031922, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-26 03:24:14,386][ INFO][PID:4042093] Allocating ssh connection to builder [2025-04-26 03:24:14,386][ INFO][PID:4042093] Checking that builder machine is OK [2025-04-26 03:24:15,629][ INFO][PID:4042093] Running remote command: copr-builder-ready fedora-40-ppc64le [2025-04-26 03:24:16,592][ INFO][PID:4042093] Red Hat subscription not needed for fedora-40-ppc64le Builder is ready to be used [2025-04-26 03:24:16,592][ INFO][PID:4042093] Filling build.info file with builder info [2025-04-26 03:24:16,690][ INFO][PID:4042093] Not re-notifying FE and msg buses for the new host. [2025-04-26 03:24:16,691][ INFO][PID:4042093] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8968517-fedora-40-ppc64le --chroot fedora-40-ppc64le --detached [2025-04-26 03:24:17,946][ INFO][PID:4042093] Downloading the builder-live.log file, attempt 1 [2025-04-26 03:24:17,947][ INFO][PID:4042093] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.5 copr-rpmbuild-log [2025-04-26 03:24:22,952][ INFO][PID:4042093] Periodic builder liveness probe: alive [2025-04-26 03:34:23,112][ INFO][PID:4042093] Periodic builder liveness probe: alive [2025-04-26 03:40:15,145][ ERROR][PID:4042093] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-26 03:40:25,146][WARNING][PID:4042093] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-26 03:40:35,147][ INFO][PID:4042093] Downloading the builder-live.log file, attempt 2 [2025-04-26 03:40:35,149][ INFO][PID:4042093] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.5 copr-rpmbuild-log [2025-04-26 03:41:35,217][ ERROR][PID:4042093] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-26 03:41:45,222][WARNING][PID:4042093] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-26 03:41:55,223][ INFO][PID:4042093] Downloading the builder-live.log file, attempt 3 [2025-04-26 03:41:55,224][ INFO][PID:4042093] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.5 copr-rpmbuild-log [2025-04-26 03:42:55,250][ ERROR][PID:4042093] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-26 03:43:05,252][WARNING][PID:4042093] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-26 03:43:15,253][ INFO][PID:4042093] Downloading the builder-live.log file, attempt 4 [2025-04-26 03:43:15,255][ INFO][PID:4042093] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.5 copr-rpmbuild-log [2025-04-26 03:44:15,323][ ERROR][PID:4042093] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-26 03:44:23,228][ INFO][PID:4042093] Periodic builder liveness probe: dead [2025-04-26 03:44:25,332][ ERROR][PID:4042093] Giving up for unstable SSH, failures: 1745638825.1465778, 1745638905.2222939, 1745638985.2524557, 1745639065.3320944 [2025-04-26 03:45:23,299][ ERROR][PID:4042093] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.5 port 22: Connection timed out [2025-04-26 03:46:33,369][ ERROR][PID:4042093] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.5 port 22: Connection timed out [2025-04-26 03:47:43,438][ ERROR][PID:4042093] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.5 port 22: Connection timed out [2025-04-26 03:48:53,508][ ERROR][PID:4042093] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.5 port 22: Connection timed out [2025-04-26 03:49:03,509][ ERROR][PID:4042093] Can't ssh to cancel build. [2025-04-26 03:49:03,510][ ERROR][PID:4042093] Re-try request for task on 'ResallocHost, ticket_id=2670763, hostname=140.211.168.5, name=copr_osuosl_p09_prod_03369908_20250426_031922, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Resalloc ticket FAILED [2025-04-26 03:49:03,511][ INFO][PID:4042093] Releasing VM back to pool [2025-04-26 03:49:03,522][ INFO][PID:4042093] Retry #2 (on other host) [2025-04-26 03:49:03,536][ INFO][PID:4042093] Trying to allocate VM: ResallocHost, ticket_id=2670948, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-26 03:52:21,637][ INFO][PID:4042093] Allocated host ResallocHost, ticket_id=2670948, hostname=140.211.168.156, name=copr_osuosl_p09_prod_03370477_20250426_034758, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-26 03:52:21,638][ INFO][PID:4042093] Allocating ssh connection to builder [2025-04-26 03:52:21,639][ INFO][PID:4042093] Checking that builder machine is OK [2025-04-26 03:52:22,918][ INFO][PID:4042093] Running remote command: copr-builder-ready fedora-40-ppc64le [2025-04-26 03:52:23,888][ INFO][PID:4042093] Red Hat subscription not needed for fedora-40-ppc64le Builder is ready to be used [2025-04-26 03:52:23,889][ INFO][PID:4042093] Filling build.info file with builder info [2025-04-26 03:52:23,900][ INFO][PID:4042093] Not re-notifying FE and msg buses for the new host. [2025-04-26 03:52:23,901][ INFO][PID:4042093] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8968517-fedora-40-ppc64le --chroot fedora-40-ppc64le --detached [2025-04-26 03:52:25,180][ INFO][PID:4042093] Downloading the builder-live.log file, attempt 1 [2025-04-26 03:52:25,182][ INFO][PID:4042093] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.156 copr-rpmbuild-log [2025-04-26 03:52:34,407][ INFO][PID:4042093] Cancel request succeeded out: success err: Warning: Permanently added '140.211.168.156' (ED25519) to the list of known hosts. [2025-04-26 03:52:34,408][ INFO][PID:4042093] Downloading results from builder [2025-04-26 03:52:34,408][ INFO][PID:4042093] rsyncing of mockbuilder@140.211.168.156:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250426/fedora-40-ppc64le/08968517-llvm started [2025-04-26 03:52:34,409][ INFO][PID:4042093] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@140.211.168.156:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250426/fedora-40-ppc64le/08968517-llvm/ &> /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250426/fedora-40-ppc64le/08968517-llvm/build-08968517.rsync.log [2025-04-26 03:52:35,762][ INFO][PID:4042093] rsyncing finished. [2025-04-26 03:52:35,763][ INFO][PID:4042093] Releasing VM back to pool [2025-04-26 03:52:35,775][ INFO][PID:4042093] Searching for 'success' file in resultdir [2025-04-26 03:52:35,776][ ERROR][PID:4042093] Build failed: Backend process error: No success file => build failure [2025-04-26 03:52:35,777][ INFO][PID:4042093] Finished build: id=8968517 failed=True timeout=108000 destdir=/var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250426 chroot=fedora-40-ppc64le [2025-04-26 03:52:35,780][ ERROR][PID:4042093] Backend process error: No success file => build failure [2025-04-26 03:52:35,781][ INFO][PID:4042093] Worker failed build, took 11.880584239959717 [2025-04-26 03:52:35,781][ INFO][PID:4042093] Sending build state back to frontend: { "builds": [ { "timeout": 108000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": true, "project_owner": "@fedora-llvm-team", "project_name": "llvm-snapshots-big-merge-20250426", "project_dirname": "llvm-snapshots-big-merge-20250426", "submitter": "kkleine", "ended_on": 1745639555.7809856, "started_on": 1745639543.9004014, "submitted_on": null, "status": 0, "chroot": "fedora-40-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "8968517-fedora-40-ppc64le", "build_id": 8968517, "package_name": "llvm", "package_version": "20.1.3-1", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/@fedora-llvm-team/llvm-snapshots-big-merge-20250426/llvm", "git_hash": "11c1bdb37c5f58c654be4a4e65a20bcd18390763", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "@fedora-llvm-team/llvm-snapshots-big-merge-20250426--kkleine", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250426/fedora-40-ppc64le/", "id": "copr_base", "module_hotfixes": true, "name": "Copr repository", "priority": null } ], "tags": [ "arch_ppc64le" ], "with_opts": [ "snapshot_build" ], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250426", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250426", "result_dir": "08968517-llvm", "built_packages": "", "id": 8968517, "mockchain_macros": { "copr_username": "@fedora-llvm-team", "copr_projectname": "llvm-snapshots-big-merge-20250426", "vendor": "Fedora Project COPR (@fedora-llvm-team/llvm-snapshots-big-merge-20250426)" } } ] } [2025-04-26 03:52:35,807][ INFO][PID:4042093] Sending fedora-messaging bus message in build.end [2025-04-26 03:52:35,827][ INFO][PID:4042093] Compressing /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250426/fedora-40-ppc64le/08968517-llvm/builder-live.log by gzip [2025-04-26 03:52:35,828][ INFO][PID:4042093] Running command 'gzip /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250426/fedora-40-ppc64le/08968517-llvm/builder-live.log' as PID 310619 [2025-04-26 03:52:35,830][ INFO][PID:4042093] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250426/fedora-40-ppc64le/08968517-llvm/builder-live.log) [2025-04-26 03:52:35,831][ INFO][PID:4042093] Compressing /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250426/fedora-40-ppc64le/08968517-llvm/backend.log by gzip [2025-04-26 03:52:35,832][ INFO][PID:4042093] Running command 'gzip /var/lib/copr/public_html/results/@fedora-llvm-team/llvm-snapshots-big-merge-20250426/fedora-40-ppc64le/08968517-llvm/backend.log' as PID 310620