[2025-04-28 06:28:40,153][ INFO][PID:2618161] Marking build as starting [2025-04-28 06:28:40,224][ INFO][PID:2618161] Trying to allocate VM: ResallocHost, ticket_id=2689310, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 06:41:58,444][ INFO][PID:2618161] Allocated host ResallocHost, ticket_id=2689310, hostname=140.211.168.158, name=copr_osuosl_p09_prod_03447628_20250428_063706, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 06:41:58,446][ INFO][PID:2618161] Allocating ssh connection to builder [2025-04-28 06:41:58,447][ INFO][PID:2618161] Checking that builder machine is OK [2025-04-28 06:41:59,716][ INFO][PID:2618161] Running remote command: copr-builder-ready fedora-41-ppc64le [2025-04-28 06:42:00,676][ INFO][PID:2618161] Red Hat subscription not needed for fedora-41-ppc64le Builder is ready to be used [2025-04-28 06:42:00,677][ INFO][PID:2618161] Filling build.info file with builder info [2025-04-28 06:42:00,680][ INFO][PID:2618161] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "osloup", "project_name": "postgresql_update_15_2", "project_dirname": "postgresql_update_15_2", "submitter": "https://src.fedoraproject.org/user/jgrulich", "ended_on": null, "started_on": 1745822520.6798553, "submitted_on": null, "status": 3, "chroot": "fedora-41-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "8971989-fedora-41-ppc64le", "build_id": 8971989, "package_name": "qt6-qtbase", "package_version": "6.9.0-2", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/osloup/postgresql_update_15_2/qt6-qtbase", "git_hash": "b4640d6fa641282e00de0587477134402b9bc5a9", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "osloup/postgresql_update_15_2--https://src.fedoraproject.org/user/jgrulich", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/osloup/postgresql_update_15_2/fedora-41-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_ppc64le" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/osloup/postgresql_update_15_2", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/osloup/postgresql_update_15_2", "result_dir": "08971989-qt6-qtbase", "built_packages": "", "id": 8971989, "mockchain_macros": { "copr_username": "osloup", "copr_projectname": "postgresql_update_15_2", "vendor": "Fedora Project COPR (osloup/postgresql_update_15_2)" } } ] } [2025-04-28 06:42:00,717][ INFO][PID:2618161] Sending fedora-messaging bus message in build.start [2025-04-28 06:42:01,410][ INFO][PID:2618161] Sending fedora-messaging bus message in chroot.start [2025-04-28 06:42:01,429][ INFO][PID:2618161] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8971989-fedora-41-ppc64le --chroot fedora-41-ppc64le --detached [2025-04-28 06:42:02,698][ INFO][PID:2618161] Downloading the builder-live.log file, attempt 1 [2025-04-28 06:42:02,700][ INFO][PID:2618161] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.158 copr-rpmbuild-log [2025-04-28 06:42:07,706][ INFO][PID:2618161] Periodic builder liveness probe: alive [2025-04-28 06:52:07,879][ INFO][PID:2618161] Periodic builder liveness probe: alive [2025-04-28 07:02:08,001][ INFO][PID:2618161] Periodic builder liveness probe: dead [2025-04-28 07:03:08,143][ ERROR][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: Connection timed out during banner exchange Connection to 140.211.168.158 port 22 timed out [2025-04-28 07:04:18,215][ ERROR][PID:2618161] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.158 port 22: Connection timed out [2025-04-28 07:04:47,139][ ERROR][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 07:04:57,140][WARNING][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 07:05:07,141][ INFO][PID:2618161] Downloading the builder-live.log file, attempt 2 [2025-04-28 07:05:07,144][ INFO][PID:2618161] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.158 copr-rpmbuild-log [2025-04-28 07:05:28,286][ ERROR][PID:2618161] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.158 port 22: Connection timed out [2025-04-28 07:06:07,226][ ERROR][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 07:06:17,231][WARNING][PID:2618161] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 07:06:27,233][ INFO][PID:2618161] Downloading the builder-live.log file, attempt 3 [2025-04-28 07:06:27,235][ INFO][PID:2618161] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.158 copr-rpmbuild-log [2025-04-28 07:06:38,368][ ERROR][PID:2618161] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.158 port 22: Connection timed out [2025-04-28 07:06:48,370][ ERROR][PID:2618161] Can't ssh to cancel build. [2025-04-28 07:07:27,307][ ERROR][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 07:07:37,318][WARNING][PID:2618161] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 07:07:47,322][ INFO][PID:2618161] Downloading the builder-live.log file, attempt 4 [2025-04-28 07:07:47,327][ INFO][PID:2618161] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.158 copr-rpmbuild-log [2025-04-28 07:08:47,421][ ERROR][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 07:08:57,426][ ERROR][PID:2618161] Giving up for unstable SSH, failures: 1745823897.1405535, 1745823977.23132, 1745824057.3187993, 1745824137.4266288 [2025-04-28 07:08:57,431][ ERROR][PID:2618161] Re-try request for task on 'ResallocHost, ticket_id=2689310, hostname=140.211.168.158, name=copr_osuosl_p09_prod_03447628_20250428_063706, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Resalloc ticket FAILED [2025-04-28 07:08:57,433][ INFO][PID:2618161] Releasing VM back to pool [2025-04-28 07:08:57,469][ INFO][PID:2618161] Retry #1 (on other host) [2025-04-28 07:08:57,499][ INFO][PID:2618161] Trying to allocate VM: ResallocHost, ticket_id=2689637, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 07:47:16,166][ INFO][PID:2618161] Allocated host ResallocHost, ticket_id=2689637, hostname=140.211.168.5, name=copr_osuosl_p09_prod_03449253_20250428_074252, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 07:47:16,173][ INFO][PID:2618161] Allocating ssh connection to builder [2025-04-28 07:47:16,174][ INFO][PID:2618161] Checking that builder machine is OK [2025-04-28 07:47:17,499][ INFO][PID:2618161] Running remote command: copr-builder-ready fedora-41-ppc64le [2025-04-28 07:47:18,500][ INFO][PID:2618161] Red Hat subscription not needed for fedora-41-ppc64le Builder is ready to be used [2025-04-28 07:47:18,502][ INFO][PID:2618161] Filling build.info file with builder info [2025-04-28 07:47:18,505][ INFO][PID:2618161] Not re-notifying FE and msg buses for the new host. [2025-04-28 07:47:18,508][ INFO][PID:2618161] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8971989-fedora-41-ppc64le --chroot fedora-41-ppc64le --detached [2025-04-28 07:47:19,794][ INFO][PID:2618161] Downloading the builder-live.log file, attempt 1 [2025-04-28 07:47:19,796][ INFO][PID:2618161] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.5 copr-rpmbuild-log [2025-04-28 07:47:24,806][ INFO][PID:2618161] Periodic builder liveness probe: alive [2025-04-28 07:57:25,017][ INFO][PID:2618161] Periodic builder liveness probe: alive [2025-04-28 08:07:25,357][ INFO][PID:2618161] Periodic builder liveness probe: alive [2025-04-28 08:17:25,642][ INFO][PID:2618161] Periodic builder liveness probe: alive [2025-04-28 08:27:25,787][ INFO][PID:2618161] Periodic builder liveness probe: dead [2025-04-28 08:28:25,927][ ERROR][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: Connection timed out during banner exchange Connection to 140.211.168.5 port 22 timed out [2025-04-28 08:29:36,019][ ERROR][PID:2618161] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.5 port 22: Connection timed out [2025-04-28 08:30:05,139][ ERROR][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 08:30:15,143][WARNING][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 08:30:25,144][ INFO][PID:2618161] Downloading the builder-live.log file, attempt 2 [2025-04-28 08:30:25,148][ INFO][PID:2618161] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.5 copr-rpmbuild-log [2025-04-28 08:30:46,109][ ERROR][PID:2618161] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.5 port 22: Connection timed out [2025-04-28 08:31:25,217][ ERROR][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 08:31:35,218][WARNING][PID:2618161] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 08:31:45,220][ INFO][PID:2618161] Downloading the builder-live.log file, attempt 3 [2025-04-28 08:31:45,222][ INFO][PID:2618161] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.5 copr-rpmbuild-log [2025-04-28 08:31:56,181][ ERROR][PID:2618161] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.5 port 22: Connection timed out [2025-04-28 08:32:06,183][ ERROR][PID:2618161] Can't ssh to cancel build. [2025-04-28 08:32:45,290][ ERROR][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 08:32:55,295][WARNING][PID:2618161] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 08:33:05,297][ INFO][PID:2618161] Downloading the builder-live.log file, attempt 4 [2025-04-28 08:33:05,298][ INFO][PID:2618161] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.5 copr-rpmbuild-log [2025-04-28 08:34:05,366][ ERROR][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 08:34:15,367][ ERROR][PID:2618161] Giving up for unstable SSH, failures: 1745829015.1436124, 1745829095.2187645, 1745829175.2958426, 1745829255.367805 [2025-04-28 08:34:15,369][ ERROR][PID:2618161] Re-try request for task on 'ResallocHost, ticket_id=2689637, hostname=140.211.168.5, name=copr_osuosl_p09_prod_03449253_20250428_074252, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Resalloc ticket FAILED [2025-04-28 08:34:15,370][ INFO][PID:2618161] Releasing VM back to pool [2025-04-28 08:34:15,386][ INFO][PID:2618161] Retry #2 (on other host) [2025-04-28 08:34:15,402][ INFO][PID:2618161] Trying to allocate VM: ResallocHost, ticket_id=2690719, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 10:01:35,534][ INFO][PID:2618161] Allocated host ResallocHost, ticket_id=2690719, hostname=140.211.168.190, name=copr_osuosl_p09_prod_03454481_20250428_095621, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 10:01:35,543][ INFO][PID:2618161] Allocating ssh connection to builder [2025-04-28 10:01:35,544][ INFO][PID:2618161] Checking that builder machine is OK [2025-04-28 10:01:36,873][ INFO][PID:2618161] Running remote command: copr-builder-ready fedora-41-ppc64le [2025-04-28 10:01:37,898][ INFO][PID:2618161] Red Hat subscription not needed for fedora-41-ppc64le Builder is ready to be used [2025-04-28 10:01:37,900][ INFO][PID:2618161] Filling build.info file with builder info [2025-04-28 10:01:37,919][ INFO][PID:2618161] Not re-notifying FE and msg buses for the new host. [2025-04-28 10:01:37,920][ INFO][PID:2618161] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8971989-fedora-41-ppc64le --chroot fedora-41-ppc64le --detached [2025-04-28 10:01:39,249][ INFO][PID:2618161] Downloading the builder-live.log file, attempt 1 [2025-04-28 10:01:39,252][ INFO][PID:2618161] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.190 copr-rpmbuild-log [2025-04-28 10:01:44,259][ INFO][PID:2618161] Periodic builder liveness probe: alive [2025-04-28 10:11:44,613][ INFO][PID:2618161] Periodic builder liveness probe: alive [2025-04-28 10:21:45,015][ INFO][PID:2618161] Periodic builder liveness probe: alive [2025-04-28 10:31:45,507][ INFO][PID:2618161] Periodic builder liveness probe: alive [2025-04-28 10:38:53,145][ ERROR][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 10:39:03,152][WARNING][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 10:39:13,162][ INFO][PID:2618161] Downloading the builder-live.log file, attempt 2 [2025-04-28 10:39:13,166][ INFO][PID:2618161] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.190 copr-rpmbuild-log [2025-04-28 10:40:13,280][ ERROR][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 10:40:23,294][WARNING][PID:2618161] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 10:40:33,296][ INFO][PID:2618161] Downloading the builder-live.log file, attempt 3 [2025-04-28 10:40:33,301][ INFO][PID:2618161] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.190 copr-rpmbuild-log [2025-04-28 10:41:33,384][ ERROR][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 10:41:43,392][WARNING][PID:2618161] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 10:41:45,969][ INFO][PID:2618161] Periodic builder liveness probe: dead [2025-04-28 10:41:53,395][ INFO][PID:2618161] Downloading the builder-live.log file, attempt 4 [2025-04-28 10:41:53,398][ INFO][PID:2618161] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.190 copr-rpmbuild-log [2025-04-28 10:42:46,065][ ERROR][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.190 port 22: Connection timed out [2025-04-28 10:42:53,471][ ERROR][PID:2618161] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 10:43:03,480][ ERROR][PID:2618161] Giving up for unstable SSH, failures: 1745836743.1520262, 1745836823.2947998, 1745836903.3922331, 1745836983.4798992 [2025-04-28 10:43:56,145][ ERROR][PID:2618161] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.190 port 22: Connection timed out [2025-04-28 10:45:06,227][ ERROR][PID:2618161] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.190 port 22: Connection timed out [2025-04-28 10:46:16,333][ ERROR][PID:2618161] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.190 port 22: Connection timed out [2025-04-28 10:46:26,335][ ERROR][PID:2618161] Can't ssh to cancel build. [2025-04-28 10:46:26,337][ ERROR][PID:2618161] Re-try request for task on 'ResallocHost, ticket_id=2690719, hostname=140.211.168.190, name=copr_osuosl_p09_prod_03454481_20250428_095621, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Resalloc ticket FAILED [2025-04-28 10:46:26,341][ INFO][PID:2618161] Releasing VM back to pool [2025-04-28 10:46:26,365][ ERROR][PID:2618161] Backend process error: Three host tried without success: {'140.211.168.5', '140.211.168.158', '140.211.168.190'} [2025-04-28 10:46:26,366][WARNING][PID:2618161] Switching not-finished job state to 'failed' [2025-04-28 10:46:26,369][ INFO][PID:2618161] Worker failed build, took 2688.447361469269 [2025-04-28 10:46:26,372][ INFO][PID:2618161] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "osloup", "project_name": "postgresql_update_15_2", "project_dirname": "postgresql_update_15_2", "submitter": "https://src.fedoraproject.org/user/jgrulich", "ended_on": 1745837186.3666155, "started_on": 1745834497.919254, "submitted_on": null, "status": 0, "chroot": "fedora-41-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "8971989-fedora-41-ppc64le", "build_id": 8971989, "package_name": "qt6-qtbase", "package_version": "6.9.0-2", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/osloup/postgresql_update_15_2/qt6-qtbase", "git_hash": "b4640d6fa641282e00de0587477134402b9bc5a9", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "osloup/postgresql_update_15_2--https://src.fedoraproject.org/user/jgrulich", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/osloup/postgresql_update_15_2/fedora-41-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_ppc64le" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/osloup/postgresql_update_15_2", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/osloup/postgresql_update_15_2", "result_dir": "08971989-qt6-qtbase", "built_packages": "", "id": 8971989, "mockchain_macros": { "copr_username": "osloup", "copr_projectname": "postgresql_update_15_2", "vendor": "Fedora Project COPR (osloup/postgresql_update_15_2)" } } ] } [2025-04-28 10:46:26,421][ INFO][PID:2618161] Sending fedora-messaging bus message in build.end [2025-04-28 10:46:26,453][ INFO][PID:2618161] Compressing /var/lib/copr/public_html/results/osloup/postgresql_update_15_2/fedora-41-ppc64le/08971989-qt6-qtbase/builder-live.log by gzip [2025-04-28 10:46:26,457][ INFO][PID:2618161] Running command 'gzip /var/lib/copr/public_html/results/osloup/postgresql_update_15_2/fedora-41-ppc64le/08971989-qt6-qtbase/builder-live.log' as PID 538986 [2025-04-28 10:46:26,458][ INFO][PID:2618161] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/osloup/postgresql_update_15_2/fedora-41-ppc64le/08971989-qt6-qtbase/builder-live.log) [2025-04-28 10:46:26,464][ INFO][PID:2618161] Compressing /var/lib/copr/public_html/results/osloup/postgresql_update_15_2/fedora-41-ppc64le/08971989-qt6-qtbase/backend.log by gzip [2025-04-28 10:46:26,468][ INFO][PID:2618161] Running command 'gzip /var/lib/copr/public_html/results/osloup/postgresql_update_15_2/fedora-41-ppc64le/08971989-qt6-qtbase/backend.log' as PID 538990