[2025-04-28 06:27:36,784][ INFO][PID:2611086] Marking build as starting [2025-04-28 06:27:37,002][ INFO][PID:2611086] Trying to allocate VM: ResallocHost, ticket_id=2689278, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 06:30:55,177][ INFO][PID:2611086] Allocated host ResallocHost, ticket_id=2689278, hostname=140.211.168.199, name=copr_osuosl_p09_prod_03447310_20250428_062531, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 06:30:55,178][ INFO][PID:2611086] Allocating ssh connection to builder [2025-04-28 06:30:55,179][ INFO][PID:2611086] Checking that builder machine is OK [2025-04-28 06:30:56,588][ INFO][PID:2611086] Running remote command: copr-builder-ready fedora-42-ppc64le [2025-04-28 06:30:57,623][ INFO][PID:2611086] Red Hat subscription not needed for fedora-42-ppc64le Builder is ready to be used [2025-04-28 06:30:57,624][ INFO][PID:2611086] Filling build.info file with builder info [2025-04-28 06:30:57,627][ INFO][PID:2611086] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "osloup", "project_name": "postgresql_update_15_2", "project_dirname": "postgresql_update_15_2", "submitter": "https://src.fedoraproject.org/user/jgrulich", "ended_on": null, "started_on": 1745821857.6276665, "submitted_on": null, "status": 3, "chroot": "fedora-42-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "8971985-fedora-42-ppc64le", "build_id": 8971985, "package_name": "qt6-qtbase", "package_version": "6.9.0-2", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/osloup/postgresql_update_15_2/qt6-qtbase", "git_hash": "7b86756d2e09f075c999c756b51d512a97f82859", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "osloup/postgresql_update_15_2--https://src.fedoraproject.org/user/jgrulich", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/osloup/postgresql_update_15_2/fedora-42-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_ppc64le" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/osloup/postgresql_update_15_2", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/osloup/postgresql_update_15_2", "result_dir": "08971985-qt6-qtbase", "built_packages": "", "id": 8971985, "mockchain_macros": { "copr_username": "osloup", "copr_projectname": "postgresql_update_15_2", "vendor": "Fedora Project COPR (osloup/postgresql_update_15_2)" } } ] } [2025-04-28 06:30:57,673][ INFO][PID:2611086] Sending fedora-messaging bus message in build.start [2025-04-28 06:30:58,252][ INFO][PID:2611086] Sending fedora-messaging bus message in chroot.start [2025-04-28 06:30:58,270][ INFO][PID:2611086] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8971985-fedora-42-ppc64le --chroot fedora-42-ppc64le --detached [2025-04-28 06:30:59,709][ INFO][PID:2611086] Downloading the builder-live.log file, attempt 1 [2025-04-28 06:30:59,711][ INFO][PID:2611086] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.199 copr-rpmbuild-log [2025-04-28 06:31:04,717][ INFO][PID:2611086] Periodic builder liveness probe: alive [2025-04-28 06:41:04,853][ INFO][PID:2611086] Periodic builder liveness probe: dead [2025-04-28 06:42:05,000][ ERROR][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: Connection timed out during banner exchange Connection to 140.211.168.199 port 22 timed out [2025-04-28 06:43:15,138][ ERROR][PID:2611086] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: Connection timed out during banner exchange Connection to 140.211.168.199 port 22 timed out [2025-04-28 06:44:25,278][ ERROR][PID:2611086] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: Connection timed out during banner exchange Connection to 140.211.168.199 port 22 timed out [2025-04-28 06:45:35,349][ ERROR][PID:2611086] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.199 port 22: Connection timed out [2025-04-28 06:45:45,351][ ERROR][PID:2611086] Can't ssh to cancel build. [2025-04-28 06:45:51,140][ ERROR][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 06:46:01,141][WARNING][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 06:46:11,142][ INFO][PID:2611086] Downloading the builder-live.log file, attempt 2 [2025-04-28 06:46:11,144][ INFO][PID:2611086] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.199 copr-rpmbuild-log [2025-04-28 06:47:11,213][ ERROR][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 06:47:21,215][WARNING][PID:2611086] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 06:47:31,217][ INFO][PID:2611086] Downloading the builder-live.log file, attempt 3 [2025-04-28 06:47:31,219][ INFO][PID:2611086] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.199 copr-rpmbuild-log [2025-04-28 06:48:31,286][ ERROR][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 06:48:41,287][WARNING][PID:2611086] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 06:48:51,289][ INFO][PID:2611086] Downloading the builder-live.log file, attempt 4 [2025-04-28 06:48:51,290][ INFO][PID:2611086] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.199 copr-rpmbuild-log [2025-04-28 06:49:51,361][ ERROR][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 06:50:01,365][ ERROR][PID:2611086] Giving up for unstable SSH, failures: 1745822761.1411285, 1745822841.2154436, 1745822921.2876933, 1745823001.3653765 [2025-04-28 06:50:01,374][ ERROR][PID:2611086] Re-try request for task on 'ResallocHost, ticket_id=2689278, hostname=140.211.168.199, name=copr_osuosl_p09_prod_03447310_20250428_062531, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Resalloc ticket FAILED [2025-04-28 06:50:01,377][ INFO][PID:2611086] Releasing VM back to pool [2025-04-28 06:50:01,406][ INFO][PID:2611086] Retry #1 (on other host) [2025-04-28 06:50:01,445][ INFO][PID:2611086] Trying to allocate VM: ResallocHost, ticket_id=2689473, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 07:08:19,768][ INFO][PID:2611086] Allocated host ResallocHost, ticket_id=2689473, hostname=140.211.168.14, name=copr_osuosl_p09_prod_03448267_20250428_070346, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 07:08:19,779][ INFO][PID:2611086] Allocating ssh connection to builder [2025-04-28 07:08:19,781][ INFO][PID:2611086] Checking that builder machine is OK [2025-04-28 07:08:21,171][ INFO][PID:2611086] Running remote command: copr-builder-ready fedora-42-ppc64le [2025-04-28 07:08:22,238][ INFO][PID:2611086] Red Hat subscription not needed for fedora-42-ppc64le Builder is ready to be used [2025-04-28 07:08:22,243][ INFO][PID:2611086] Filling build.info file with builder info [2025-04-28 07:08:22,259][ INFO][PID:2611086] Not re-notifying FE and msg buses for the new host. [2025-04-28 07:08:22,262][ INFO][PID:2611086] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8971985-fedora-42-ppc64le --chroot fedora-42-ppc64le --detached [2025-04-28 07:08:23,644][ INFO][PID:2611086] Downloading the builder-live.log file, attempt 1 [2025-04-28 07:08:23,649][ INFO][PID:2611086] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.14 copr-rpmbuild-log [2025-04-28 07:08:28,672][ INFO][PID:2611086] Periodic builder liveness probe: alive [2025-04-28 07:18:28,922][ INFO][PID:2611086] Periodic builder liveness probe: alive [2025-04-28 07:27:50,143][ ERROR][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 07:28:00,145][WARNING][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 07:28:10,146][ INFO][PID:2611086] Downloading the builder-live.log file, attempt 2 [2025-04-28 07:28:10,148][ INFO][PID:2611086] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.14 copr-rpmbuild-log [2025-04-28 07:28:29,082][ INFO][PID:2611086] Periodic builder liveness probe: dead [2025-04-28 07:29:10,216][ ERROR][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 07:29:20,218][WARNING][PID:2611086] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 07:29:29,152][ ERROR][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.14 port 22: Connection timed out [2025-04-28 07:29:30,219][ INFO][PID:2611086] Downloading the builder-live.log file, attempt 3 [2025-04-28 07:29:30,220][ INFO][PID:2611086] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.14 copr-rpmbuild-log [2025-04-28 07:30:30,288][ ERROR][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 07:30:39,224][ ERROR][PID:2611086] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.14 port 22: Connection timed out [2025-04-28 07:30:40,289][WARNING][PID:2611086] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 07:30:50,291][ INFO][PID:2611086] Downloading the builder-live.log file, attempt 4 [2025-04-28 07:30:50,293][ INFO][PID:2611086] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.14 copr-rpmbuild-log [2025-04-28 07:31:49,297][ ERROR][PID:2611086] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.14 port 22: Connection timed out [2025-04-28 07:31:50,360][ ERROR][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 07:32:00,361][ ERROR][PID:2611086] Giving up for unstable SSH, failures: 1745825280.1452162, 1745825360.2181227, 1745825440.289857, 1745825520.3614786 [2025-04-28 07:32:59,368][ ERROR][PID:2611086] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.14 port 22: Connection timed out [2025-04-28 07:33:09,371][ ERROR][PID:2611086] Can't ssh to cancel build. [2025-04-28 07:33:09,372][ ERROR][PID:2611086] Re-try request for task on 'ResallocHost, ticket_id=2689473, hostname=140.211.168.14, name=copr_osuosl_p09_prod_03448267_20250428_070346, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Resalloc ticket FAILED [2025-04-28 07:33:09,373][ INFO][PID:2611086] Releasing VM back to pool [2025-04-28 07:33:09,391][ INFO][PID:2611086] Retry #2 (on other host) [2025-04-28 07:33:09,403][ INFO][PID:2611086] Trying to allocate VM: ResallocHost, ticket_id=2689833, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 08:21:28,112][ INFO][PID:2611086] Allocated host ResallocHost, ticket_id=2689833, hostname=140.211.168.243, name=copr_osuosl_p09_prod_03450610_20250428_081658, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 08:21:28,114][ INFO][PID:2611086] Allocating ssh connection to builder [2025-04-28 08:21:28,114][ INFO][PID:2611086] Checking that builder machine is OK [2025-04-28 08:21:29,385][ INFO][PID:2611086] Running remote command: copr-builder-ready fedora-42-ppc64le [2025-04-28 08:21:30,359][ INFO][PID:2611086] Red Hat subscription not needed for fedora-42-ppc64le Builder is ready to be used [2025-04-28 08:21:30,360][ INFO][PID:2611086] Filling build.info file with builder info [2025-04-28 08:21:30,365][ INFO][PID:2611086] Not re-notifying FE and msg buses for the new host. [2025-04-28 08:21:30,366][ INFO][PID:2611086] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8971985-fedora-42-ppc64le --chroot fedora-42-ppc64le --detached [2025-04-28 08:21:31,632][ INFO][PID:2611086] Downloading the builder-live.log file, attempt 1 [2025-04-28 08:21:31,634][ INFO][PID:2611086] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.243 copr-rpmbuild-log [2025-04-28 08:21:36,643][ INFO][PID:2611086] Periodic builder liveness probe: alive [2025-04-28 08:31:36,838][ INFO][PID:2611086] Periodic builder liveness probe: alive [2025-04-28 08:41:36,967][ INFO][PID:2611086] Periodic builder liveness probe: alive [2025-04-28 08:50:37,139][ ERROR][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 08:50:47,147][WARNING][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 08:50:57,149][ INFO][PID:2611086] Downloading the builder-live.log file, attempt 2 [2025-04-28 08:50:57,152][ INFO][PID:2611086] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.243 copr-rpmbuild-log [2025-04-28 08:51:37,192][ INFO][PID:2611086] Periodic builder liveness probe: dead [2025-04-28 08:51:57,221][ ERROR][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 08:52:07,228][WARNING][PID:2611086] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 08:52:17,229][ INFO][PID:2611086] Downloading the builder-live.log file, attempt 3 [2025-04-28 08:52:17,231][ INFO][PID:2611086] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.243 copr-rpmbuild-log [2025-04-28 08:52:37,287][ ERROR][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.243 port 22: Connection timed out [2025-04-28 08:53:17,300][ ERROR][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 08:53:27,303][WARNING][PID:2611086] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 08:53:37,307][ INFO][PID:2611086] Downloading the builder-live.log file, attempt 4 [2025-04-28 08:53:37,310][ INFO][PID:2611086] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.243 copr-rpmbuild-log [2025-04-28 08:53:47,374][ ERROR][PID:2611086] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.243 port 22: Connection timed out [2025-04-28 08:54:37,384][ ERROR][PID:2611086] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 08:54:47,401][ ERROR][PID:2611086] Giving up for unstable SSH, failures: 1745830247.1471872, 1745830327.2287412, 1745830407.303718, 1745830487.401099 [2025-04-28 08:54:57,501][ ERROR][PID:2611086] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.243 port 22: Connection timed out [2025-04-28 08:56:07,617][ ERROR][PID:2611086] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.243 port 22: Connection timed out [2025-04-28 08:56:17,618][ ERROR][PID:2611086] Can't ssh to cancel build. [2025-04-28 08:56:17,620][ ERROR][PID:2611086] Re-try request for task on 'ResallocHost, ticket_id=2689833, hostname=140.211.168.243, name=copr_osuosl_p09_prod_03450610_20250428_081658, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Resalloc ticket FAILED [2025-04-28 08:56:17,621][ INFO][PID:2611086] Releasing VM back to pool [2025-04-28 08:56:17,640][ ERROR][PID:2611086] Backend process error: Three host tried without success: {'140.211.168.199', '140.211.168.14', '140.211.168.243'} [2025-04-28 08:56:17,644][WARNING][PID:2611086] Switching not-finished job state to 'failed' [2025-04-28 08:56:17,647][ INFO][PID:2611086] Worker failed build, took 2087.2783575057983 [2025-04-28 08:56:17,658][ INFO][PID:2611086] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "osloup", "project_name": "postgresql_update_15_2", "project_dirname": "postgresql_update_15_2", "submitter": "https://src.fedoraproject.org/user/jgrulich", "ended_on": 1745830577.6441514, "started_on": 1745828490.365794, "submitted_on": null, "status": 0, "chroot": "fedora-42-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "8971985-fedora-42-ppc64le", "build_id": 8971985, "package_name": "qt6-qtbase", "package_version": "6.9.0-2", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/osloup/postgresql_update_15_2/qt6-qtbase", "git_hash": "7b86756d2e09f075c999c756b51d512a97f82859", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "osloup/postgresql_update_15_2--https://src.fedoraproject.org/user/jgrulich", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/osloup/postgresql_update_15_2/fedora-42-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_ppc64le" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/osloup/postgresql_update_15_2", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/osloup/postgresql_update_15_2", "result_dir": "08971985-qt6-qtbase", "built_packages": "", "id": 8971985, "mockchain_macros": { "copr_username": "osloup", "copr_projectname": "postgresql_update_15_2", "vendor": "Fedora Project COPR (osloup/postgresql_update_15_2)" } } ] } [2025-04-28 08:56:17,708][ INFO][PID:2611086] Sending fedora-messaging bus message in build.end [2025-04-28 08:56:17,732][ INFO][PID:2611086] Compressing /var/lib/copr/public_html/results/osloup/postgresql_update_15_2/fedora-42-ppc64le/08971985-qt6-qtbase/builder-live.log by gzip [2025-04-28 08:56:17,736][ INFO][PID:2611086] Running command 'gzip /var/lib/copr/public_html/results/osloup/postgresql_update_15_2/fedora-42-ppc64le/08971985-qt6-qtbase/builder-live.log' as PID 3604701 [2025-04-28 08:56:17,741][ INFO][PID:2611086] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/osloup/postgresql_update_15_2/fedora-42-ppc64le/08971985-qt6-qtbase/builder-live.log) [2025-04-28 08:56:17,743][ INFO][PID:2611086] Compressing /var/lib/copr/public_html/results/osloup/postgresql_update_15_2/fedora-42-ppc64le/08971985-qt6-qtbase/backend.log by gzip [2025-04-28 08:56:17,746][ INFO][PID:2611086] Running command 'gzip /var/lib/copr/public_html/results/osloup/postgresql_update_15_2/fedora-42-ppc64le/08971985-qt6-qtbase/backend.log' as PID 3604704