[2025-04-28 06:28:39,259][ INFO][PID:2618016] Marking build as starting [2025-04-28 06:28:39,355][ INFO][PID:2618016] Trying to allocate VM: ResallocHost, ticket_id=2689299, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 06:34:57,541][ INFO][PID:2618016] Allocated host ResallocHost, ticket_id=2689299, hostname=140.211.168.94, name=copr_osuosl_p09_prod_03447437_20250428_062956, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 06:34:57,542][ INFO][PID:2618016] Allocating ssh connection to builder [2025-04-28 06:34:57,544][ INFO][PID:2618016] Checking that builder machine is OK [2025-04-28 06:34:58,851][ INFO][PID:2618016] Running remote command: copr-builder-ready fedora-40-ppc64le [2025-04-28 06:34:59,877][ INFO][PID:2618016] Red Hat subscription not needed for fedora-40-ppc64le Builder is ready to be used [2025-04-28 06:34:59,878][ INFO][PID:2618016] Filling build.info file with builder info [2025-04-28 06:34:59,879][ INFO][PID:2618016] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "osloup", "project_name": "postgresql_demodularization", "project_dirname": "postgresql_demodularization", "submitter": "https://src.fedoraproject.org/user/jgrulich", "ended_on": null, "started_on": 1745822099.8797011, "submitted_on": null, "status": 3, "chroot": "fedora-40-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "8971988-fedora-40-ppc64le", "build_id": 8971988, "package_name": "qt6-qtbase", "package_version": "6.9.0-2", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/osloup/postgresql_demodularization/qt6-qtbase", "git_hash": "a06659a4dcf6c721b1c1a4d7337939ee593f7597", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "osloup/postgresql_demodularization--https://src.fedoraproject.org/user/jgrulich", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/osloup/postgresql_demodularization/fedora-40-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_ppc64le" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/osloup/postgresql_demodularization", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/osloup/postgresql_demodularization", "result_dir": "08971988-qt6-qtbase", "built_packages": "", "id": 8971988, "mockchain_macros": { "copr_username": "osloup", "copr_projectname": "postgresql_demodularization", "vendor": "Fedora Project COPR (osloup/postgresql_demodularization)" } } ] } [2025-04-28 06:34:59,921][ INFO][PID:2618016] Sending fedora-messaging bus message in build.start [2025-04-28 06:35:00,479][ INFO][PID:2618016] Sending fedora-messaging bus message in chroot.start [2025-04-28 06:35:00,498][ INFO][PID:2618016] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8971988-fedora-40-ppc64le --chroot fedora-40-ppc64le --detached [2025-04-28 06:35:01,914][ INFO][PID:2618016] Downloading the builder-live.log file, attempt 1 [2025-04-28 06:35:01,916][ INFO][PID:2618016] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.94 copr-rpmbuild-log [2025-04-28 06:35:06,920][ INFO][PID:2618016] Periodic builder liveness probe: alive [2025-04-28 06:45:07,041][ INFO][PID:2618016] Periodic builder liveness probe: alive [2025-04-28 06:51:58,139][ ERROR][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 06:52:08,140][WARNING][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 06:52:18,141][ INFO][PID:2618016] Downloading the builder-live.log file, attempt 2 [2025-04-28 06:52:18,143][ INFO][PID:2618016] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.94 copr-rpmbuild-log [2025-04-28 06:53:18,211][ ERROR][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 06:53:28,212][WARNING][PID:2618016] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 06:53:38,214][ INFO][PID:2618016] Downloading the builder-live.log file, attempt 3 [2025-04-28 06:53:38,216][ INFO][PID:2618016] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.94 copr-rpmbuild-log [2025-04-28 06:54:38,284][ ERROR][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 06:54:48,286][WARNING][PID:2618016] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 06:54:58,287][ INFO][PID:2618016] Downloading the builder-live.log file, attempt 4 [2025-04-28 06:54:58,289][ INFO][PID:2618016] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.94 copr-rpmbuild-log [2025-04-28 06:55:07,225][ INFO][PID:2618016] Periodic builder liveness probe: dead [2025-04-28 06:55:58,356][ ERROR][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 06:56:07,294][ ERROR][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.94 port 22: Connection timed out [2025-04-28 06:56:08,358][ ERROR][PID:2618016] Giving up for unstable SSH, failures: 1745823128.1406896, 1745823208.2126315, 1745823288.2859464, 1745823368.3578765 [2025-04-28 06:57:17,364][ ERROR][PID:2618016] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.94 port 22: Connection timed out [2025-04-28 06:58:27,435][ ERROR][PID:2618016] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.94 port 22: Connection timed out [2025-04-28 06:59:37,504][ ERROR][PID:2618016] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.94 port 22: Connection timed out [2025-04-28 06:59:47,505][ ERROR][PID:2618016] Can't ssh to cancel build. [2025-04-28 06:59:47,507][ ERROR][PID:2618016] Re-try request for task on 'ResallocHost, ticket_id=2689299, hostname=140.211.168.94, name=copr_osuosl_p09_prod_03447437_20250428_062956, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Resalloc ticket FAILED [2025-04-28 06:59:47,508][ INFO][PID:2618016] Releasing VM back to pool [2025-04-28 06:59:47,521][ INFO][PID:2618016] Retry #1 (on other host) [2025-04-28 06:59:47,563][ INFO][PID:2618016] Trying to allocate VM: ResallocHost, ticket_id=2689490, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 07:13:05,879][ INFO][PID:2618016] Allocated host ResallocHost, ticket_id=2689490, hostname=140.211.168.229, name=copr_osuosl_p09_prod_03448398_20250428_070805, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 07:13:05,884][ INFO][PID:2618016] Allocating ssh connection to builder [2025-04-28 07:13:05,885][ INFO][PID:2618016] Checking that builder machine is OK [2025-04-28 07:13:07,227][ INFO][PID:2618016] Running remote command: copr-builder-ready fedora-40-ppc64le [2025-04-28 07:13:08,216][ INFO][PID:2618016] Red Hat subscription not needed for fedora-40-ppc64le Builder is ready to be used [2025-04-28 07:13:08,217][ INFO][PID:2618016] Filling build.info file with builder info [2025-04-28 07:13:08,218][ INFO][PID:2618016] Not re-notifying FE and msg buses for the new host. [2025-04-28 07:13:08,219][ INFO][PID:2618016] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8971988-fedora-40-ppc64le --chroot fedora-40-ppc64le --detached [2025-04-28 07:13:09,532][ INFO][PID:2618016] Downloading the builder-live.log file, attempt 1 [2025-04-28 07:13:09,534][ INFO][PID:2618016] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.229 copr-rpmbuild-log [2025-04-28 07:13:14,551][ INFO][PID:2618016] Periodic builder liveness probe: alive [2025-04-28 07:23:14,715][ INFO][PID:2618016] Periodic builder liveness probe: alive [2025-04-28 07:32:51,143][ ERROR][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 07:33:01,144][WARNING][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 07:33:11,146][ INFO][PID:2618016] Downloading the builder-live.log file, attempt 2 [2025-04-28 07:33:11,148][ INFO][PID:2618016] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.229 copr-rpmbuild-log [2025-04-28 07:33:14,871][ INFO][PID:2618016] Periodic builder liveness probe: dead [2025-04-28 07:34:11,216][ ERROR][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 07:34:14,943][ ERROR][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.229 port 22: Connection timed out [2025-04-28 07:34:21,220][WARNING][PID:2618016] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 07:34:31,221][ INFO][PID:2618016] Downloading the builder-live.log file, attempt 3 [2025-04-28 07:34:31,222][ INFO][PID:2618016] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.229 copr-rpmbuild-log [2025-04-28 07:35:25,013][ ERROR][PID:2618016] SSH connection lost on #2 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.229 port 22: Connection timed out [2025-04-28 07:35:31,290][ ERROR][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 07:35:41,295][WARNING][PID:2618016] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 07:35:51,296][ INFO][PID:2618016] Downloading the builder-live.log file, attempt 4 [2025-04-28 07:35:51,298][ INFO][PID:2618016] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.229 copr-rpmbuild-log [2025-04-28 07:36:35,084][ ERROR][PID:2618016] SSH connection lost on #3 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.229 port 22: Connection timed out [2025-04-28 07:36:51,365][ ERROR][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 07:37:01,366][ ERROR][PID:2618016] Giving up for unstable SSH, failures: 1745825581.1445127, 1745825661.2199311, 1745825741.2954488, 1745825821.3668225 [2025-04-28 07:37:45,156][ ERROR][PID:2618016] SSH connection lost on #4 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 140.211.168.229 port 22: Connection timed out [2025-04-28 07:37:55,157][ ERROR][PID:2618016] Can't ssh to cancel build. [2025-04-28 07:37:55,158][ ERROR][PID:2618016] Re-try request for task on 'ResallocHost, ticket_id=2689490, hostname=140.211.168.229, name=copr_osuosl_p09_prod_03448398_20250428_070805, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Resalloc ticket FAILED [2025-04-28 07:37:55,159][ INFO][PID:2618016] Releasing VM back to pool [2025-04-28 07:37:55,173][ INFO][PID:2618016] Retry #2 (on other host) [2025-04-28 07:37:55,187][ INFO][PID:2618016] Trying to allocate VM: ResallocHost, ticket_id=2689838, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 08:33:14,049][ INFO][PID:2618016] Allocated host ResallocHost, ticket_id=2689838, hostname=140.211.168.9, name=copr_osuosl_p09_prod_03451158_20250428_082836, requested_tags=['copr_builder', 'arch_ppc64le'] [2025-04-28 08:33:14,057][ INFO][PID:2618016] Allocating ssh connection to builder [2025-04-28 08:33:14,058][ INFO][PID:2618016] Checking that builder machine is OK [2025-04-28 08:33:15,366][ INFO][PID:2618016] Running remote command: copr-builder-ready fedora-40-ppc64le [2025-04-28 08:33:16,315][ INFO][PID:2618016] Red Hat subscription not needed for fedora-40-ppc64le Builder is ready to be used [2025-04-28 08:33:16,317][ INFO][PID:2618016] Filling build.info file with builder info [2025-04-28 08:33:16,322][ INFO][PID:2618016] Not re-notifying FE and msg buses for the new host. [2025-04-28 08:33:16,324][ INFO][PID:2618016] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8971988-fedora-40-ppc64le --chroot fedora-40-ppc64le --detached [2025-04-28 08:33:17,597][ INFO][PID:2618016] Downloading the builder-live.log file, attempt 1 [2025-04-28 08:33:17,599][ INFO][PID:2618016] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.9 copr-rpmbuild-log [2025-04-28 08:33:22,605][ INFO][PID:2618016] Periodic builder liveness probe: alive [2025-04-28 08:43:22,733][ INFO][PID:2618016] Periodic builder liveness probe: alive [2025-04-28 08:53:22,928][ INFO][PID:2618016] Periodic builder liveness probe: alive [2025-04-28 09:03:23,428][ INFO][PID:2618016] Periodic builder liveness probe: alive [2025-04-28 09:13:23,921][ INFO][PID:2618016] Periodic builder liveness probe: alive [2025-04-28 09:23:24,381][ INFO][PID:2618016] Periodic builder liveness probe: alive [2025-04-28 09:31:00,144][ ERROR][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 09:31:10,156][WARNING][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 09:31:20,160][ INFO][PID:2618016] Downloading the builder-live.log file, attempt 2 [2025-04-28 09:31:20,169][ INFO][PID:2618016] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.9 copr-rpmbuild-log [2025-04-28 09:32:20,249][ ERROR][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 09:32:30,257][WARNING][PID:2618016] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 09:32:40,260][ INFO][PID:2618016] Downloading the builder-live.log file, attempt 3 [2025-04-28 09:32:40,264][ INFO][PID:2618016] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.9 copr-rpmbuild-log [2025-04-28 09:33:24,891][ INFO][PID:2618016] Periodic builder liveness probe: alive [2025-04-28 09:33:40,367][ ERROR][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 09:33:50,370][WARNING][PID:2618016] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-04-28 09:34:00,373][ INFO][PID:2618016] Downloading the builder-live.log file, attempt 4 [2025-04-28 09:34:00,379][ INFO][PID:2618016] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.9 copr-rpmbuild-log [2025-04-28 09:35:00,454][ ERROR][PID:2618016] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-04-28 09:35:10,460][ ERROR][PID:2618016] Giving up for unstable SSH, failures: 1745832670.156538, 1745832750.257798, 1745832830.3704436, 1745832910.460694 [2025-04-28 09:35:10,462][ ERROR][PID:2618016] Re-try request for task on 'ResallocHost, ticket_id=2689838, hostname=140.211.168.9, name=copr_osuosl_p09_prod_03451158_20250428_082836, requested_tags=['copr_builder', 'arch_ppc64le']': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-04-28 09:35:10,463][ INFO][PID:2618016] Releasing VM back to pool [2025-04-28 09:35:10,478][ ERROR][PID:2618016] Backend process error: Three host tried without success: {'140.211.168.9', '140.211.168.94', '140.211.168.229'} [2025-04-28 09:35:10,480][WARNING][PID:2618016] Switching not-finished job state to 'failed' [2025-04-28 09:35:10,481][ INFO][PID:2618016] Worker failed build, took 3714.1582012176514 [2025-04-28 09:35:10,482][ INFO][PID:2618016] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "osloup", "project_name": "postgresql_demodularization", "project_dirname": "postgresql_demodularization", "submitter": "https://src.fedoraproject.org/user/jgrulich", "ended_on": 1745832910.4802854, "started_on": 1745829196.3220842, "submitted_on": null, "status": 0, "chroot": "fedora-40-ppc64le", "arch": "ppc64le", "buildroot_pkgs": [], "task_id": "8971988-fedora-40-ppc64le", "build_id": 8971988, "package_name": "qt6-qtbase", "package_version": "6.9.0-2", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/osloup/postgresql_demodularization/qt6-qtbase", "git_hash": "a06659a4dcf6c721b1c1a4d7337939ee593f7597", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "osloup/postgresql_demodularization--https://src.fedoraproject.org/user/jgrulich", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/osloup/postgresql_demodularization/fedora-40-ppc64le/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_ppc64le" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/osloup/postgresql_demodularization", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/osloup/postgresql_demodularization", "result_dir": "08971988-qt6-qtbase", "built_packages": "", "id": 8971988, "mockchain_macros": { "copr_username": "osloup", "copr_projectname": "postgresql_demodularization", "vendor": "Fedora Project COPR (osloup/postgresql_demodularization)" } } ] } [2025-04-28 09:35:10,540][ INFO][PID:2618016] Sending fedora-messaging bus message in build.end [2025-04-28 09:35:10,560][ INFO][PID:2618016] Compressing /var/lib/copr/public_html/results/osloup/postgresql_demodularization/fedora-40-ppc64le/08971988-qt6-qtbase/builder-live.log by gzip [2025-04-28 09:35:10,562][ INFO][PID:2618016] Running command 'gzip /var/lib/copr/public_html/results/osloup/postgresql_demodularization/fedora-40-ppc64le/08971988-qt6-qtbase/builder-live.log' as PID 4050170 [2025-04-28 09:35:10,564][ INFO][PID:2618016] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/osloup/postgresql_demodularization/fedora-40-ppc64le/08971988-qt6-qtbase/builder-live.log) [2025-04-28 09:35:10,565][ INFO][PID:2618016] Compressing /var/lib/copr/public_html/results/osloup/postgresql_demodularization/fedora-40-ppc64le/08971988-qt6-qtbase/backend.log by gzip [2025-04-28 09:35:10,566][ INFO][PID:2618016] Running command 'gzip /var/lib/copr/public_html/results/osloup/postgresql_demodularization/fedora-40-ppc64le/08971988-qt6-qtbase/backend.log' as PID 4050171