[2026-07-29 15:26:47,545][ INFO][PID:2124162] Marking build as starting [2026-07-29 15:26:47,640][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:26:47,649][ INFO][PID:2124162] VM allocation process starts [2026-07-29 15:26:47,689][ INFO][PID:2124162] Trying to allocate VM: ResallocHost, ticket_id=4165888, requested_tags=['arch_x86_64', 'copr_builder'] [2026-07-29 15:26:52,695][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:26:57,697][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:27:02,699][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:27:05,785][ INFO][PID:2124162] Allocated host ResallocHost, ticket_id=4165888, hostname=3.91.37.230, name=aws_x86_64_spot_prod_15089239_20260729_150850, requested_tags=['arch_x86_64', 'copr_builder'] [2026-07-29 15:27:05,787][ INFO][PID:2124162] Allocating ssh connection to builder [2026-07-29 15:27:05,795][ INFO][PID:2124162] Checking that builder machine is OK [2026-07-29 15:27:06,003][ INFO][PID:2124162] Installed copr-rpmbuild version: 1.8 [2026-07-29 15:27:06,005][ INFO][PID:2124162] Running remote command: copr-builder-ready srpm-builds [2026-07-29 15:27:06,252][ INFO][PID:2124162] Red Hat subscription not needed for srpm-builds Builder is ready to be used [2026-07-29 15:27:06,253][ INFO][PID:2124162] Filling build.info file with builder info [2026-07-29 15:27:06,254][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:27:06,256][ INFO][PID:2124162] Sending build state back to frontend: { "builds": [ { "timeout": 108000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": null, "enable_net": true, "project_owner": "ryanabx", "project_name": "cosmic-epoch", "project_dirname": "cosmic-epoch", "submitter": "ryanabx", "ended_on": null, "started_on": 1785338826.2559743, "submitted_on": null, "status": 3, "chroot": "srpm-builds", "arch": "x86_64", "buildroot_pkgs": null, "task_id": "10788741", "build_id": 10788741, "package_name": "cosmic-term", "package_version": null, "git_repo": null, "git_hash": null, "git_branch": null, "source_type": 9, "source_json": "{\"script\": \"#!/bin/bash -xe\\n\\nexport NAME=cosmic-term\\n\\nSCRIPT=cosmic-packaging-bootstrap.py\\nRPM_REPO_NAME=cosmic-packaging\\nRPM_REPO=https://forge.fedoraproject.org/cosmic/$RPM_REPO_NAME.git\\n\\ngit clone $RPM_REPO\\n\\npython3 $RPM_REPO_NAME/scripts/$SCRIPT $NAME --input $(pwd)/cosmic-packaging --output $(pwd)\\n\\nrm -rf $RPM_REPO_NAME\\n\\nls -a\\n\\ncat $NAME.spec\\n\", \"chroot\": \"fedora-44-x86_64\", \"builddeps\": \"git cargo python3\", \"resultdir\": \"\", \"repos\": \"\", \"fedora_review\": true}", "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": null, "sandbox": "ryanabx/cosmic-epoch--ryanabx", "results": null, "appstream": false, "allow_user_ssh": null, "ssh_public_keys": null, "storage": null, "repos": [], "background": false, "distributions_in_build": [ "fedora-43", "fedora-44", "fedora-rawhide" ], "distributions_in_project": [ "fedora-43", "fedora-44", "fedora-rawhide" ], "destdir": "/var/lib/copr/public_html/results/ryanabx/cosmic-epoch", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/ryanabx/cosmic-epoch", "result_dir": "10788741", "built_packages": "", "tags": [ "arch_x86_64" ], "rpmeta_prediction": null, "id": 10788741, "mockchain_macros": { "copr_username": "ryanabx", "copr_projectname": "cosmic-epoch", "vendor": "Fedora Project COPR (ryanabx/cosmic-epoch)" } } ] } [2026-07-29 15:27:06,302][ INFO][PID:2124162] Sending fedora-messaging bus message in build.start [2026-07-29 15:27:06,937][ INFO][PID:2124162] Sending fedora-messaging bus message in chroot.start [2026-07-29 15:27:06,970][ INFO][PID:2124162] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --srpm --task-url https://copr.fedorainfracloud.org/backend/get-srpm-build-task/10788741 --detached [2026-07-29 15:27:07,364][ INFO][PID:2124162] The copr-rpmbuild seems started, per: stdout: stderr: Warning: Permanently added '3.91.37.230' (ED25519) to the list of known hosts. [2026-07-29 15:27:07,366][ INFO][PID:2124162] Downloading the builder-live.log file, attempt 1 [2026-07-29 15:27:07,370][ INFO][PID:2124162] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.91.37.230 copr-rpmbuild-log [2026-07-29 15:27:12,382][ INFO][PID:2124162] Periodic builder liveness probe: alive [2026-07-29 15:27:12,384][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:27:17,387][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:27:22,390][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:27:27,398][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:27:32,411][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:27:37,413][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:27:38,583][ ERROR][PID:2124162] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2026-07-29 15:27:42,414][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:27:47,417][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:27:48,601][WARNING][PID:2124162] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2026-07-29 15:27:52,420][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:27:57,425][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:27:58,607][ INFO][PID:2124162] Downloading the builder-live.log file, attempt 2 [2026-07-29 15:27:58,609][ INFO][PID:2124162] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.91.37.230 copr-rpmbuild-log [2026-07-29 15:28:02,427][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:28:07,430][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:28:12,432][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:28:17,433][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:28:22,435][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:28:27,438][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:28:32,442][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:28:37,444][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:28:42,446][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:28:47,459][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:28:52,463][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:28:57,479][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:28:58,681][ ERROR][PID:2124162] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2026-07-29 15:29:02,489][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:29:07,491][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:29:08,697][WARNING][PID:2124162] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2026-07-29 15:29:12,494][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:29:17,495][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:29:18,698][ INFO][PID:2124162] Downloading the builder-live.log file, attempt 3 [2026-07-29 15:29:18,700][ INFO][PID:2124162] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.91.37.230 copr-rpmbuild-log [2026-07-29 15:29:22,497][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:29:27,500][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:29:32,509][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:29:37,511][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:29:42,517][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:29:47,520][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:29:52,522][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:29:57,526][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:30:02,533][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:30:07,535][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:30:12,542][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:30:17,546][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:30:18,772][ ERROR][PID:2124162] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2026-07-29 15:30:22,549][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:30:27,551][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:30:28,795][WARNING][PID:2124162] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2026-07-29 15:30:32,556][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:30:37,560][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:30:38,800][ INFO][PID:2124162] Downloading the builder-live.log file, attempt 4 [2026-07-29 15:30:38,809][ INFO][PID:2124162] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.91.37.230 copr-rpmbuild-log [2026-07-29 15:30:42,562][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:30:47,564][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:30:52,567][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:30:57,569][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:31:02,572][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:31:07,577][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:31:12,579][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:31:17,582][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:31:22,588][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:31:27,590][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:31:32,595][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:31:37,598][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:31:38,889][ ERROR][PID:2124162] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2026-07-29 15:31:42,602][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:31:47,607][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:31:48,925][ ERROR][PID:2124162] Giving up for unstable SSH, failures: 1785338868.6018243, 1785338948.6975093, 1785339028.795381, 1785339108.925045 [2026-07-29 15:31:48,936][ ERROR][PID:2124162] Re-try request for task on 'ResallocHost, ticket_id=4165888, hostname=3.91.37.230, name=aws_x86_64_spot_prod_15089239_20260729_150850, requested_tags=['arch_x86_64', 'copr_builder']': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2026-07-29 15:31:48,953][ INFO][PID:2124162] VM Release request [2026-07-29 15:31:48,967][ INFO][PID:2124162] Retry #1 (on other host) [2026-07-29 15:31:48,967][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:31:48,968][ INFO][PID:2124162] VM allocation process starts [2026-07-29 15:31:48,978][ INFO][PID:2124162] Trying to allocate VM: ResallocHost, ticket_id=4166025, requested_tags=['arch_x86_64', 'copr_builder'] [2026-07-29 15:31:51,998][ INFO][PID:2124162] Allocated host ResallocHost, ticket_id=4166025, hostname=2620:52:6:1161:dead:beef:cafe:c125, name=vmhost_x86_01_prod_15090648_20260729_151906, requested_tags=['arch_x86_64', 'copr_builder'] [2026-07-29 15:31:51,999][ INFO][PID:2124162] Allocating ssh connection to builder [2026-07-29 15:31:52,000][ INFO][PID:2124162] Checking that builder machine is OK [2026-07-29 15:31:52,610][ INFO][PID:2124162] Installed copr-rpmbuild version: 1.8 [2026-07-29 15:31:52,612][ INFO][PID:2124162] Running remote command: copr-builder-ready srpm-builds [2026-07-29 15:31:52,942][ INFO][PID:2124162] Red Hat subscription not needed for srpm-builds Builder is ready to be used [2026-07-29 15:31:52,943][ INFO][PID:2124162] Filling build.info file with builder info [2026-07-29 15:31:52,945][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:31:52,946][ INFO][PID:2124162] Not re-notifying FE and msg buses for the new host. [2026-07-29 15:31:52,947][ INFO][PID:2124162] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --srpm --task-url https://copr.fedorainfracloud.org/backend/get-srpm-build-task/10788741 --detached [2026-07-29 15:31:53,569][ INFO][PID:2124162] The copr-rpmbuild seems started, per: stdout: stderr: Warning: Permanently added '2620:52:6:1161:dead:beef:cafe:c125' (ED25519) to the list of known hosts. [2026-07-29 15:31:53,570][ INFO][PID:2124162] Downloading the builder-live.log file, attempt 1 [2026-07-29 15:31:53,572][ INFO][PID:2124162] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:6:1161:dead:beef:cafe:c125 copr-rpmbuild-log [2026-07-29 15:31:58,570][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:32:03,573][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:32:08,576][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:32:13,578][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:32:18,581][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:32:23,586][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:32:28,588][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:32:33,591][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:32:38,595][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:32:43,599][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:32:48,602][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:32:53,604][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:32:58,606][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:33:03,611][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:33:08,613][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:33:13,619][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:33:18,622][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:33:23,629][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:33:28,632][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:33:33,634][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:33:38,637][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:33:43,640][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:33:48,647][ INFO][PID:2124162] Checking for cancel request [2026-07-29 15:33:52,879][ INFO][PID:2124162] Downloading results from builder [2026-07-29 15:33:52,880][ INFO][PID:2124162] rsyncing of mockbuilder@[2620:52:6:1161:dead:beef:cafe:c125]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/ryanabx/cosmic-epoch/srpm-builds/10788741 started [2026-07-29 15:33:52,881][ INFO][PID:2124162] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:6:1161:dead:beef:cafe:c125]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/ryanabx/cosmic-epoch/srpm-builds/10788741/ &> /var/lib/copr/public_html/results/ryanabx/cosmic-epoch/srpm-builds/10788741/build-10788741.rsync.log [2026-07-29 15:33:55,084][ INFO][PID:2124162] rsyncing finished. [2026-07-29 15:33:55,085][ INFO][PID:2124162] VM Release request [2026-07-29 15:33:55,098][ INFO][PID:2124162] Searching for 'success' file in resultdir [2026-07-29 15:33:55,100][ INFO][PID:2124162] Getting build details [2026-07-29 15:33:55,101][ INFO][PID:2124162] Retrieving SRPM info from /var/lib/copr/public_html/results/ryanabx/cosmic-epoch/srpm-builds/10788741 [2026-07-29 15:33:55,102][ INFO][PID:2124162] SRPM URL: https://download.copr.fedorainfracloud.org/results/ryanabx/cosmic-epoch/srpm-builds/10788741/cosmic-term-1.5.0^git20260728.543cbb0-1.src.rpm [2026-07-29 15:33:55,103][ INFO][PID:2124162] build details: {'srpm_url': 'https://download.copr.fedorainfracloud.org/results/ryanabx/cosmic-epoch/srpm-builds/10788741/cosmic-term-1.5.0^git20260728.543cbb0-1.src.rpm', 'pkg_name': 'cosmic-term', 'pkg_version': '1.5.0^git20260728.543cbb0-1'} [2026-07-29 15:33:55,104][ INFO][PID:2124162] Finished build: id=10788741 failed=False timeout=108000 destdir=/var/lib/copr/public_html/results/ryanabx/cosmic-epoch chroot=srpm-builds [2026-07-29 15:33:55,107][ INFO][PID:2124162] Worker succeeded build, took 122.16098237037659 [2026-07-29 15:33:55,108][ INFO][PID:2124162] Sending build state back to frontend: { "builds": [ { "timeout": 108000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": null, "enable_net": true, "project_owner": "ryanabx", "project_name": "cosmic-epoch", "project_dirname": "cosmic-epoch", "submitter": "ryanabx", "ended_on": 1785339235.1074996, "started_on": 1785339112.9465172, "submitted_on": null, "status": 1, "chroot": "srpm-builds", "arch": "x86_64", "buildroot_pkgs": null, "task_id": "10788741", "build_id": 10788741, "package_name": "cosmic-term", "package_version": null, "git_repo": null, "git_hash": null, "git_branch": null, "source_type": 9, "source_json": "{\"script\": \"#!/bin/bash -xe\\n\\nexport NAME=cosmic-term\\n\\nSCRIPT=cosmic-packaging-bootstrap.py\\nRPM_REPO_NAME=cosmic-packaging\\nRPM_REPO=https://forge.fedoraproject.org/cosmic/$RPM_REPO_NAME.git\\n\\ngit clone $RPM_REPO\\n\\npython3 $RPM_REPO_NAME/scripts/$SCRIPT $NAME --input $(pwd)/cosmic-packaging --output $(pwd)\\n\\nrm -rf $RPM_REPO_NAME\\n\\nls -a\\n\\ncat $NAME.spec\\n\", \"chroot\": \"fedora-44-x86_64\", \"builddeps\": \"git cargo python3\", \"resultdir\": \"\", \"repos\": \"\", \"fedora_review\": true}", "pkg_name": "cosmic-term", "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": "https://download.copr.fedorainfracloud.org/results/ryanabx/cosmic-epoch/srpm-builds/10788741/cosmic-term-1.5.0^git20260728.543cbb0-1.src.rpm", "uses_devel_repo": null, "sandbox": "ryanabx/cosmic-epoch--ryanabx", "results": { "architecture_specific_tags": { "fedora-43": { "excludearch": [ "athlon", "geode", "i386", "i486", "i586", "i686", "pentium3", "pentium4" ] }, "fedora-44": { "excludearch": [ "athlon", "geode", "i386", "i486", "i586", "i686", "pentium3", "pentium4" ] }, "fedora-rawhide": { "excludearch": [ "athlon", "geode", "i386", "i486", "i586", "i686", "pentium3", "pentium4" ] } }, "name": "cosmic-term", "epoch": null, "version": "1.5.0^git20260728.543cbb0", "release": "1" }, "appstream": false, "allow_user_ssh": null, "ssh_public_keys": null, "storage": null, "repos": [], "background": false, "distributions_in_build": [ "fedora-43", "fedora-44", "fedora-rawhide" ], "distributions_in_project": [ "fedora-43", "fedora-44", "fedora-rawhide" ], "destdir": "/var/lib/copr/public_html/results/ryanabx/cosmic-epoch", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/ryanabx/cosmic-epoch", "result_dir": "10788741", "built_packages": "", "tags": [ "arch_x86_64" ], "rpmeta_prediction": null, "pkg_version": "1.5.0^git20260728.543cbb0-1", "id": 10788741, "mockchain_macros": { "copr_username": "ryanabx", "copr_projectname": "cosmic-epoch", "vendor": "Fedora Project COPR (ryanabx/cosmic-epoch)" } } ] } [2026-07-29 15:33:55,183][ INFO][PID:2124162] Sending fedora-messaging bus message in build.end [2026-07-29 15:33:55,216][ INFO][PID:2124162] Compressing /var/lib/copr/public_html/results/ryanabx/cosmic-epoch/srpm-builds/10788741/builder-live.log by gzip [2026-07-29 15:33:55,218][ INFO][PID:2124162] Running command 'gzip /var/lib/copr/public_html/results/ryanabx/cosmic-epoch/srpm-builds/10788741/builder-live.log' as PID 2203489 [2026-07-29 15:33:55,240][ INFO][PID:2124162] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/ryanabx/cosmic-epoch/srpm-builds/10788741/builder-live.log) [2026-07-29 15:33:55,241][ INFO][PID:2124162] Compressing /var/lib/copr/public_html/results/ryanabx/cosmic-epoch/srpm-builds/10788741/backend.log by gzip [2026-07-29 15:33:55,242][ INFO][PID:2124162] Running command 'gzip /var/lib/copr/public_html/results/ryanabx/cosmic-epoch/srpm-builds/10788741/backend.log' as PID 2203496