[2025-03-07 13:26:11,239][ INFO][PID:2919302] Marking build as starting [2025-03-07 13:26:11,261][ INFO][PID:2919302] Trying to allocate VM [2025-03-07 13:26:14,291][ INFO][PID:2919302] Allocated host ResallocHost, ticket_id=1694609, hostname=3.85.89.138, name=aws_x86_64_spot_prod_01753648_20250307_132123 [2025-03-07 13:26:14,292][ INFO][PID:2919302] Allocating ssh connection to builder [2025-03-07 13:26:14,292][ INFO][PID:2919302] Checking that builder machine is OK [2025-03-07 13:26:14,622][ INFO][PID:2919302] Running remote command: copr-builder-ready fedora-rawhide-x86_64 [2025-03-07 13:26:14,819][ INFO][PID:2919302] Red Hat subscription not needed for fedora-rawhide-x86_64 Builder is ready to be used [2025-03-07 13:26:14,820][ INFO][PID:2919302] Filling build.info file with builder info [2025-03-07 13:26:14,821][ INFO][PID:2919302] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "ksurma", "project_name": "sphinx-8.2.3", "project_dirname": "sphinx-8.2.3", "submitter": "https://src.fedoraproject.org/user/kkeithle", "ended_on": null, "started_on": 1741353974.8213909, "submitted_on": null, "status": 3, "chroot": "fedora-rawhide-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8739210-fedora-rawhide-x86_64", "build_id": 8739210, "package_name": "ceph", "package_version": "2:19.2.1-6", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/ksurma/sphinx-8.2.3/ceph", "git_hash": "53cb34e06d98e125082c2a791d1aea105dad1693", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "ksurma/sphinx-8.2.3--https://src.fedoraproject.org/user/kkeithle", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/ksurma/sphinx-8.2.3/fedora-rawhide-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "http://kojipkgs.fedoraproject.org/repos/rawhide/latest/$basearch/", "id": "http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch", "name": "Additional repo http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/ksurma/sphinx-8.2.3", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/ksurma/sphinx-8.2.3", "result_dir": "08739210-ceph", "built_packages": "", "id": 8739210, "mockchain_macros": { "copr_username": "ksurma", "copr_projectname": "sphinx-8.2.3", "vendor": "Fedora Project COPR (ksurma/sphinx-8.2.3)" } } ] } [2025-03-07 13:26:14,844][ INFO][PID:2919302] Sending fedora-messaging bus message in build.start [2025-03-07 13:26:15,440][ INFO][PID:2919302] Sending fedora-messaging bus message in chroot.start [2025-03-07 13:26:15,457][ INFO][PID:2919302] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8739210-fedora-rawhide-x86_64 --chroot fedora-rawhide-x86_64 --detached [2025-03-07 13:26:15,810][ INFO][PID:2919302] Downloading the builder-live.log file, attempt 1 [2025-03-07 13:26:15,819][ INFO][PID:2919302] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.85.89.138 copr-rpmbuild-log [2025-03-07 13:26:20,817][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 13:34:52,513][ ERROR][PID:2919302] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-03-07 13:35:02,514][WARNING][PID:2919302] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-03-07 13:35:12,515][ INFO][PID:2919302] Downloading the builder-live.log file, attempt 2 [2025-03-07 13:35:12,517][ INFO][PID:2919302] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.85.89.138 copr-rpmbuild-log [2025-03-07 13:36:12,585][ ERROR][PID:2919302] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-03-07 13:36:20,924][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 13:36:22,588][WARNING][PID:2919302] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-03-07 13:36:32,589][ INFO][PID:2919302] Downloading the builder-live.log file, attempt 3 [2025-03-07 13:36:32,594][ INFO][PID:2919302] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.85.89.138 copr-rpmbuild-log [2025-03-07 13:37:32,671][ ERROR][PID:2919302] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-03-07 13:37:42,672][WARNING][PID:2919302] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-03-07 13:37:52,673][ INFO][PID:2919302] Downloading the builder-live.log file, attempt 4 [2025-03-07 13:37:52,674][ INFO][PID:2919302] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.85.89.138 copr-rpmbuild-log [2025-03-07 13:38:52,737][ ERROR][PID:2919302] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-03-07 13:39:02,740][ ERROR][PID:2919302] Giving up for unstable SSH, failures: 1741354502.5147688, 1741354582.5880647, 1741354662.6725328, 1741354742.7402472 [2025-03-07 13:39:02,742][ ERROR][PID:2919302] Re-try request for task on 'ResallocHost, ticket_id=1694609, hostname=3.85.89.138, name=aws_x86_64_spot_prod_01753648_20250307_132123': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-03-07 13:39:02,742][ INFO][PID:2919302] Releasing VM back to pool [2025-03-07 13:39:02,758][ INFO][PID:2919302] Retry #1 (on other host) [2025-03-07 13:39:02,759][ INFO][PID:2919302] Trying to allocate VM [2025-03-07 13:39:05,797][ INFO][PID:2919302] Allocated host ResallocHost, ticket_id=1694806, hostname=3.237.87.247, name=aws_x86_64_spot_prod_01754103_20250307_133025 [2025-03-07 13:39:05,798][ INFO][PID:2919302] Allocating ssh connection to builder [2025-03-07 13:39:05,798][ INFO][PID:2919302] Checking that builder machine is OK [2025-03-07 13:40:05,868][ ERROR][PID:2919302] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke: OUT: ERR: ssh: connect to host 3.237.87.247 port 22: Connection timed out [2025-03-07 13:40:15,869][ ERROR][PID:2919302] Re-try request for task on 'ResallocHost, ticket_id=1694806, hostname=3.237.87.247, name=aws_x86_64_spot_prod_01754103_20250307_133025': Unable to finish after 1 SSH attempts [2025-03-07 13:40:15,870][ INFO][PID:2919302] Releasing VM back to pool [2025-03-07 13:40:15,882][ INFO][PID:2919302] Retry #2 (on other host) [2025-03-07 13:40:15,882][ INFO][PID:2919302] Trying to allocate VM [2025-03-07 13:40:18,923][ INFO][PID:2919302] Allocated host ResallocHost, ticket_id=1694820, hostname=2620:52:3:1:dead:beef:cafe:c188, name=copr_hv_x86_64_03_prod_01754404_20250307_133732 [2025-03-07 13:40:18,924][ INFO][PID:2919302] Allocating ssh connection to builder [2025-03-07 13:40:18,925][ INFO][PID:2919302] Checking that builder machine is OK [2025-03-07 13:40:19,458][ INFO][PID:2919302] Running remote command: copr-builder-ready fedora-rawhide-x86_64 [2025-03-07 13:40:19,771][ INFO][PID:2919302] Red Hat subscription not needed for fedora-rawhide-x86_64 Builder is ready to be used [2025-03-07 13:40:19,772][ INFO][PID:2919302] Filling build.info file with builder info [2025-03-07 13:40:19,776][ INFO][PID:2919302] Not re-notifying FE and msg buses for the new host. [2025-03-07 13:40:19,777][ INFO][PID:2919302] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8739210-fedora-rawhide-x86_64 --chroot fedora-rawhide-x86_64 --detached [2025-03-07 13:40:20,336][ INFO][PID:2919302] Downloading the builder-live.log file, attempt 1 [2025-03-07 13:40:20,338][ INFO][PID:2919302] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c188 copr-rpmbuild-log [2025-03-07 13:46:25,390][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 13:56:25,466][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 14:06:25,554][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 14:16:25,668][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 14:26:25,759][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 14:36:25,840][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 14:46:25,914][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 14:56:25,987][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 15:06:26,067][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 15:16:26,205][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 15:26:26,339][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 15:36:26,413][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 15:46:26,488][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 15:56:26,577][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 16:06:26,659][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 16:16:26,885][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 16:26:27,077][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 16:36:27,187][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 16:46:27,295][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 16:56:27,387][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 17:06:27,472][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 17:16:27,663][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 17:26:27,776][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 17:36:27,861][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 17:46:27,960][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 17:56:28,039][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 18:06:28,165][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 18:16:28,285][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 18:26:28,472][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 18:36:28,565][ INFO][PID:2919302] Periodic builder liveness probe: alive [2025-03-07 18:40:39,240][ INFO][PID:2919302] Downloading results from builder [2025-03-07 18:40:39,241][ INFO][PID:2919302] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c188]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/ksurma/sphinx-8.2.3/fedora-rawhide-x86_64/08739210-ceph started [2025-03-07 18:40:39,242][ INFO][PID:2919302] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c188]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/ksurma/sphinx-8.2.3/fedora-rawhide-x86_64/08739210-ceph/ &> /var/lib/copr/public_html/results/ksurma/sphinx-8.2.3/fedora-rawhide-x86_64/08739210-ceph/build-08739210.rsync.log [2025-03-07 18:40:45,258][ INFO][PID:2919302] rsyncing finished. [2025-03-07 18:40:45,259][ ERROR][PID:2919302] Failed to download data from builder due to rsync error, see the rsync log file for details. [2025-03-07 18:40:45,259][ ERROR][PID:2919302] SSH connection lost on #1 attempt, let's retry after 10s, Failed to download data from builder due to rsync error, see the rsync log file for details. [2025-03-07 18:40:55,260][ INFO][PID:2919302] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c188]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/ksurma/sphinx-8.2.3/fedora-rawhide-x86_64/08739210-ceph started [2025-03-07 18:40:55,261][ INFO][PID:2919302] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c188]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/ksurma/sphinx-8.2.3/fedora-rawhide-x86_64/08739210-ceph/ &> /var/lib/copr/public_html/results/ksurma/sphinx-8.2.3/fedora-rawhide-x86_64/08739210-ceph/build-08739210.rsync.log [2025-03-07 18:40:56,179][ INFO][PID:2919302] rsyncing finished. [2025-03-07 18:40:56,180][ INFO][PID:2919302] Releasing VM back to pool [2025-03-07 18:40:56,190][ INFO][PID:2919302] Searching for 'success' file in resultdir [2025-03-07 18:40:56,191][ ERROR][PID:2919302] Build failed: Backend process error: No success file => build failure [2025-03-07 18:40:56,191][ INFO][PID:2919302] Finished build: id=8739210 failed=True timeout=18000 destdir=/var/lib/copr/public_html/results/ksurma/sphinx-8.2.3 chroot=fedora-rawhide-x86_64 [2025-03-07 18:40:56,193][ ERROR][PID:2919302] Backend process error: No success file => build failure [2025-03-07 18:40:56,194][ INFO][PID:2919302] Worker failed build, took 18036.417734622955 [2025-03-07 18:40:56,194][ INFO][PID:2919302] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "ksurma", "project_name": "sphinx-8.2.3", "project_dirname": "sphinx-8.2.3", "submitter": "https://src.fedoraproject.org/user/kkeithle", "ended_on": 1741372856.1943536, "started_on": 1741354819.776619, "submitted_on": null, "status": 0, "chroot": "fedora-rawhide-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8739210-fedora-rawhide-x86_64", "build_id": 8739210, "package_name": "ceph", "package_version": "2:19.2.1-6", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/ksurma/sphinx-8.2.3/ceph", "git_hash": "53cb34e06d98e125082c2a791d1aea105dad1693", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "ksurma/sphinx-8.2.3--https://src.fedoraproject.org/user/kkeithle", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": 0, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/ksurma/sphinx-8.2.3/fedora-rawhide-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null }, { "baseurl": "http://kojipkgs.fedoraproject.org/repos/rawhide/latest/$basearch/", "id": "http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch", "name": "Additional repo http_kojipkgs_fedoraproject_org_repos_rawhide_latest_basearch" } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/ksurma/sphinx-8.2.3", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/ksurma/sphinx-8.2.3", "result_dir": "08739210-ceph", "built_packages": "", "id": 8739210, "mockchain_macros": { "copr_username": "ksurma", "copr_projectname": "sphinx-8.2.3", "vendor": "Fedora Project COPR (ksurma/sphinx-8.2.3)" } } ] } [2025-03-07 18:40:56,220][ INFO][PID:2919302] Sending fedora-messaging bus message in build.end [2025-03-07 18:40:56,292][ INFO][PID:2919302] Compressing /var/lib/copr/public_html/results/ksurma/sphinx-8.2.3/fedora-rawhide-x86_64/08739210-ceph/builder-live.log by gzip [2025-03-07 18:40:56,294][ INFO][PID:2919302] Running command 'gzip /var/lib/copr/public_html/results/ksurma/sphinx-8.2.3/fedora-rawhide-x86_64/08739210-ceph/builder-live.log' as PID 768075 [2025-03-07 18:40:56,566][ INFO][PID:2919302] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/ksurma/sphinx-8.2.3/fedora-rawhide-x86_64/08739210-ceph/builder-live.log) [2025-03-07 18:40:56,568][ INFO][PID:2919302] Compressing /var/lib/copr/public_html/results/ksurma/sphinx-8.2.3/fedora-rawhide-x86_64/08739210-ceph/backend.log by gzip [2025-03-07 18:40:56,572][ INFO][PID:2919302] Running command 'gzip /var/lib/copr/public_html/results/ksurma/sphinx-8.2.3/fedora-rawhide-x86_64/08739210-ceph/backend.log' as PID 768130