[2025-03-19 12:48:54,839][ INFO][PID:2453536] Marking build as starting [2025-03-19 12:48:54,880][ INFO][PID:2453536] Trying to allocate VM [2025-03-19 12:50:43,364][ INFO][PID:2453536] Allocated host ResallocHost, ticket_id=1913165, hostname=34.228.23.189, name=aws_x86_64_spot_prod_02100798_20250319_124859 [2025-03-19 12:50:43,374][ INFO][PID:2453536] Allocating ssh connection to builder [2025-03-19 12:50:43,379][ INFO][PID:2453536] Checking that builder machine is OK [2025-03-19 12:50:43,752][ INFO][PID:2453536] Running remote command: copr-builder-ready fedora-40-x86_64 [2025-03-19 12:50:44,062][ INFO][PID:2453536] Red Hat subscription not needed for fedora-40-x86_64 Builder is ready to be used [2025-03-19 12:50:44,063][ INFO][PID:2453536] Filling build.info file with builder info [2025-03-19 12:50:44,067][ INFO][PID:2453536] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "thrnciar", "project_name": "setuptools-impact-check-control", "project_dirname": "setuptools-impact-check-control", "submitter": "https://src.fedoraproject.org/user/stransky", "ended_on": null, "started_on": 1742388644.0645485, "submitted_on": null, "status": 3, "chroot": "fedora-40-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8792684-fedora-40-x86_64", "build_id": 8792684, "package_name": "firefox", "package_version": "136.0.1-1", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/thrnciar/setuptools-impact-check-control/firefox", "git_hash": "c819b973a6abb1df566a8ee8f36fa74bc044e7b0", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "thrnciar/setuptools-impact-check-control--https://src.fedoraproject.org/user/stransky", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/thrnciar/setuptools-impact-check-control/fedora-40-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/thrnciar/setuptools-impact-check-control", "result_dir": "08792684-firefox", "built_packages": "", "id": 8792684, "mockchain_macros": { "copr_username": "thrnciar", "copr_projectname": "setuptools-impact-check-control", "vendor": "Fedora Project COPR (thrnciar/setuptools-impact-check-control)" } } ] } [2025-03-19 12:50:44,133][ INFO][PID:2453536] Sending fedora-messaging bus message in build.start [2025-03-19 12:50:45,544][ INFO][PID:2453536] Sending fedora-messaging bus message in chroot.start [2025-03-19 12:50:45,570][ INFO][PID:2453536] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8792684-fedora-40-x86_64 --chroot fedora-40-x86_64 --detached [2025-03-19 12:50:45,950][ INFO][PID:2453536] Downloading the builder-live.log file, attempt 1 [2025-03-19 12:50:45,955][ INFO][PID:2453536] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.228.23.189 copr-rpmbuild-log [2025-03-19 12:50:50,968][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 13:00:51,220][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 13:10:51,406][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 13:20:51,675][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 13:30:51,906][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 13:33:39,318][ ERROR][PID:2453536] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-03-19 13:33:49,319][WARNING][PID:2453536] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-03-19 13:33:59,321][ INFO][PID:2453536] Downloading the builder-live.log file, attempt 2 [2025-03-19 13:33:59,349][ INFO][PID:2453536] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.228.23.189 copr-rpmbuild-log [2025-03-19 13:33:59,385][ ERROR][PID:2453536] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-03-19 13:34:09,395][WARNING][PID:2453536] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-03-19 13:34:19,396][ INFO][PID:2453536] Downloading the builder-live.log file, attempt 3 [2025-03-19 13:34:19,398][ INFO][PID:2453536] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.228.23.189 copr-rpmbuild-log [2025-03-19 13:34:19,409][ ERROR][PID:2453536] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-03-19 13:34:29,412][WARNING][PID:2453536] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-03-19 13:34:39,413][ INFO][PID:2453536] Downloading the builder-live.log file, attempt 4 [2025-03-19 13:34:39,415][ INFO][PID:2453536] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.228.23.189 copr-rpmbuild-log [2025-03-19 13:35:39,493][ ERROR][PID:2453536] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-03-19 13:35:49,506][ ERROR][PID:2453536] Giving up for unstable SSH, failures: 1742391229.3194118, 1742391249.3956037, 1742391269.4127562, 1742391349.5066137 [2025-03-19 13:35:49,508][ ERROR][PID:2453536] Re-try request for task on 'ResallocHost, ticket_id=1913165, hostname=34.228.23.189, name=aws_x86_64_spot_prod_02100798_20250319_124859': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-03-19 13:35:49,509][ INFO][PID:2453536] Releasing VM back to pool [2025-03-19 13:35:49,537][ INFO][PID:2453536] Retry #1 (on other host) [2025-03-19 13:35:49,540][ INFO][PID:2453536] Trying to allocate VM [2025-03-19 13:41:07,765][ INFO][PID:2453536] Allocated host ResallocHost, ticket_id=1914470, hostname=2620:52:3:1:dead:beef:cafe:c18e, name=copr_hv_x86_64_03_prod_02102659_20250319_133852 [2025-03-19 13:41:07,787][ INFO][PID:2453536] Allocating ssh connection to builder [2025-03-19 13:41:07,787][ INFO][PID:2453536] Checking that builder machine is OK [2025-03-19 13:41:08,482][ INFO][PID:2453536] Running remote command: copr-builder-ready fedora-40-x86_64 [2025-03-19 13:41:08,876][ INFO][PID:2453536] Red Hat subscription not needed for fedora-40-x86_64 Builder is ready to be used [2025-03-19 13:41:08,878][ INFO][PID:2453536] Filling build.info file with builder info [2025-03-19 13:41:08,896][ INFO][PID:2453536] Not re-notifying FE and msg buses for the new host. [2025-03-19 13:41:08,897][ INFO][PID:2453536] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8792684-fedora-40-x86_64 --chroot fedora-40-x86_64 --detached [2025-03-19 13:41:09,611][ INFO][PID:2453536] Downloading the builder-live.log file, attempt 1 [2025-03-19 13:41:09,613][ INFO][PID:2453536] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c18e copr-rpmbuild-log [2025-03-19 13:41:14,620][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 13:51:14,911][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 14:01:15,147][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 14:11:15,366][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 14:21:15,663][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 14:31:16,018][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 14:41:16,392][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 14:51:16,586][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 15:01:16,710][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 15:11:16,859][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 15:21:17,077][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 15:31:17,201][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 15:41:17,333][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 15:51:17,468][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 16:01:17,586][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 16:11:17,751][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 16:21:18,058][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 16:31:18,205][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 16:41:18,358][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 16:51:18,438][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 17:01:18,600][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 17:11:18,810][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 17:21:19,063][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 17:31:19,194][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 17:41:19,303][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 17:51:19,472][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 18:01:19,821][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 18:11:20,082][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 18:21:20,360][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 18:31:20,612][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 18:41:20,894][ INFO][PID:2453536] Periodic builder liveness probe: alive [2025-03-19 18:41:29,595][ INFO][PID:2453536] Downloading results from builder [2025-03-19 18:41:29,618][ INFO][PID:2453536] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c18e]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-40-x86_64/08792684-firefox started [2025-03-19 18:41:29,643][ INFO][PID:2453536] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c18e]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-40-x86_64/08792684-firefox/ &> /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-40-x86_64/08792684-firefox/build-08792684.rsync.log [2025-03-19 18:41:40,818][ INFO][PID:2453536] rsyncing finished. [2025-03-19 18:41:40,822][ ERROR][PID:2453536] Failed to download data from builder due to rsync error, see the rsync log file for details. [2025-03-19 18:41:40,823][ ERROR][PID:2453536] SSH connection lost on #1 attempt, let's retry after 10s, Failed to download data from builder due to rsync error, see the rsync log file for details. [2025-03-19 18:41:50,826][ INFO][PID:2453536] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c18e]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-40-x86_64/08792684-firefox started [2025-03-19 18:41:50,831][ INFO][PID:2453536] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c18e]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-40-x86_64/08792684-firefox/ &> /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-40-x86_64/08792684-firefox/build-08792684.rsync.log [2025-03-19 18:41:52,047][ INFO][PID:2453536] rsyncing finished. [2025-03-19 18:41:52,050][ INFO][PID:2453536] Releasing VM back to pool [2025-03-19 18:41:52,115][ INFO][PID:2453536] Searching for 'success' file in resultdir [2025-03-19 18:41:52,119][ ERROR][PID:2453536] Build failed: Backend process error: No success file => build failure [2025-03-19 18:41:52,197][ INFO][PID:2453536] Finished build: id=8792684 failed=True timeout=18000 destdir=/var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control chroot=fedora-40-x86_64 [2025-03-19 18:41:52,299][ ERROR][PID:2453536] Backend process error: No success file => build failure [2025-03-19 18:41:52,303][ INFO][PID:2453536] Worker failed build, took 18043.40710759163 [2025-03-19 18:41:52,315][ INFO][PID:2453536] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "thrnciar", "project_name": "setuptools-impact-check-control", "project_dirname": "setuptools-impact-check-control", "submitter": "https://src.fedoraproject.org/user/stransky", "ended_on": 1742409712.3032231, "started_on": 1742391668.8961155, "submitted_on": null, "status": 0, "chroot": "fedora-40-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8792684-fedora-40-x86_64", "build_id": 8792684, "package_name": "firefox", "package_version": "136.0.1-1", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/thrnciar/setuptools-impact-check-control/firefox", "git_hash": "c819b973a6abb1df566a8ee8f36fa74bc044e7b0", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "thrnciar/setuptools-impact-check-control--https://src.fedoraproject.org/user/stransky", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/thrnciar/setuptools-impact-check-control/fedora-40-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/thrnciar/setuptools-impact-check-control", "result_dir": "08792684-firefox", "built_packages": "", "id": 8792684, "mockchain_macros": { "copr_username": "thrnciar", "copr_projectname": "setuptools-impact-check-control", "vendor": "Fedora Project COPR (thrnciar/setuptools-impact-check-control)" } } ] } [2025-03-19 18:41:52,841][ INFO][PID:2453536] Sending fedora-messaging bus message in build.end [2025-03-19 18:41:53,129][ INFO][PID:2453536] Compressing /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-40-x86_64/08792684-firefox/builder-live.log by gzip [2025-03-19 18:41:53,147][ INFO][PID:2453536] Running command 'gzip /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-40-x86_64/08792684-firefox/builder-live.log' as PID 1804638 [2025-03-19 18:41:54,673][ INFO][PID:2453536] Finished after 1 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-40-x86_64/08792684-firefox/builder-live.log) [2025-03-19 18:41:54,685][ INFO][PID:2453536] Compressing /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-40-x86_64/08792684-firefox/backend.log by gzip [2025-03-19 18:41:54,693][ INFO][PID:2453536] Running command 'gzip /var/lib/copr/public_html/results/thrnciar/setuptools-impact-check-control/fedora-40-x86_64/08792684-firefox/backend.log' as PID 1804833