[2025-02-25 09:15:36,508][ INFO][PID:2080691] Marking build as starting [2025-02-25 09:15:36,555][ INFO][PID:2080691] Trying to allocate VM [2025-02-25 09:15:42,722][ INFO][PID:2080691] Allocated host ResallocHost, ticket_id=1497557, hostname=54.242.133.73, name=aws_x86_64_spot_prod_01435399_20250225_091206 [2025-02-25 09:15:42,722][ INFO][PID:2080691] Allocating ssh connection to builder [2025-02-25 09:15:42,725][ INFO][PID:2080691] Checking that builder machine is OK [2025-02-25 09:15:43,107][ INFO][PID:2080691] Running remote command: copr-builder-ready fedora-rawhide-x86_64 [2025-02-25 09:15:43,337][ INFO][PID:2080691] Red Hat subscription not needed for fedora-rawhide-x86_64 Builder is ready to be used [2025-02-25 09:15:43,339][ INFO][PID:2080691] Filling build.info file with builder info [2025-02-25 09:15:43,341][ INFO][PID:2080691] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "lbalhar", "project_name": "sdt-devel-no-dtrace", "project_dirname": "sdt-devel-no-dtrace", "submitter": "https://src.fedoraproject.org/user/jakub", "ended_on": null, "started_on": 1740474943.341158, "submitted_on": null, "status": 3, "chroot": "fedora-rawhide-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8698062-fedora-rawhide-x86_64", "build_id": 8698062, "package_name": "gcc", "package_version": "15.0.1-0.7", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/lbalhar/sdt-devel-no-dtrace/gcc", "git_hash": "e7ba5cedc267ecfe5ed778fdcf6fe2ca06e6bb8f", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "lbalhar/sdt-devel-no-dtrace--https://src.fedoraproject.org/user/jakub", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/lbalhar/sdt-devel-no-dtrace/fedora-rawhide-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/lbalhar/sdt-devel-no-dtrace", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/lbalhar/sdt-devel-no-dtrace", "result_dir": "08698062-gcc", "built_packages": "", "id": 8698062, "mockchain_macros": { "copr_username": "lbalhar", "copr_projectname": "sdt-devel-no-dtrace", "vendor": "Fedora Project COPR (lbalhar/sdt-devel-no-dtrace)" } } ] } [2025-02-25 09:15:43,368][ INFO][PID:2080691] Sending fedora-messaging bus message in build.start [2025-02-25 09:15:45,134][ INFO][PID:2080691] Sending fedora-messaging bus message in chroot.start [2025-02-25 09:15:45,219][ INFO][PID:2080691] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8698062-fedora-rawhide-x86_64 --chroot fedora-rawhide-x86_64 --detached [2025-02-25 09:15:45,607][ INFO][PID:2080691] Downloading the builder-live.log file, attempt 1 [2025-02-25 09:15:45,609][ INFO][PID:2080691] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@54.242.133.73 copr-rpmbuild-log [2025-02-25 09:15:50,631][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 09:25:50,744][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 09:35:50,863][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 09:45:51,026][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 09:55:51,152][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 10:05:51,241][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 10:15:51,343][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 10:25:51,450][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 10:35:51,570][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 10:45:51,657][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 10:55:51,743][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 11:05:51,829][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 11:15:51,964][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 11:25:52,065][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 11:35:52,143][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 11:45:52,233][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 11:55:52,310][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 12:05:52,426][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 12:15:52,544][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 12:25:43,642][ ERROR][PID:2080691] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 12:25:52,659][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 12:25:53,643][WARNING][PID:2080691] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-02-25 12:26:03,644][ INFO][PID:2080691] Downloading the builder-live.log file, attempt 2 [2025-02-25 12:26:03,689][ INFO][PID:2080691] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@54.242.133.73 copr-rpmbuild-log [2025-02-25 12:26:03,698][ ERROR][PID:2080691] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 12:26:13,704][WARNING][PID:2080691] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-02-25 12:26:23,704][ INFO][PID:2080691] Downloading the builder-live.log file, attempt 3 [2025-02-25 12:26:23,706][ INFO][PID:2080691] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@54.242.133.73 copr-rpmbuild-log [2025-02-25 12:26:23,714][ ERROR][PID:2080691] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 12:26:33,719][WARNING][PID:2080691] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-02-25 12:26:43,720][ INFO][PID:2080691] Downloading the builder-live.log file, attempt 4 [2025-02-25 12:26:43,721][ INFO][PID:2080691] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@54.242.133.73 copr-rpmbuild-log [2025-02-25 12:27:43,790][ ERROR][PID:2080691] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 12:27:53,795][ ERROR][PID:2080691] Giving up for unstable SSH, failures: 1740486353.6434612, 1740486373.7041433, 1740486393.7192888, 1740486473.7947352 [2025-02-25 12:27:53,796][ ERROR][PID:2080691] Re-try request for task on 'ResallocHost, ticket_id=1497557, hostname=54.242.133.73, name=aws_x86_64_spot_prod_01435399_20250225_091206': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-02-25 12:27:53,796][ INFO][PID:2080691] Releasing VM back to pool [2025-02-25 12:27:53,842][ INFO][PID:2080691] Retry #1 (on other host) [2025-02-25 12:27:53,843][ INFO][PID:2080691] Trying to allocate VM [2025-02-25 12:27:56,884][ INFO][PID:2080691] Allocated host ResallocHost, ticket_id=1500006, hostname=107.21.168.213, name=aws_x86_64_spot_prod_01439323_20250225_122551 [2025-02-25 12:27:56,884][ INFO][PID:2080691] Allocating ssh connection to builder [2025-02-25 12:27:56,885][ INFO][PID:2080691] Checking that builder machine is OK [2025-02-25 12:27:57,216][ INFO][PID:2080691] Running remote command: copr-builder-ready fedora-rawhide-x86_64 [2025-02-25 12:27:57,420][ INFO][PID:2080691] Red Hat subscription not needed for fedora-rawhide-x86_64 Builder is ready to be used [2025-02-25 12:27:57,421][ INFO][PID:2080691] Filling build.info file with builder info [2025-02-25 12:27:57,429][ INFO][PID:2080691] Not re-notifying FE and msg buses for the new host. [2025-02-25 12:27:57,430][ INFO][PID:2080691] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8698062-fedora-rawhide-x86_64 --chroot fedora-rawhide-x86_64 --detached [2025-02-25 12:27:57,775][ INFO][PID:2080691] Downloading the builder-live.log file, attempt 1 [2025-02-25 12:27:57,776][ INFO][PID:2080691] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@107.21.168.213 copr-rpmbuild-log [2025-02-25 12:35:52,850][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 12:45:52,963][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 12:55:53,062][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 13:05:53,171][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 13:15:53,293][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 13:25:53,386][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 13:35:53,482][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 13:45:53,575][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 13:51:32,870][ ERROR][PID:2080691] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 13:51:42,871][WARNING][PID:2080691] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-02-25 13:51:52,872][ INFO][PID:2080691] Downloading the builder-live.log file, attempt 2 [2025-02-25 13:51:52,882][ INFO][PID:2080691] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@107.21.168.213 copr-rpmbuild-log [2025-02-25 13:52:52,953][ ERROR][PID:2080691] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 13:53:02,958][WARNING][PID:2080691] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-02-25 13:53:12,959][ INFO][PID:2080691] Downloading the builder-live.log file, attempt 3 [2025-02-25 13:53:12,961][ INFO][PID:2080691] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@107.21.168.213 copr-rpmbuild-log [2025-02-25 13:54:13,030][ ERROR][PID:2080691] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 13:54:23,041][WARNING][PID:2080691] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-02-25 13:54:33,041][ INFO][PID:2080691] Downloading the builder-live.log file, attempt 4 [2025-02-25 13:54:33,043][ INFO][PID:2080691] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@107.21.168.213 copr-rpmbuild-log [2025-02-25 13:55:33,111][ ERROR][PID:2080691] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 13:55:43,116][ ERROR][PID:2080691] Giving up for unstable SSH, failures: 1740491502.8717015, 1740491582.9586537, 1740491663.040977, 1740491743.1164281 [2025-02-25 13:55:43,117][ ERROR][PID:2080691] Re-try request for task on 'ResallocHost, ticket_id=1500006, hostname=107.21.168.213, name=aws_x86_64_spot_prod_01439323_20250225_122551': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-02-25 13:55:43,117][ INFO][PID:2080691] Releasing VM back to pool [2025-02-25 13:55:43,159][ INFO][PID:2080691] Retry #2 (on other host) [2025-02-25 13:55:43,160][ INFO][PID:2080691] Trying to allocate VM [2025-02-25 13:55:46,186][ INFO][PID:2080691] Allocated host ResallocHost, ticket_id=1501220, hostname=2620:52:3:1:dead:beef:cafe:c14a, name=copr_hv_x86_64_02_prod_01442487_20250225_135125 [2025-02-25 13:55:46,187][ INFO][PID:2080691] Allocating ssh connection to builder [2025-02-25 13:55:46,187][ INFO][PID:2080691] Checking that builder machine is OK [2025-02-25 13:55:46,802][ INFO][PID:2080691] Running remote command: copr-builder-ready fedora-rawhide-x86_64 [2025-02-25 13:55:47,131][ INFO][PID:2080691] Red Hat subscription not needed for fedora-rawhide-x86_64 Builder is ready to be used [2025-02-25 13:55:47,132][ INFO][PID:2080691] Filling build.info file with builder info [2025-02-25 13:55:47,139][ INFO][PID:2080691] Not re-notifying FE and msg buses for the new host. [2025-02-25 13:55:47,140][ INFO][PID:2080691] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8698062-fedora-rawhide-x86_64 --chroot fedora-rawhide-x86_64 --detached [2025-02-25 13:55:47,729][ INFO][PID:2080691] Downloading the builder-live.log file, attempt 1 [2025-02-25 13:55:47,730][ INFO][PID:2080691] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c14a copr-rpmbuild-log [2025-02-25 13:55:57,737][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 14:05:57,812][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 14:15:57,937][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 14:25:58,066][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 14:35:58,178][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 14:45:58,310][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 14:55:58,409][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 15:05:58,499][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 15:15:58,638][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 15:25:58,743][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 15:35:58,827][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 15:45:58,930][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 15:55:59,063][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 16:05:59,174][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 16:15:59,319][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 16:25:59,439][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 16:35:59,545][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 16:45:59,653][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 16:55:59,745][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 17:05:59,851][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 17:15:59,984][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 17:26:00,100][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 17:36:00,174][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 17:46:00,257][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 17:56:00,345][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 18:06:00,435][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 18:16:00,544][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 18:26:00,655][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 18:36:00,814][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 18:46:00,918][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 18:56:01,010][ INFO][PID:2080691] Periodic builder liveness probe: alive [2025-02-25 18:56:09,991][ INFO][PID:2080691] Downloading results from builder [2025-02-25 18:56:10,011][ INFO][PID:2080691] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c14a]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/lbalhar/sdt-devel-no-dtrace/fedora-rawhide-x86_64/08698062-gcc started [2025-02-25 18:56:10,039][ INFO][PID:2080691] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c14a]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/lbalhar/sdt-devel-no-dtrace/fedora-rawhide-x86_64/08698062-gcc/ &> /var/lib/copr/public_html/results/lbalhar/sdt-devel-no-dtrace/fedora-rawhide-x86_64/08698062-gcc/build-08698062.rsync.log [2025-02-25 18:56:14,954][ INFO][PID:2080691] rsyncing finished. [2025-02-25 18:56:14,956][ INFO][PID:2080691] Releasing VM back to pool [2025-02-25 18:56:14,971][ INFO][PID:2080691] Searching for 'success' file in resultdir [2025-02-25 18:56:14,977][ ERROR][PID:2080691] Build failed: Backend process error: No success file => build failure [2025-02-25 18:56:15,073][ INFO][PID:2080691] Finished build: id=8698062 failed=True timeout=18000 destdir=/var/lib/copr/public_html/results/lbalhar/sdt-devel-no-dtrace chroot=fedora-rawhide-x86_64 [2025-02-25 18:56:15,176][ ERROR][PID:2080691] Backend process error: No success file => build failure [2025-02-25 18:56:15,178][ INFO][PID:2080691] Worker failed build, took 18028.03871822357 [2025-02-25 18:56:15,194][ INFO][PID:2080691] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "lbalhar", "project_name": "sdt-devel-no-dtrace", "project_dirname": "sdt-devel-no-dtrace", "submitter": "https://src.fedoraproject.org/user/jakub", "ended_on": 1740509775.1783218, "started_on": 1740491747.1396036, "submitted_on": null, "status": 0, "chroot": "fedora-rawhide-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8698062-fedora-rawhide-x86_64", "build_id": 8698062, "package_name": "gcc", "package_version": "15.0.1-0.7", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/lbalhar/sdt-devel-no-dtrace/gcc", "git_hash": "e7ba5cedc267ecfe5ed778fdcf6fe2ca06e6bb8f", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "lbalhar/sdt-devel-no-dtrace--https://src.fedoraproject.org/user/jakub", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/lbalhar/sdt-devel-no-dtrace/fedora-rawhide-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/lbalhar/sdt-devel-no-dtrace", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/lbalhar/sdt-devel-no-dtrace", "result_dir": "08698062-gcc", "built_packages": "", "id": 8698062, "mockchain_macros": { "copr_username": "lbalhar", "copr_projectname": "sdt-devel-no-dtrace", "vendor": "Fedora Project COPR (lbalhar/sdt-devel-no-dtrace)" } } ] } [2025-02-25 18:56:15,855][ INFO][PID:2080691] Sending fedora-messaging bus message in build.end [2025-02-25 18:56:16,250][ INFO][PID:2080691] Compressing /var/lib/copr/public_html/results/lbalhar/sdt-devel-no-dtrace/fedora-rawhide-x86_64/08698062-gcc/builder-live.log by gzip [2025-02-25 18:56:16,278][ INFO][PID:2080691] Running command 'gzip /var/lib/copr/public_html/results/lbalhar/sdt-devel-no-dtrace/fedora-rawhide-x86_64/08698062-gcc/builder-live.log' as PID 1856226 [2025-02-25 18:56:17,014][ INFO][PID:2080691] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/lbalhar/sdt-devel-no-dtrace/fedora-rawhide-x86_64/08698062-gcc/builder-live.log) [2025-02-25 18:56:17,020][ INFO][PID:2080691] Compressing /var/lib/copr/public_html/results/lbalhar/sdt-devel-no-dtrace/fedora-rawhide-x86_64/08698062-gcc/backend.log by gzip [2025-02-25 18:56:17,022][ INFO][PID:2080691] Running command 'gzip /var/lib/copr/public_html/results/lbalhar/sdt-devel-no-dtrace/fedora-rawhide-x86_64/08698062-gcc/backend.log' as PID 1856269