[2025-07-10 23:35:37,053][ INFO][PID:1853592] Marking build as starting [2025-07-10 23:35:37,282][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:35:37,283][ INFO][PID:1853592] VM allocation process starts [2025-07-10 23:35:37,294][ INFO][PID:1853592] Trying to allocate VM: ResallocHost, ticket_id=3886175, requested_tags=['copr_builder', 'arch_x86_64'] [2025-07-10 23:35:40,347][ INFO][PID:1853592] Allocated host ResallocHost, ticket_id=3886175, hostname=34.239.159.215, name=aws_x86_64_spot_prod_05272748_20250710_232914, requested_tags=['copr_builder', 'arch_x86_64'] [2025-07-10 23:35:40,348][ INFO][PID:1853592] Allocating ssh connection to builder [2025-07-10 23:35:40,348][ INFO][PID:1853592] Checking that builder machine is OK [2025-07-10 23:35:40,735][ INFO][PID:1853592] Running remote command: copr-builder-ready centos-stream-10-x86_64 [2025-07-10 23:35:40,939][ INFO][PID:1853592] Red Hat subscription not needed for centos-stream-10-x86_64 Builder is ready to be used [2025-07-10 23:35:40,940][ INFO][PID:1853592] Filling build.info file with builder info [2025-07-10 23:35:40,941][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:35:40,941][ INFO][PID:1853592] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "minfrin", "project_name": "minfrin-test", "project_dirname": "minfrin-test", "submitter": "minfrin", "ended_on": null, "started_on": 1752190540.9418015, "submitted_on": null, "status": 3, "chroot": "centos-stream-10-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "9264237-centos-stream-10-x86_64", "build_id": 9264237, "package_name": "httpd", "package_version": "1:2.4.63-7", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/minfrin/minfrin-test/httpd", "git_hash": "b254d65c933ccbceca0ebfd4b1ada8d026c50fdf", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "minfrin/minfrin-test--minfrin", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/minfrin/minfrin-test/centos-stream-10-x86_64/", "id": "copr_base", "module_hotfixes": true, "name": "Copr repository", "priority": null } ], "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/minfrin/minfrin-test", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/minfrin/minfrin-test", "result_dir": "09264237-httpd", "built_packages": "", "id": 9264237, "mockchain_macros": { "copr_username": "minfrin", "copr_projectname": "minfrin-test", "vendor": "Fedora Project COPR (minfrin/minfrin-test)" } } ] } [2025-07-10 23:35:41,104][ INFO][PID:1853592] Sending fedora-messaging bus message in build.start [2025-07-10 23:35:41,747][ INFO][PID:1853592] Sending fedora-messaging bus message in chroot.start [2025-07-10 23:35:41,771][ INFO][PID:1853592] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/9264237-centos-stream-10-x86_64 --chroot centos-stream-10-x86_64 --detached [2025-07-10 23:35:42,090][ INFO][PID:1853592] Downloading the builder-live.log file, attempt 1 [2025-07-10 23:35:42,092][ INFO][PID:1853592] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.239.159.215 copr-rpmbuild-log [2025-07-10 23:35:47,099][ INFO][PID:1853592] Periodic builder liveness probe: alive [2025-07-10 23:35:47,100][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:35:52,100][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:35:57,101][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:36:02,102][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:36:07,104][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:36:12,105][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:36:17,106][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:36:22,107][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:36:27,109][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:36:29,082][ ERROR][PID:1853592] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-10 23:36:32,110][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:36:37,111][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:36:39,082][WARNING][PID:1853592] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-10 23:36:42,112][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:36:47,113][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:36:49,083][ INFO][PID:1853592] Downloading the builder-live.log file, attempt 2 [2025-07-10 23:36:49,084][ INFO][PID:1853592] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.239.159.215 copr-rpmbuild-log [2025-07-10 23:36:52,114][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:36:57,115][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:37:02,117][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:37:07,118][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:37:12,119][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:37:17,120][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:37:22,123][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:37:27,123][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:37:32,125][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:37:37,125][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:37:42,127][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:37:47,128][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:37:49,152][ ERROR][PID:1853592] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-10 23:37:52,129][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:37:57,130][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:37:59,153][WARNING][PID:1853592] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-10 23:38:02,131][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:38:07,132][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:38:09,155][ INFO][PID:1853592] Downloading the builder-live.log file, attempt 3 [2025-07-10 23:38:09,156][ INFO][PID:1853592] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.239.159.215 copr-rpmbuild-log [2025-07-10 23:38:12,133][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:38:17,134][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:38:22,135][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:38:27,136][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:38:32,138][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:38:37,139][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:38:42,140][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:38:47,141][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:38:52,142][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:38:57,143][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:39:02,144][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:39:07,145][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:39:09,224][ ERROR][PID:1853592] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-10 23:39:12,146][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:39:17,147][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:39:19,227][WARNING][PID:1853592] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-07-10 23:39:22,148][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:39:27,149][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:39:29,228][ INFO][PID:1853592] Downloading the builder-live.log file, attempt 4 [2025-07-10 23:39:29,229][ INFO][PID:1853592] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.239.159.215 copr-rpmbuild-log [2025-07-10 23:39:32,150][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:39:37,151][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:39:42,152][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:39:47,153][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:39:52,154][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:39:57,155][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:40:02,156][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:40:07,157][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:40:12,158][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:40:17,159][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:40:22,160][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:40:27,161][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:40:29,284][ ERROR][PID:1853592] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-07-10 23:40:32,162][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:40:37,163][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:40:39,288][ ERROR][PID:1853592] Giving up for unstable SSH, failures: 1752190599.0827863, 1752190679.153413, 1752190759.2277818, 1752190839.2885287 [2025-07-10 23:40:39,289][ ERROR][PID:1853592] Re-try request for task on 'ResallocHost, ticket_id=3886175, hostname=34.239.159.215, name=aws_x86_64_spot_prod_05272748_20250710_232914, requested_tags=['copr_builder', 'arch_x86_64']': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-07-10 23:40:39,289][ INFO][PID:1853592] Releasing VM back to pool [2025-07-10 23:40:39,298][ INFO][PID:1853592] Retry #1 (on other host) [2025-07-10 23:40:39,299][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:40:39,299][ INFO][PID:1853592] VM allocation process starts [2025-07-10 23:40:39,367][ INFO][PID:1853592] Trying to allocate VM: ResallocHost, ticket_id=3886234, requested_tags=['copr_builder', 'arch_x86_64'] [2025-07-10 23:40:42,433][ INFO][PID:1853592] Allocated host ResallocHost, ticket_id=3886234, hostname=13.220.138.144, name=aws_x86_64_spot_prod_05272718_20250710_232720, requested_tags=['copr_builder', 'arch_x86_64'] [2025-07-10 23:40:42,434][ INFO][PID:1853592] Allocating ssh connection to builder [2025-07-10 23:40:42,434][ INFO][PID:1853592] Checking that builder machine is OK [2025-07-10 23:40:42,871][ INFO][PID:1853592] Running remote command: copr-builder-ready centos-stream-10-x86_64 [2025-07-10 23:40:43,077][ INFO][PID:1853592] Red Hat subscription not needed for centos-stream-10-x86_64 Builder is ready to be used [2025-07-10 23:40:43,078][ INFO][PID:1853592] Filling build.info file with builder info [2025-07-10 23:40:43,078][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:40:43,079][ INFO][PID:1853592] Not re-notifying FE and msg buses for the new host. [2025-07-10 23:40:43,080][ INFO][PID:1853592] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/9264237-centos-stream-10-x86_64 --chroot centos-stream-10-x86_64 --detached [2025-07-10 23:40:43,421][ INFO][PID:1853592] Downloading the builder-live.log file, attempt 1 [2025-07-10 23:40:43,422][ INFO][PID:1853592] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@13.220.138.144 copr-rpmbuild-log [2025-07-10 23:40:48,422][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:40:53,423][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:40:58,424][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:41:03,425][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:41:08,426][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:41:13,427][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:41:18,428][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:41:23,429][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:41:28,430][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:41:33,431][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:41:38,432][ INFO][PID:1853592] Checking for cancel request [2025-07-10 23:41:41,034][ INFO][PID:1853592] Downloading results from builder [2025-07-10 23:41:41,035][ INFO][PID:1853592] rsyncing of mockbuilder@13.220.138.144:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/minfrin/minfrin-test/centos-stream-10-x86_64/09264237-httpd started [2025-07-10 23:41:41,036][ INFO][PID:1853592] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@13.220.138.144:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/minfrin/minfrin-test/centos-stream-10-x86_64/09264237-httpd/ &> /var/lib/copr/public_html/results/minfrin/minfrin-test/centos-stream-10-x86_64/09264237-httpd/build-09264237.rsync.log [2025-07-10 23:41:41,315][ INFO][PID:1853592] rsyncing finished. [2025-07-10 23:41:41,315][ INFO][PID:1853592] Releasing VM back to pool [2025-07-10 23:41:41,324][ INFO][PID:1853592] Searching for 'success' file in resultdir [2025-07-10 23:41:41,325][ ERROR][PID:1853592] Build failed: Backend process error: No success file => build failure [2025-07-10 23:41:41,325][ INFO][PID:1853592] Finished build: id=9264237 failed=True timeout=18000 destdir=/var/lib/copr/public_html/results/minfrin/minfrin-test chroot=centos-stream-10-x86_64 [2025-07-10 23:41:41,327][ ERROR][PID:1853592] Backend process error: No success file => build failure [2025-07-10 23:41:41,327][ INFO][PID:1853592] Worker failed build, took 58.24830985069275 [2025-07-10 23:41:41,328][ INFO][PID:1853592] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "minfrin", "project_name": "minfrin-test", "project_dirname": "minfrin-test", "submitter": "minfrin", "ended_on": 1752190901.3279357, "started_on": 1752190843.0796258, "submitted_on": null, "status": 0, "chroot": "centos-stream-10-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "9264237-centos-stream-10-x86_64", "build_id": 9264237, "package_name": "httpd", "package_version": "1:2.4.63-7", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/minfrin/minfrin-test/httpd", "git_hash": "b254d65c933ccbceca0ebfd4b1ada8d026c50fdf", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "minfrin/minfrin-test--minfrin", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/minfrin/minfrin-test/centos-stream-10-x86_64/", "id": "copr_base", "module_hotfixes": true, "name": "Copr repository", "priority": null } ], "background": false, "fedora_review": false, "isolation": "default", "repo_priority": null, "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/minfrin/minfrin-test", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/minfrin/minfrin-test", "result_dir": "09264237-httpd", "built_packages": "", "id": 9264237, "mockchain_macros": { "copr_username": "minfrin", "copr_projectname": "minfrin-test", "vendor": "Fedora Project COPR (minfrin/minfrin-test)" } } ] } [2025-07-10 23:41:41,510][ INFO][PID:1853592] Sending fedora-messaging bus message in build.end [2025-07-10 23:41:41,534][ INFO][PID:1853592] Compressing /var/lib/copr/public_html/results/minfrin/minfrin-test/centos-stream-10-x86_64/09264237-httpd/builder-live.log by gzip [2025-07-10 23:41:41,536][ INFO][PID:1853592] Running command 'gzip /var/lib/copr/public_html/results/minfrin/minfrin-test/centos-stream-10-x86_64/09264237-httpd/builder-live.log' as PID 1881013 [2025-07-10 23:41:41,540][ INFO][PID:1853592] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/minfrin/minfrin-test/centos-stream-10-x86_64/09264237-httpd/builder-live.log) [2025-07-10 23:41:41,540][ INFO][PID:1853592] Compressing /var/lib/copr/public_html/results/minfrin/minfrin-test/centos-stream-10-x86_64/09264237-httpd/backend.log by gzip [2025-07-10 23:41:41,541][ INFO][PID:1853592] Running command 'gzip /var/lib/copr/public_html/results/minfrin/minfrin-test/centos-stream-10-x86_64/09264237-httpd/backend.log' as PID 1881014