[2025-02-25 16:02:43,024][ INFO][PID:664755] Marking build as starting [2025-02-25 16:02:43,130][ INFO][PID:664755] Trying to allocate VM [2025-02-25 16:07:01,361][ INFO][PID:664755] Allocated host ResallocHost, ticket_id=1503380, hostname=3.81.171.141, name=aws_x86_64_spot_prod_01447801_20250225_160508 [2025-02-25 16:07:01,362][ INFO][PID:664755] Allocating ssh connection to builder [2025-02-25 16:07:01,362][ INFO][PID:664755] Checking that builder machine is OK [2025-02-25 16:07:01,727][ INFO][PID:664755] Running remote command: copr-builder-ready fedora-rawhide-x86_64 [2025-02-25 16:07:01,945][ INFO][PID:664755] Red Hat subscription not needed for fedora-rawhide-x86_64 Builder is ready to be used [2025-02-25 16:07:01,947][ INFO][PID:664755] Filling build.info file with builder info [2025-02-25 16:07:01,960][ INFO][PID:664755] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "osloup", "project_name": "docbook5_style_xsl_license_and_jar_build", "project_dirname": "docbook5_style_xsl_license_and_jar_build", "submitter": "https://src.fedoraproject.org/user/jakub", "ended_on": null, "started_on": 1740499621.960049, "submitted_on": null, "status": 3, "chroot": "fedora-rawhide-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8699113-fedora-rawhide-x86_64", "build_id": 8699113, "package_name": "gcc", "package_version": "15.0.1-0.8", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/osloup/docbook5_style_xsl_license_and_jar_build/gcc", "git_hash": "9e7590ec374ba3012bd2d2b524583a3205412f66", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "osloup/docbook5_style_xsl_license_and_jar_build--https://src.fedoraproject.org/user/jakub", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/osloup/docbook5_style_xsl_license_and_jar_build/fedora-rawhide-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/osloup/docbook5_style_xsl_license_and_jar_build", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/osloup/docbook5_style_xsl_license_and_jar_build", "result_dir": "08699113-gcc", "built_packages": "", "id": 8699113, "mockchain_macros": { "copr_username": "osloup", "copr_projectname": "docbook5_style_xsl_license_and_jar_build", "vendor": "Fedora Project COPR (osloup/docbook5_style_xsl_license_and_jar_build)" } } ] } [2025-02-25 16:07:02,012][ INFO][PID:664755] Sending fedora-messaging bus message in build.start [2025-02-25 16:07:02,847][ INFO][PID:664755] Sending fedora-messaging bus message in chroot.start [2025-02-25 16:07:02,877][ INFO][PID:664755] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8699113-fedora-rawhide-x86_64 --chroot fedora-rawhide-x86_64 --detached [2025-02-25 16:07:03,265][ INFO][PID:664755] Downloading the builder-live.log file, attempt 1 [2025-02-25 16:07:03,268][ INFO][PID:664755] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.81.171.141 copr-rpmbuild-log [2025-02-25 16:07:08,279][ INFO][PID:664755] Periodic builder liveness probe: alive [2025-02-25 16:17:08,420][ INFO][PID:664755] Periodic builder liveness probe: alive [2025-02-25 16:27:08,546][ INFO][PID:664755] Periodic builder liveness probe: alive [2025-02-25 16:34:53,648][ ERROR][PID:664755] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 16:35:03,649][WARNING][PID:664755] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-02-25 16:35:13,650][ INFO][PID:664755] Downloading the builder-live.log file, attempt 2 [2025-02-25 16:35:13,657][ INFO][PID:664755] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.81.171.141 copr-rpmbuild-log [2025-02-25 16:36:13,731][ ERROR][PID:664755] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 16:36:23,735][WARNING][PID:664755] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-02-25 16:36:33,736][ INFO][PID:664755] Downloading the builder-live.log file, attempt 3 [2025-02-25 16:36:33,737][ INFO][PID:664755] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.81.171.141 copr-rpmbuild-log [2025-02-25 16:37:08,652][ INFO][PID:664755] Periodic builder liveness probe: alive [2025-02-25 16:37:33,805][ ERROR][PID:664755] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 16:37:43,806][WARNING][PID:664755] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-02-25 16:37:53,807][ INFO][PID:664755] Downloading the builder-live.log file, attempt 4 [2025-02-25 16:37:53,808][ INFO][PID:664755] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@3.81.171.141 copr-rpmbuild-log [2025-02-25 16:38:53,877][ ERROR][PID:664755] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 16:39:03,881][ ERROR][PID:664755] Giving up for unstable SSH, failures: 1740501303.6496415, 1740501383.7353911, 1740501463.8062952, 1740501543.8814712 [2025-02-25 16:39:03,882][ ERROR][PID:664755] Re-try request for task on 'ResallocHost, ticket_id=1503380, hostname=3.81.171.141, name=aws_x86_64_spot_prod_01447801_20250225_160508': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-02-25 16:39:03,883][ INFO][PID:664755] Releasing VM back to pool [2025-02-25 16:39:03,897][ INFO][PID:664755] Retry #1 (on other host) [2025-02-25 16:39:03,898][ INFO][PID:664755] Trying to allocate VM [2025-02-25 16:48:22,068][ INFO][PID:664755] Allocated host ResallocHost, ticket_id=1504013, hostname=34.229.194.103, name=aws_x86_64_spot_prod_01450091_20250225_164602 [2025-02-25 16:48:22,074][ INFO][PID:664755] Allocating ssh connection to builder [2025-02-25 16:48:22,075][ INFO][PID:664755] Checking that builder machine is OK [2025-02-25 16:48:22,396][ INFO][PID:664755] Running remote command: copr-builder-ready fedora-rawhide-x86_64 [2025-02-25 16:48:22,600][ INFO][PID:664755] Red Hat subscription not needed for fedora-rawhide-x86_64 Builder is ready to be used [2025-02-25 16:48:22,601][ INFO][PID:664755] Filling build.info file with builder info [2025-02-25 16:48:22,610][ INFO][PID:664755] Not re-notifying FE and msg buses for the new host. [2025-02-25 16:48:22,610][ INFO][PID:664755] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8699113-fedora-rawhide-x86_64 --chroot fedora-rawhide-x86_64 --detached [2025-02-25 16:48:22,986][ INFO][PID:664755] Downloading the builder-live.log file, attempt 1 [2025-02-25 16:48:22,988][ INFO][PID:664755] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.229.194.103 copr-rpmbuild-log [2025-02-25 16:48:27,997][ INFO][PID:664755] Periodic builder liveness probe: alive [2025-02-25 16:49:44,055][ ERROR][PID:664755] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 16:49:54,056][WARNING][PID:664755] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-02-25 16:50:04,056][ INFO][PID:664755] Downloading the builder-live.log file, attempt 2 [2025-02-25 16:50:04,058][ INFO][PID:664755] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.229.194.103 copr-rpmbuild-log [2025-02-25 16:51:04,126][ ERROR][PID:664755] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 16:51:14,127][WARNING][PID:664755] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-02-25 16:51:24,128][ INFO][PID:664755] Downloading the builder-live.log file, attempt 3 [2025-02-25 16:51:24,129][ INFO][PID:664755] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.229.194.103 copr-rpmbuild-log [2025-02-25 16:52:24,198][ ERROR][PID:664755] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 16:52:34,202][WARNING][PID:664755] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-02-25 16:52:44,204][ INFO][PID:664755] Downloading the builder-live.log file, attempt 4 [2025-02-25 16:52:44,205][ INFO][PID:664755] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@34.229.194.103 copr-rpmbuild-log [2025-02-25 16:53:44,274][ ERROR][PID:664755] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 16:53:54,276][ ERROR][PID:664755] Giving up for unstable SSH, failures: 1740502194.0559676, 1740502274.1271944, 1740502354.2021987, 1740502434.2759714 [2025-02-25 16:53:54,276][ ERROR][PID:664755] Re-try request for task on 'ResallocHost, ticket_id=1504013, hostname=34.229.194.103, name=aws_x86_64_spot_prod_01450091_20250225_164602': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-02-25 16:53:54,277][ INFO][PID:664755] Releasing VM back to pool [2025-02-25 16:53:54,286][ INFO][PID:664755] Retry #2 (on other host) [2025-02-25 16:53:54,287][ INFO][PID:664755] Trying to allocate VM [2025-02-25 16:55:22,457][ INFO][PID:664755] Allocated host ResallocHost, ticket_id=1504154, hostname=52.90.236.14, name=aws_x86_64_spot_prod_01450328_20250225_165343 [2025-02-25 16:55:22,458][ INFO][PID:664755] Allocating ssh connection to builder [2025-02-25 16:55:22,459][ INFO][PID:664755] Checking that builder machine is OK [2025-02-25 16:55:22,802][ INFO][PID:664755] Running remote command: copr-builder-ready fedora-rawhide-x86_64 [2025-02-25 16:55:23,003][ INFO][PID:664755] Red Hat subscription not needed for fedora-rawhide-x86_64 Builder is ready to be used [2025-02-25 16:55:23,004][ INFO][PID:664755] Filling build.info file with builder info [2025-02-25 16:55:23,005][ INFO][PID:664755] Not re-notifying FE and msg buses for the new host. [2025-02-25 16:55:23,005][ INFO][PID:664755] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8699113-fedora-rawhide-x86_64 --chroot fedora-rawhide-x86_64 --detached [2025-02-25 16:55:23,360][ INFO][PID:664755] Downloading the builder-live.log file, attempt 1 [2025-02-25 16:55:23,361][ INFO][PID:664755] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@52.90.236.14 copr-rpmbuild-log [2025-02-25 16:58:28,399][ INFO][PID:664755] Periodic builder liveness probe: alive [2025-02-25 17:07:50,680][ ERROR][PID:664755] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 17:08:00,681][WARNING][PID:664755] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-02-25 17:08:10,682][ INFO][PID:664755] Downloading the builder-live.log file, attempt 2 [2025-02-25 17:08:10,687][ INFO][PID:664755] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@52.90.236.14 copr-rpmbuild-log [2025-02-25 17:08:28,514][ INFO][PID:664755] Periodic builder liveness probe: alive [2025-02-25 17:09:10,756][ ERROR][PID:664755] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 17:09:20,760][WARNING][PID:664755] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-02-25 17:09:30,761][ INFO][PID:664755] Downloading the builder-live.log file, attempt 3 [2025-02-25 17:09:30,764][ INFO][PID:664755] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@52.90.236.14 copr-rpmbuild-log [2025-02-25 17:10:30,838][ ERROR][PID:664755] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 17:10:40,839][WARNING][PID:664755] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts [2025-02-25 17:10:50,841][ INFO][PID:664755] Downloading the builder-live.log file, attempt 4 [2025-02-25 17:10:50,844][ INFO][PID:664755] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@52.90.236.14 copr-rpmbuild-log [2025-02-25 17:11:50,919][ ERROR][PID:664755] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke. [2025-02-25 17:12:00,922][ ERROR][PID:664755] Giving up for unstable SSH, failures: 1740503280.6810765, 1740503360.7608404, 1740503440.839516, 1740503520.9219162 [2025-02-25 17:12:00,923][ ERROR][PID:664755] Re-try request for task on 'ResallocHost, ticket_id=1504154, hostname=52.90.236.14, name=aws_x86_64_spot_prod_01450328_20250225_165343': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts [2025-02-25 17:12:00,924][ INFO][PID:664755] Releasing VM back to pool [2025-02-25 17:12:00,958][ ERROR][PID:664755] Backend process error: Three host tried without success: {'52.90.236.14', '34.229.194.103', '3.81.171.141'} [2025-02-25 17:12:00,959][WARNING][PID:664755] Switching not-finished job state to 'failed' [2025-02-25 17:12:00,960][ INFO][PID:664755] Worker failed build, took 997.9543404579163 [2025-02-25 17:12:00,962][ INFO][PID:664755] Sending build state back to frontend: { "builds": [ { "timeout": 18000, "frontend_base_url": "https://copr.fedorainfracloud.org", "memory_reqs": 2048, "enable_net": false, "project_owner": "osloup", "project_name": "docbook5_style_xsl_license_and_jar_build", "project_dirname": "docbook5_style_xsl_license_and_jar_build", "submitter": "https://src.fedoraproject.org/user/jakub", "ended_on": 1740503520.9595134, "started_on": 1740502523.005173, "submitted_on": null, "status": 0, "chroot": "fedora-rawhide-x86_64", "arch": "x86_64", "buildroot_pkgs": [], "task_id": "8699113-fedora-rawhide-x86_64", "build_id": 8699113, "package_name": "gcc", "package_version": "15.0.1-0.8", "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/osloup/docbook5_style_xsl_license_and_jar_build/gcc", "git_hash": "9e7590ec374ba3012bd2d2b524583a3205412f66", "git_branch": null, "source_type": null, "source_json": null, "pkg_name": null, "pkg_main_version": null, "pkg_epoch": null, "pkg_release": null, "srpm_url": null, "uses_devel_repo": false, "sandbox": "osloup/docbook5_style_xsl_license_and_jar_build--https://src.fedoraproject.org/user/jakub", "results": null, "appstream": false, "allow_user_ssh": false, "ssh_public_keys": null, "storage": null, "background": true, "fedora_review": false, "isolation": "default", "repo_priority": null, "repos": [ { "baseurl": "https://download.copr.fedorainfracloud.org/results/osloup/docbook5_style_xsl_license_and_jar_build/fedora-rawhide-x86_64/", "id": "copr_base", "name": "Copr repository", "priority": null } ], "tags": [ "arch_x86_64" ], "with_opts": [], "without_opts": [], "destdir": "/var/lib/copr/public_html/results/osloup/docbook5_style_xsl_license_and_jar_build", "results_repo_url": "https://download.copr.fedorainfracloud.org/results/osloup/docbook5_style_xsl_license_and_jar_build", "result_dir": "08699113-gcc", "built_packages": "", "id": 8699113, "mockchain_macros": { "copr_username": "osloup", "copr_projectname": "docbook5_style_xsl_license_and_jar_build", "vendor": "Fedora Project COPR (osloup/docbook5_style_xsl_license_and_jar_build)" } } ] } [2025-02-25 17:12:01,028][ INFO][PID:664755] Sending fedora-messaging bus message in build.end [2025-02-25 17:12:01,061][ INFO][PID:664755] Compressing /var/lib/copr/public_html/results/osloup/docbook5_style_xsl_license_and_jar_build/fedora-rawhide-x86_64/08699113-gcc/builder-live.log by gzip [2025-02-25 17:12:01,063][ INFO][PID:664755] Running command 'gzip /var/lib/copr/public_html/results/osloup/docbook5_style_xsl_license_and_jar_build/fedora-rawhide-x86_64/08699113-gcc/builder-live.log' as PID 1188367 [2025-02-25 17:12:01,072][ INFO][PID:664755] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/osloup/docbook5_style_xsl_license_and_jar_build/fedora-rawhide-x86_64/08699113-gcc/builder-live.log) [2025-02-25 17:12:01,074][ INFO][PID:664755] Compressing /var/lib/copr/public_html/results/osloup/docbook5_style_xsl_license_and_jar_build/fedora-rawhide-x86_64/08699113-gcc/backend.log by gzip [2025-02-25 17:12:01,077][ INFO][PID:664755] Running command 'gzip /var/lib/copr/public_html/results/osloup/docbook5_style_xsl_license_and_jar_build/fedora-rawhide-x86_64/08699113-gcc/backend.log' as PID 1188369