[2025-01-24 13:11:06,483][  INFO][PID:2572167] Marking build as starting
[2025-01-24 13:11:06,534][  INFO][PID:2572167] Trying to allocate VM
[2025-01-24 13:11:18,737][  INFO][PID:2572167] Allocated host ResallocHost, ticket_id=912379, hostname=140.211.168.177, name=copr_osuosl_p09_prod_00809721_20250124_000658
[2025-01-24 13:11:18,739][  INFO][PID:2572167] Allocating ssh connection to builder
[2025-01-24 13:11:18,740][  INFO][PID:2572167] Checking that builder machine is OK
[2025-01-24 13:11:20,037][  INFO][PID:2572167] Running remote command: copr-builder-ready epel-9-ppc64le
[2025-01-24 13:11:21,061][  INFO][PID:2572167] Checking Red Hat subscription (attempt #1)
Red Hat subscription active
Builder is ready to be used

[2025-01-24 13:11:21,062][  INFO][PID:2572167] Filling build.info file with builder info
[2025-01-24 13:11:21,063][  INFO][PID:2572167] Sending build state back to frontend: {
    "builds": [
        {
            "timeout": 18000,
            "frontend_base_url": "https://copr.fedorainfracloud.org",
            "memory_reqs": 2048,
            "enable_net": false,
            "project_owner": "luhliarik",
            "project_name": "nginx-mod-headers-more",
            "project_dirname": "nginx-mod-headers-more",
            "submitter": "luhliarik",
            "ended_on": null,
            "started_on": 1737724281.0634022,
            "submitted_on": null,
            "status": 3,
            "chroot": "epel-9-ppc64le",
            "arch": "ppc64le",
            "buildroot_pkgs": [],
            "task_id": "8567235-epel-9-ppc64le",
            "build_id": 8567235,
            "package_name": "nginx-mod-headers-more",
            "package_version": "0.37-1",
            "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/luhliarik/nginx-mod-headers-more/nginx-mod-headers-more",
            "git_hash": "9c1a0837e94576aa04410e0399e12b93f96d4edf",
            "git_branch": null,
            "source_type": null,
            "source_json": null,
            "pkg_name": null,
            "pkg_main_version": null,
            "pkg_epoch": null,
            "pkg_release": null,
            "srpm_url": null,
            "uses_devel_repo": false,
            "sandbox": "luhliarik/nginx-mod-headers-more--luhliarik",
            "results": null,
            "appstream": false,
            "allow_user_ssh": false,
            "ssh_public_keys": null,
            "storage": 0,
            "background": false,
            "fedora_review": false,
            "isolation": "default",
            "repo_priority": null,
            "repos": [
                {
                    "baseurl": "https://download.copr.fedorainfracloud.org/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/",
                    "id": "copr_base",
                    "name": "Copr repository",
                    "priority": null
                }
            ],
            "tags": [
                "arch_ppc64le",
                "arch_power9"
            ],
            "with_opts": [],
            "without_opts": [],
            "destdir": "/var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more",
            "results_repo_url": "https://download.copr.fedorainfracloud.org/results/luhliarik/nginx-mod-headers-more",
            "result_dir": "08567235-nginx-mod-headers-more",
            "built_packages": "",
            "id": 8567235,
            "mockchain_macros": {
                "copr_username": "luhliarik",
                "copr_projectname": "nginx-mod-headers-more",
                "vendor": "Fedora Project COPR (luhliarik/nginx-mod-headers-more)"
            }
        }
    ]
}
[2025-01-24 13:11:21,139][  INFO][PID:2572167] Sending fedora-messaging bus message in build.start
[2025-01-24 13:11:21,969][  INFO][PID:2572167] Sending fedora-messaging bus message in chroot.start
[2025-01-24 13:11:21,992][  INFO][PID:2572167] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8567235-epel-9-ppc64le --chroot epel-9-ppc64le --detached
[2025-01-24 13:11:23,350][  INFO][PID:2572167] Downloading the builder-live.log file, attempt 1
[2025-01-24 13:11:23,353][  INFO][PID:2572167] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.177 copr-rpmbuild-log
[2025-01-24 13:11:28,371][  INFO][PID:2572167] Periodic builder liveness probe: alive
[2025-01-24 13:21:28,503][  INFO][PID:2572167] Periodic builder liveness probe: alive
[2025-01-24 13:25:40,230][ ERROR][PID:2572167] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke.
[2025-01-24 13:25:50,231][WARNING][PID:2572167] SSH connection lost on #1 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts
[2025-01-24 13:26:00,232][  INFO][PID:2572167] Downloading the builder-live.log file, attempt 2
[2025-01-24 13:26:00,233][  INFO][PID:2572167] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.177 copr-rpmbuild-log
[2025-01-24 13:27:00,289][ ERROR][PID:2572167] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke.
[2025-01-24 13:27:10,293][WARNING][PID:2572167] SSH connection lost on #2 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts
[2025-01-24 13:27:20,293][  INFO][PID:2572167] Downloading the builder-live.log file, attempt 3
[2025-01-24 13:27:20,294][  INFO][PID:2572167] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.177 copr-rpmbuild-log
[2025-01-24 13:28:20,357][ ERROR][PID:2572167] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke.
[2025-01-24 13:28:30,362][WARNING][PID:2572167] SSH connection lost on #3 attempt, let's retry after 10s, Unable to finish after 1 SSH attempts
[2025-01-24 13:28:40,363][  INFO][PID:2572167] Downloading the builder-live.log file, attempt 4
[2025-01-24 13:28:40,364][  INFO][PID:2572167] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@140.211.168.177 copr-rpmbuild-log
[2025-01-24 13:29:40,432][ ERROR][PID:2572167] SSH connection lost on #1 attempt, let's retry after 10s, Connection broke.
[2025-01-24 13:29:50,441][ ERROR][PID:2572167] Giving up for unstable SSH, failures: 1737725150.231168, 1737725230.2929623, 1737725310.3623652, 1737725390.4412982
[2025-01-24 13:29:50,444][ ERROR][PID:2572167] Re-try request for task on 'ResallocHost, ticket_id=912379, hostname=140.211.168.177, name=copr_osuosl_p09_prod_00809721_20250124_000658': SSH problems when downloading live log: Stopped following builder for broken SSH: Unable to finish after 1 SSH attempts
[2025-01-24 13:29:50,445][  INFO][PID:2572167] Releasing VM back to pool
[2025-01-24 13:29:50,459][  INFO][PID:2572167] Retry #1 (on other host)
[2025-01-24 13:29:50,461][  INFO][PID:2572167] Trying to allocate VM
[2025-01-24 13:29:53,497][  INFO][PID:2572167] Allocated host ResallocHost, ticket_id=912829, hostname=2620:52:3:1:dead:beef:cafe:c28a, name=copr_p09_01_prod_00817000_20250124_131602
[2025-01-24 13:29:53,498][  INFO][PID:2572167] Allocating ssh connection to builder
[2025-01-24 13:29:53,499][  INFO][PID:2572167] Checking that builder machine is OK
[2025-01-24 13:29:54,242][  INFO][PID:2572167] Running remote command: copr-builder-ready epel-9-ppc64le
[2025-01-24 13:29:54,678][  INFO][PID:2572167] Checking Red Hat subscription (attempt #1)
Red Hat subscription active
Builder is ready to be used

[2025-01-24 13:29:54,679][  INFO][PID:2572167] Filling build.info file with builder info
[2025-01-24 13:29:54,683][  INFO][PID:2572167] Not re-notifying FE and msg buses for the new host.
[2025-01-24 13:29:54,684][  INFO][PID:2572167] Starting remote build: copr-rpmbuild --verbose --drop-resultdir --task-url https://copr.fedorainfracloud.org/backend/get-build-task/8567235-epel-9-ppc64le --chroot epel-9-ppc64le --detached
[2025-01-24 13:29:55,422][  INFO][PID:2572167] Downloading the builder-live.log file, attempt 1
[2025-01-24 13:29:55,424][  INFO][PID:2572167] Popen command started: ssh -F /home/copr/.ssh/config mockbuilder@2620:52:3:1:dead:beef:cafe:c28a copr-rpmbuild-log
[2025-01-24 13:31:30,438][  INFO][PID:2572167] Periodic builder liveness probe: alive
[2025-01-24 13:32:18,844][  INFO][PID:2572167] Downloading results from builder
[2025-01-24 13:32:18,845][  INFO][PID:2572167] rsyncing of mockbuilder@[2620:52:3:1:dead:beef:cafe:c28a]:/var/lib/copr-rpmbuild/results/ to /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more started
[2025-01-24 13:32:18,846][  INFO][PID:2572167] Popen command started: /usr/bin/rsync -rltDvH --chmod=D755,F644 -e 'ssh -F /home/copr/.ssh/config' mockbuilder@[2620:52:3:1:dead:beef:cafe:c28a]:/var/lib/copr-rpmbuild/results/ /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more/ &> /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more/build-08567235.rsync.log
[2025-01-24 13:32:19,500][  INFO][PID:2572167] rsyncing finished.
[2025-01-24 13:32:19,501][  INFO][PID:2572167] Releasing VM back to pool
[2025-01-24 13:32:19,511][  INFO][PID:2572167] Searching for 'success' file in resultdir
[2025-01-24 13:32:19,512][  INFO][PID:2572167] Going to sign pkgs from source: 8567235-epel-9-ppc64le in chroot: /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le
[2025-01-24 13:32:19,513][  INFO][PID:2572167] Calling '/bin/sign -u luhliarik#nginx-mod-headers-more@copr.fedorahosted.org -p' (attempt #1)
[2025-01-24 13:32:19,686][  INFO][PID:2572167] Calling '/bin/sign -4 -h sha256 -u luhliarik#nginx-mod-headers-more@copr.fedorahosted.org -r /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more/nginx-mod-headers-more-0.37-1.el9.src.rpm' (attempt #1)
[2025-01-24 13:32:20,474][  INFO][PID:2572167] signed rpm: /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more/nginx-mod-headers-more-0.37-1.el9.src.rpm
[2025-01-24 13:32:20,475][  INFO][PID:2572167] Calling '/bin/sign -4 -h sha256 -u luhliarik#nginx-mod-headers-more@copr.fedorahosted.org -r /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more/nginx-mod-headers-more-debugsource-0.37-1.el9.ppc64le.rpm' (attempt #1)
[2025-01-24 13:32:21,263][  INFO][PID:2572167] signed rpm: /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more/nginx-mod-headers-more-debugsource-0.37-1.el9.ppc64le.rpm
[2025-01-24 13:32:21,264][  INFO][PID:2572167] Calling '/bin/sign -4 -h sha256 -u luhliarik#nginx-mod-headers-more@copr.fedorahosted.org -r /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more/nginx-mod-headers-more-debuginfo-0.37-1.el9.ppc64le.rpm' (attempt #1)
[2025-01-24 13:32:21,921][  INFO][PID:2572167] signed rpm: /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more/nginx-mod-headers-more-debuginfo-0.37-1.el9.ppc64le.rpm
[2025-01-24 13:32:21,921][  INFO][PID:2572167] Calling '/bin/sign -4 -h sha256 -u luhliarik#nginx-mod-headers-more@copr.fedorahosted.org -r /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more/nginx-mod-headers-more-0.37-1.el9.ppc64le.rpm' (attempt #1)
[2025-01-24 13:32:22,717][  INFO][PID:2572167] signed rpm: /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more/nginx-mod-headers-more-0.37-1.el9.ppc64le.rpm
[2025-01-24 13:32:22,718][  INFO][PID:2572167] Sign done
[2025-01-24 13:32:22,719][  INFO][PID:2572167] Incremental createrepo run, adding 08567235-nginx-mod-headers-more into https://download.copr.fedorainfracloud.org/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le, (auto-create-repo=True)
[2025-01-24 13:32:22,720][  INFO][PID:2572167] Running command 'copr-repo --batched /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le --add 08567235-nginx-mod-headers-more --no-appstream-metadata' as PID 2780695
[2025-01-24 13:32:23,440][  INFO][PID:2572167] Finished after 0 seconds with exit code 0 (copr-repo --batched /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le --add 08567235-nginx-mod-headers-more --no-appstream-metadata)
[2025-01-24 13:32:23,442][  INFO][PID:2572167] Getting build details
[2025-01-24 13:32:23,442][  INFO][PID:2572167] Listing built binary packages in /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more
[2025-01-24 13:32:23,443][  INFO][PID:2572167] Built packages:
['nginx-mod-headers-more-debuginfo 0.37', 'nginx-mod-headers-more 0.37', 'nginx-mod-headers-more-debugsource 0.37']
[2025-01-24 13:32:23,444][  INFO][PID:2572167] build details: {'built_packages': 'nginx-mod-headers-more-debuginfo 0.37\nnginx-mod-headers-more 0.37\nnginx-mod-headers-more-debugsource 0.37'}
[2025-01-24 13:32:23,444][  INFO][PID:2572167] Retrieving pubkey
[2025-01-24 13:32:23,445][  INFO][PID:2572167] Calling '/bin/sign -u luhliarik#nginx-mod-headers-more@copr.fedorahosted.org -p' (attempt #1)
[2025-01-24 13:32:23,756][  INFO][PID:2572167] Added pubkey for user luhliarik project nginx-mod-headers-more into: /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/pubkey.gpg
[2025-01-24 13:32:23,757][  INFO][PID:2572167] Finished build: id=8567235 failed=False timeout=18000 destdir=/var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more chroot=epel-9-ppc64le 
[2025-01-24 13:32:23,759][  INFO][PID:2572167] Worker succeeded build, took 149.0755546092987
[2025-01-24 13:32:23,759][  INFO][PID:2572167] Sending build state back to frontend: {
    "builds": [
        {
            "timeout": 18000,
            "frontend_base_url": "https://copr.fedorainfracloud.org",
            "memory_reqs": 2048,
            "enable_net": false,
            "project_owner": "luhliarik",
            "project_name": "nginx-mod-headers-more",
            "project_dirname": "nginx-mod-headers-more",
            "submitter": "luhliarik",
            "ended_on": 1737725543.7590723,
            "started_on": 1737725394.6835177,
            "submitted_on": null,
            "status": 1,
            "chroot": "epel-9-ppc64le",
            "arch": "ppc64le",
            "buildroot_pkgs": [],
            "task_id": "8567235-epel-9-ppc64le",
            "build_id": 8567235,
            "package_name": "nginx-mod-headers-more",
            "package_version": "0.37-1",
            "git_repo": "https://copr-dist-git.fedorainfracloud.org/git/luhliarik/nginx-mod-headers-more/nginx-mod-headers-more",
            "git_hash": "9c1a0837e94576aa04410e0399e12b93f96d4edf",
            "git_branch": null,
            "source_type": null,
            "source_json": null,
            "pkg_name": null,
            "pkg_main_version": null,
            "pkg_epoch": null,
            "pkg_release": null,
            "srpm_url": null,
            "uses_devel_repo": false,
            "sandbox": "luhliarik/nginx-mod-headers-more--luhliarik",
            "results": {
                "packages": [
                    {
                        "name": "nginx-mod-headers-more-debuginfo",
                        "epoch": null,
                        "version": "0.37",
                        "release": "1.el9",
                        "arch": "ppc64le"
                    },
                    {
                        "name": "nginx-mod-headers-more",
                        "epoch": null,
                        "version": "0.37",
                        "release": "1.el9",
                        "arch": "src"
                    },
                    {
                        "name": "nginx-mod-headers-more",
                        "epoch": null,
                        "version": "0.37",
                        "release": "1.el9",
                        "arch": "ppc64le"
                    },
                    {
                        "name": "nginx-mod-headers-more-debugsource",
                        "epoch": null,
                        "version": "0.37",
                        "release": "1.el9",
                        "arch": "ppc64le"
                    }
                ]
            },
            "appstream": false,
            "allow_user_ssh": false,
            "ssh_public_keys": null,
            "storage": 0,
            "background": false,
            "fedora_review": false,
            "isolation": "default",
            "repo_priority": null,
            "repos": [
                {
                    "baseurl": "https://download.copr.fedorainfracloud.org/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/",
                    "id": "copr_base",
                    "name": "Copr repository",
                    "priority": null
                }
            ],
            "tags": [
                "arch_ppc64le",
                "arch_power9"
            ],
            "with_opts": [],
            "without_opts": [],
            "destdir": "/var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more",
            "results_repo_url": "https://download.copr.fedorainfracloud.org/results/luhliarik/nginx-mod-headers-more",
            "result_dir": "08567235-nginx-mod-headers-more",
            "built_packages": "nginx-mod-headers-more-debuginfo 0.37\nnginx-mod-headers-more 0.37\nnginx-mod-headers-more-debugsource 0.37",
            "id": 8567235,
            "mockchain_macros": {
                "copr_username": "luhliarik",
                "copr_projectname": "nginx-mod-headers-more",
                "vendor": "Fedora Project COPR (luhliarik/nginx-mod-headers-more)"
            }
        }
    ]
}
[2025-01-24 13:32:23,800][  INFO][PID:2572167] Sending fedora-messaging bus message in build.end
[2025-01-24 13:32:23,819][  INFO][PID:2572167] Compressing /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more/builder-live.log by gzip
[2025-01-24 13:32:23,821][  INFO][PID:2572167] Running command 'gzip /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more/builder-live.log' as PID 2780806
[2025-01-24 13:32:23,828][  INFO][PID:2572167] Finished after 0 seconds with exit code 0 (gzip /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more/builder-live.log)
[2025-01-24 13:32:23,829][  INFO][PID:2572167] Compressing /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more/backend.log by gzip
[2025-01-24 13:32:23,830][  INFO][PID:2572167] Running command 'gzip /var/lib/copr/public_html/results/luhliarik/nginx-mod-headers-more/epel-9-ppc64le/08567235-nginx-mod-headers-more/backend.log' as PID 2780808