Package coprs :: Package views :: Package backend_ns :: Module backend_general
[hide private]
[frames] | no frames]

Source Code for Module coprs.views.backend_ns.backend_general

  1  import flask 
  2  import time 
  3  import sqlalchemy 
  4   
  5  from copr_common.enums import StatusEnum 
  6  from coprs import db, app 
  7  from coprs import helpers 
  8  from coprs import models 
  9  from coprs import exceptions 
 10  from coprs.logic import actions_logic 
 11  from coprs.logic.builds_logic import BuildsLogic, BuildChrootsLogic 
 12  from coprs.logic.complex_logic import ComplexLogic 
 13  from coprs.logic.coprs_logic import CoprChrootsLogic 
 14  from coprs.logic.packages_logic import PackagesLogic 
 15   
 16  from coprs.views import misc 
 17  from coprs.views.backend_ns import backend_ns 
 18  from sqlalchemy.sql import false, true 
 19   
 20  import json 
 21  import urllib 
 22  import logging 
 23   
 24  log = logging.getLogger(__name__) 
25 26 27 @backend_ns.route("/importing/") 28 -def dist_git_importing_queue():
29 """ 30 Return list of builds that are waiting for dist-git to import the sources. 31 """ 32 tasks = [] 33 34 builds_for_import = BuildsLogic.get_build_importing_queue().filter(models.Build.is_background == false()).limit(100).all() 35 if not builds_for_import: 36 builds_for_import = BuildsLogic.get_build_importing_queue().filter(models.Build.is_background == true()).limit(30).all() 37 38 for build in builds_for_import: 39 branches = set() 40 for b_ch in build.build_chroots: 41 branches.add(b_ch.mock_chroot.distgit_branch_name) 42 43 tasks.append({ 44 "build_id": build.id, 45 "owner": build.copr.owner_name, 46 "project": build.copr_dirname, 47 "branches": list(branches), 48 "srpm_url": build.srpm_url, 49 }) 50 51 return flask.jsonify(tasks)
52
53 54 @backend_ns.route("/import-completed/", methods=["POST", "PUT"]) 55 @misc.backend_authenticated 56 -def dist_git_upload_completed():
57 app.logger.debug(flask.request.json) 58 build_id = flask.request.json.get("build_id") 59 pkg_name = flask.request.json.get("pkg_name") 60 pkg_version = flask.request.json.get("pkg_evr") 61 62 try: 63 build = ComplexLogic.get_build_safe(build_id) 64 except ObjectNotFound: 65 return flask.jsonify({"updated": False}) 66 67 collected_branch_chroots = [] 68 for branch, git_hash in flask.request.json.get("branch_commits", {}).items(): 69 branch_chroots = BuildsLogic.get_buildchroots_by_build_id_and_branch(build_id, branch) 70 71 if not PackagesLogic.get(build.copr_dir.id, pkg_name).first(): 72 try: 73 package = PackagesLogic.add( 74 build.copr.user, build.copr_dir, 75 pkg_name, build.source_type, build.source_json) 76 db.session.add(package) 77 db.session.commit() 78 except (sqlalchemy.exc.IntegrityError, exceptions.DuplicateException) as e: 79 app.logger.exception(e) 80 db.session.rollback() 81 82 package = PackagesLogic.get(build.copr_dir.id, pkg_name).first() 83 build.package_id = package.id 84 build.pkg_version = pkg_version 85 86 for ch in branch_chroots: 87 ch.status = StatusEnum("pending") 88 ch.git_hash = git_hash 89 db.session.add(ch) 90 collected_branch_chroots.append((ch.task_id)) 91 92 final_source_status = StatusEnum("succeeded") 93 for ch in build.build_chroots: 94 if ch.task_id not in collected_branch_chroots: 95 final_source_status = StatusEnum("failed") 96 ch.status = StatusEnum("failed") 97 db.session.add(ch) 98 99 build.source_status = final_source_status 100 db.session.add(build) 101 db.session.commit() 102 103 BuildsLogic.delete_local_source(build) 104 return flask.jsonify({"updated": True})
105
106 107 -def get_build_record(task):
108 if not task: 109 return None 110 111 build_config = helpers.generate_build_config(task.build.copr, task.mock_chroot.name) 112 build_record = None 113 try: 114 build_record = { 115 "task_id": task.task_id, 116 "build_id": task.build.id, 117 "project_owner": task.build.copr.owner_name, 118 "project_name": task.build.copr_name, 119 "project_dirname": task.build.copr_dirname, 120 "submitter": task.build.user.name if task.build.user else None, # there is no user for webhook builds 121 "chroot": task.mock_chroot.name, 122 "repos": task.build.repos, 123 "memory_reqs": task.build.memory_reqs, 124 "timeout": task.build.timeout, 125 "enable_net": task.build.enable_net, 126 "git_repo": task.build.package.dist_git_repo, 127 "git_hash": task.git_hash, 128 "source_type": helpers.BuildSourceEnum("scm"), 129 "source_json": json.dumps( 130 {'clone_url': task.build.package.dist_git_clone_url, 'committish': task.git_hash}), 131 "fetch_sources_only": True, 132 "package_name": task.build.package.name, 133 "package_version": task.build.pkg_version, 134 "repos": build_config.get("repos"), 135 "buildroot_pkgs": build_config.get("additional_packages"), 136 "use_bootstrap_container": build_config.get("use_bootstrap_container"), 137 "with_opts": build_config.get("with_opts"), 138 "without_opts": build_config.get("without_opts"), 139 } 140 141 except Exception as err: 142 app.logger.exception(err) 143 return None 144 145 return build_record
146
147 148 -def get_srpm_build_record(task):
149 if not task: 150 return None 151 152 if task.source_type_text == "custom": 153 chroot = task.source_json_dict['chroot'] 154 else: 155 chroot = None 156 157 try: 158 build_record = { 159 "task_id": task.task_id, 160 "build_id": task.id, 161 "project_owner": task.copr.owner_name, 162 "project_name": task.copr_name, 163 "project_dirname": task.copr_dirname, 164 "source_type": task.source_type, 165 "source_json": task.source_json, 166 "chroot": chroot, 167 } 168 169 except Exception as err: 170 app.logger.exception(err) 171 return None 172 173 return build_record
174
175 176 @backend_ns.route("/pending-action/") 177 -def pending_action():
178 """ 179 Return a single action. 180 """ 181 action_record = None 182 action = actions_logic.ActionsLogic.get_waiting().first() 183 if action: 184 action_record = action.to_dict(options={ 185 "__columns_except__": ["result", "message", "ended_on"] 186 }) 187 return flask.jsonify(action_record)
188
189 190 @backend_ns.route("/pending-jobs/") 191 -def pending_jobs():
192 """ 193 Return the job queue. 194 """ 195 build_records = ([get_build_record(task) for task in BuildsLogic.get_pending_build_tasks()] + 196 [get_srpm_build_record(task) for task in BuildsLogic.get_pending_srpm_build_tasks()]) 197 log.info('Selected build records: {}'.format(build_records)) 198 return flask.jsonify(build_records)
199
200 201 @backend_ns.route("/get-build-task/<task_id>") 202 -def get_build_task(task_id):
203 try: 204 task = BuildsLogic.get_build_task(task_id) 205 except exceptions.MalformedArgumentException: 206 jsonout = flask.jsonify({'msg': 'Invalid task ID'}) 207 jsonout.status_code = 500 208 return jsonout 209 except sqlalchemy.orm.exc.NoResultFound: 210 jsonout = flask.jsonify({'msg': 'Specified task ID not found'}) 211 jsonout.status_code = 404 212 return jsonout 213 build_record = get_build_record(task) 214 return flask.jsonify(build_record)
215
216 217 @backend_ns.route("/get-srpm-build-task/<build_id>") 218 -def get_srpm_build_task(build_id):
219 try: 220 task = BuildsLogic.get_srpm_build_task(build_id) 221 except sqlalchemy.orm.exc.NoResultFound: 222 jsonout = flask.jsonify({'msg': 'Specified task ID not found'}) 223 jsonout.status_code = 404 224 return jsonout 225 build_record = get_srpm_build_record(task) 226 return flask.jsonify(build_record)
227
228 229 @backend_ns.route("/update/", methods=["POST", "PUT"]) 230 @misc.backend_authenticated 231 -def update():
232 result = {} 233 234 request_data = flask.request.json 235 for typ, logic_cls in [("actions", actions_logic.ActionsLogic), 236 ("builds", BuildsLogic)]: 237 238 if typ not in request_data: 239 continue 240 241 to_update = {} 242 for obj in request_data[typ]: 243 to_update[obj["id"]] = obj 244 245 existing = {} 246 for obj in logic_cls.get_by_ids(to_update.keys()).all(): 247 existing[obj.id] = obj 248 249 non_existing_ids = list(set(to_update.keys()) - set(existing.keys())) 250 251 for i, obj in existing.items(): 252 logic_cls.update_state_from_dict(obj, to_update[i]) 253 254 db.session.commit() 255 result.update({"updated_{0}_ids".format(typ): list(existing.keys()), 256 "non_existing_{0}_ids".format(typ): non_existing_ids}) 257 258 return flask.jsonify(result)
259
260 261 @backend_ns.route("/starting_build/", methods=["POST", "PUT"]) 262 @misc.backend_authenticated 263 -def starting_build():
264 """ 265 Check if the build is not cancelled and set it to starting state 266 """ 267 data = flask.request.json 268 269 try: 270 build = ComplexLogic.get_build_safe(data.get('build_id')) 271 except ObjectNotFound: 272 return flask.jsonify({"can_start": False}) 273 274 if build.canceled: 275 return flask.jsonify({"can_start": False}) 276 277 BuildsLogic.update_state_from_dict(build, data) 278 db.session.commit() 279 return flask.jsonify({"can_start": True})
280
281 282 @backend_ns.route("/reschedule_all_running/", methods=["POST", "PUT"]) 283 @misc.backend_authenticated 284 -def reschedule_all_running():
285 to_reschedule = \ 286 BuildsLogic.get_build_tasks(StatusEnum("starting")).all() + \ 287 BuildsLogic.get_build_tasks(StatusEnum("running")).all() 288 289 for build_chroot in to_reschedule: 290 build_chroot.status = StatusEnum("pending") 291 db.session.add(build_chroot) 292 293 to_reschedule = \ 294 BuildsLogic.get_srpm_build_tasks(StatusEnum("starting")).all() + \ 295 BuildsLogic.get_srpm_build_tasks(StatusEnum("running")).all() 296 297 for build in to_reschedule: 298 build.source_status = StatusEnum("pending") 299 db.session.add(build) 300 301 db.session.commit() 302 303 return "OK", 200
304
305 306 @backend_ns.route("/reschedule_build_chroot/", methods=["POST", "PUT"]) 307 @misc.backend_authenticated 308 -def reschedule_build_chroot():
309 response = {} 310 build_id = flask.request.json.get("build_id") 311 task_id = flask.request.json.get("task_id") 312 chroot = flask.request.json.get("chroot") 313 314 try: 315 build = ComplexLogic.get_build_safe(build_id) 316 except ObjectNotFound: 317 response["result"] = "noop" 318 response["msg"] = "Build {} wasn't found".format(build_id) 319 return flask.jsonify(response) 320 321 if build.canceled: 322 response["result"] = "noop" 323 response["msg"] = "build was cancelled, ignoring" 324 return flask.jsonify(response) 325 326 run_statuses = set([StatusEnum("starting"), StatusEnum("running")]) 327 328 if task_id == build.task_id: 329 if build.source_status in run_statuses: 330 log.info("rescheduling srpm build {}".format(build.id)) 331 BuildsLogic.update_state_from_dict(build, { 332 "task_id": task_id, 333 "status": StatusEnum("pending") 334 }) 335 db.session.commit() 336 response["result"] = "done" 337 else: 338 response["result"] = "noop" 339 response["msg"] = "build is not in running states, ignoring" 340 else: 341 build_chroot = build.chroots_dict_by_name.get(chroot) 342 if build_chroot and build_chroot.status in run_statuses: 343 log.info("rescheduling build {} chroot: {}".format(build.id, build_chroot.name)) 344 BuildsLogic.update_state_from_dict(build, { 345 "task_id": task_id, 346 "chroot": chroot, 347 "status": StatusEnum("pending") 348 }) 349 db.session.commit() 350 response["result"] = "done" 351 else: 352 response["result"] = "noop" 353 response["msg"] = "build chroot is not in running states, ignoring" 354 355 return flask.jsonify(response)
356