Package coprs :: Package logic :: Module builds_logic
[hide private]
[frames] | no frames]

Source Code for Module coprs.logic.builds_logic

   1  import tempfile 
   2  import shutil 
   3  import json 
   4  import os 
   5  import pprint 
   6  import time 
   7  import flask 
   8  import sqlite3 
   9  import requests 
  10   
  11  from flask import request 
  12  from sqlalchemy.sql import text 
  13  from sqlalchemy import or_ 
  14  from sqlalchemy import and_ 
  15  from sqlalchemy import func 
  16  from sqlalchemy.orm import joinedload 
  17  from sqlalchemy.orm.exc import NoResultFound 
  18  from sqlalchemy.sql import false,true 
  19  from werkzeug.utils import secure_filename 
  20  from sqlalchemy import desc, asc, bindparam, Integer, String 
  21  from collections import defaultdict 
  22   
  23  from copr_common.enums import FailTypeEnum, StatusEnum 
  24  from coprs import app 
  25  from coprs import db 
  26  from coprs import exceptions 
  27  from coprs import models 
  28  from coprs import helpers 
  29  from coprs.constants import DEFAULT_BUILD_TIMEOUT, MAX_BUILD_TIMEOUT 
  30  from coprs.exceptions import MalformedArgumentException, ActionInProgressException, InsufficientRightsException, UnrepeatableBuildException 
  31   
  32  from coprs.logic import coprs_logic 
  33  from coprs.logic import users_logic 
  34  from coprs.logic.actions_logic import ActionsLogic 
  35  from coprs.models import BuildChroot,Build,Package,MockChroot 
  36  from .coprs_logic import MockChrootsLogic 
  37   
  38  log = app.logger 
39 40 41 -class BuildsLogic(object):
42 @classmethod
43 - def get(cls, build_id):
44 return models.Build.query.filter(models.Build.id == build_id)
45 46 @classmethod
47 - def get_build_tasks(cls, status, background=None):
48 """ Returns tasks with given status. If background is specified then 49 returns normal jobs (false) or background jobs (true) 50 """ 51 result = models.BuildChroot.query.join(models.Build)\ 52 .filter(models.BuildChroot.status == status)\ 53 .order_by(models.Build.id.asc()) 54 if background is not None: 55 result = result.filter(models.Build.is_background == (true() if background else false())) 56 return result
57 58 @classmethod
59 - def get_srpm_build_tasks(cls, status, background=None):
60 """ Returns srpm build tasks with given status. If background is 61 specified then returns normal jobs (false) or background jobs (true) 62 """ 63 result = models.Build.query\ 64 .filter(models.Build.source_status == status)\ 65 .order_by(models.Build.id.asc()) 66 if background is not None: 67 result = result.filter(models.Build.is_background == (true() if background else false())) 68 return result
69 70 @classmethod
71 - def get_recent_tasks(cls, user=None, limit=None):
72 if not limit: 73 limit = 100 74 75 query = models.Build.query 76 if user is not None: 77 query = query.filter(models.Build.user_id == user.id) 78 79 query = query.join( 80 models.BuildChroot.query 81 .filter(models.BuildChroot.ended_on.isnot(None)) 82 .order_by(models.BuildChroot.ended_on.desc()) 83 .subquery() 84 ).order_by(models.Build.id.desc()) 85 86 # Workaround - otherwise it could take less records than `limit` even though there are more of them. 87 query = query.limit(limit if limit > 100 else 100) 88 return list(query.all()[:4])
89 90 @classmethod
91 - def get_running_tasks_by_time(cls, start, end):
92 result = models.BuildChroot.query\ 93 .filter(models.BuildChroot.ended_on > start)\ 94 .filter(models.BuildChroot.started_on < end)\ 95 .order_by(models.BuildChroot.started_on.asc()) 96 97 return result
98 99 @classmethod
100 - def get_chroot_histogram(cls, start, end):
101 chroots = [] 102 chroot_query = BuildChroot.query\ 103 .filter(models.BuildChroot.started_on < end)\ 104 .filter(models.BuildChroot.ended_on > start)\ 105 .with_entities(BuildChroot.mock_chroot_id, 106 func.count(BuildChroot.mock_chroot_id))\ 107 .group_by(BuildChroot.mock_chroot_id)\ 108 .order_by(BuildChroot.mock_chroot_id) 109 110 for chroot in chroot_query: 111 chroots.append([chroot[0], chroot[1]]) 112 113 mock_chroots = coprs_logic.MockChrootsLogic.get_multiple() 114 for mock_chroot in mock_chroots: 115 for l in chroots: 116 if l[0] == mock_chroot.id: 117 l[0] = mock_chroot.name 118 119 return chroots
120 121 @classmethod
122 - def get_pending_jobs_bucket(cls, start, end):
123 query = text(""" 124 SELECT COUNT(*) as result 125 FROM build_chroot JOIN build on build.id = build_chroot.build_id 126 WHERE 127 build.submitted_on < :end 128 AND ( 129 build_chroot.started_on > :start 130 OR (build_chroot.started_on is NULL AND build_chroot.status = :status) 131 -- for currently pending builds we need to filter on status=pending because there might be 132 -- failed builds that have started_on=NULL 133 ) 134 AND NOT build.canceled 135 """) 136 137 res = db.engine.execute(query, start=start, end=end, status=StatusEnum("pending")) 138 return res.first().result
139 140 @classmethod
141 - def get_running_jobs_bucket(cls, start, end):
142 query = text(""" 143 SELECT COUNT(*) as result 144 FROM build_chroot 145 WHERE 146 started_on < :end 147 AND (ended_on > :start OR (ended_on is NULL AND status = :status)) 148 -- for currently running builds we need to filter on status=running because there might be failed 149 -- builds that have ended_on=NULL 150 """) 151 152 res = db.engine.execute(query, start=start, end=end, status=StatusEnum("running")) 153 return res.first().result
154 155 @classmethod
156 - def get_cached_graph_data(cls, params):
157 data = { 158 "pending": [], 159 "running": [], 160 } 161 result = models.BuildsStatistics.query\ 162 .filter(models.BuildsStatistics.stat_type == params["type"])\ 163 .filter(models.BuildsStatistics.time >= params["start"])\ 164 .filter(models.BuildsStatistics.time <= params["end"])\ 165 .order_by(models.BuildsStatistics.time) 166 167 for row in result: 168 data["pending"].append(row.pending) 169 data["running"].append(row.running) 170 171 return data
172 173 @classmethod
174 - def get_task_graph_data(cls, type):
175 data = [["pending"], ["running"], ["avg running"], ["time"]] 176 params = cls.get_graph_parameters(type) 177 cached_data = cls.get_cached_graph_data(params) 178 data[0].extend(cached_data["pending"]) 179 data[1].extend(cached_data["running"]) 180 181 for i in range(len(data[0]) - 1, params["steps"]): 182 step_start = params["start"] + i * params["step"] 183 step_end = step_start + params["step"] 184 pending = cls.get_pending_jobs_bucket(step_start, step_end) 185 running = cls.get_running_jobs_bucket(step_start, step_end) 186 data[0].append(pending) 187 data[1].append(running) 188 cls.cache_graph_data(type, time=step_start, pending=pending, running=running) 189 190 running_total = 0 191 for i in range(1, params["steps"] + 1): 192 running_total += data[1][i] 193 194 data[2].extend([running_total * 1.0 / params["steps"]] * (len(data[0]) - 1)) 195 196 for i in range(params["start"], params["end"], params["step"]): 197 data[3].append(time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(i))) 198 199 return data
200 201 @classmethod
202 - def get_small_graph_data(cls, type):
203 data = [[""]] 204 params = cls.get_graph_parameters(type) 205 cached_data = cls.get_cached_graph_data(params) 206 data[0].extend(cached_data["running"]) 207 208 for i in range(len(data[0]) - 1, params["steps"]): 209 step_start = params["start"] + i * params["step"] 210 step_end = step_start + params["step"] 211 running = cls.get_running_jobs_bucket(step_start, step_end) 212 data[0].append(running) 213 cls.cache_graph_data(type, time=step_start, running=running) 214 215 return data
216 217 @classmethod
218 - def cache_graph_data(cls, type, time, pending=0, running=0):
219 query = models.BuildsStatistics.query\ 220 .filter(models.BuildsStatistics.stat_type == type)\ 221 .filter(models.BuildsStatistics.time == time).first() 222 if query: 223 return 224 225 cached_data = models.BuildsStatistics( 226 time = time, 227 stat_type = type, 228 running = running, 229 pending = pending 230 ) 231 db.session.merge(cached_data) 232 db.session.commit()
233 234 @classmethod
235 - def get_graph_parameters(cls, type):
236 if type is "10min": 237 # 24 hours with 10 minute intervals 238 step = 600 239 steps = 144 240 elif type is "30min": 241 # 24 hours with 30 minute intervals 242 step = 1800 243 steps = 48 244 elif type is "24h": 245 # 90 days with 24 hour intervals 246 step = 86400 247 steps = 90 248 249 end = int(time.time()) 250 end = end - (end % step) # align graph interval to a multiple of step 251 start = end - (steps * step) 252 253 return { 254 "type": type, 255 "step": step, 256 "steps": steps, 257 "start": start, 258 "end": end, 259 }
260 261 @classmethod
262 - def get_build_importing_queue(cls, background=None):
263 """ 264 Returns Builds which are waiting to be uploaded to dist git 265 """ 266 query = (models.Build.query 267 .filter(models.Build.canceled == false()) 268 .filter(models.Build.source_status == StatusEnum("importing")) 269 .order_by(models.Build.id.asc())) 270 if background is not None: 271 query = query.filter(models.Build.is_background == (true() if background else false())) 272 return query
273 274 @classmethod
275 - def get_pending_srpm_build_tasks(cls, background=None):
276 query = (models.Build.query 277 .filter(models.Build.canceled == false()) 278 .filter(models.Build.source_status == StatusEnum("pending")) 279 .order_by(models.Build.is_background.asc(), models.Build.id.asc())) 280 if background is not None: 281 query = query.filter(models.Build.is_background == (true() if background else false())) 282 return query
283 284 @classmethod
285 - def get_pending_build_tasks(cls, background=None):
286 query = (models.BuildChroot.query.join(models.Build) 287 .filter(models.Build.canceled == false()) 288 .filter(or_( 289 models.BuildChroot.status == StatusEnum("pending"), 290 and_( 291 models.BuildChroot.status == StatusEnum("running"), 292 models.BuildChroot.started_on < int(time.time() - 1.1 * MAX_BUILD_TIMEOUT), 293 models.BuildChroot.ended_on.is_(None) 294 ) 295 )) 296 .order_by(models.Build.is_background.asc(), models.Build.id.asc())) 297 if background is not None: 298 query = query.filter(models.Build.is_background == (true() if background else false())) 299 return query
300 301 @classmethod
302 - def get_build_task(cls, task_id):
303 try: 304 build_id, chroot_name = task_id.split("-", 1) 305 except ValueError: 306 raise MalformedArgumentException("Invalid task_id {}".format(task_id)) 307 308 build_chroot = BuildChrootsLogic.get_by_build_id_and_name(build_id, chroot_name) 309 return build_chroot.join(models.Build).first()
310 311 @classmethod
312 - def get_srpm_build_task(cls, build_id):
313 return BuildsLogic.get_by_id(build_id).first()
314 315 @classmethod
316 - def get_multiple(cls):
317 return models.Build.query.order_by(models.Build.id.desc())
318 319 @classmethod
320 - def get_multiple_by_copr(cls, copr):
321 """ Get collection of builds in copr sorted by build_id descending 322 """ 323 return cls.get_multiple().filter(models.Build.copr == copr)
324 325 @classmethod
326 - def get_multiple_by_user(cls, user):
327 """ Get collection of builds in copr sorted by build_id descending 328 form the copr belonging to `user` 329 """ 330 return cls.get_multiple().join(models.Build.copr).filter( 331 models.Copr.user == user)
332 333 @classmethod
334 - def init_db(cls):
335 if db.engine.url.drivername == "sqlite": 336 return 337 338 status_to_order = """ 339 CREATE OR REPLACE FUNCTION status_to_order (x integer) 340 RETURNS integer AS $$ BEGIN 341 RETURN CASE WHEN x = 3 THEN 1 342 WHEN x = 6 THEN 2 343 WHEN x = 7 THEN 3 344 WHEN x = 4 THEN 4 345 WHEN x = 0 THEN 5 346 WHEN x = 1 THEN 6 347 WHEN x = 5 THEN 7 348 WHEN x = 2 THEN 8 349 WHEN x = 8 THEN 9 350 WHEN x = 9 THEN 10 351 ELSE x 352 END; END; 353 $$ LANGUAGE plpgsql; 354 """ 355 356 order_to_status = """ 357 CREATE OR REPLACE FUNCTION order_to_status (x integer) 358 RETURNS integer AS $$ BEGIN 359 RETURN CASE WHEN x = 1 THEN 3 360 WHEN x = 2 THEN 6 361 WHEN x = 3 THEN 7 362 WHEN x = 4 THEN 4 363 WHEN x = 5 THEN 0 364 WHEN x = 6 THEN 1 365 WHEN x = 7 THEN 5 366 WHEN x = 8 THEN 2 367 WHEN x = 9 THEN 8 368 WHEN x = 10 THEN 9 369 ELSE x 370 END; END; 371 $$ LANGUAGE plpgsql; 372 """ 373 374 db.engine.connect() 375 db.engine.execute(status_to_order) 376 db.engine.execute(order_to_status)
377 378 @classmethod
379 - def get_copr_builds_list(cls, copr, dirname=''):
380 query_select = """ 381 SELECT build.id, build.source_status, MAX(package.name) AS pkg_name, build.pkg_version, build.submitted_on, 382 MIN(statuses.started_on) AS started_on, MAX(statuses.ended_on) AS ended_on, order_to_status(MIN(statuses.st)) AS status, 383 build.canceled, MIN("group".name) AS group_name, MIN(copr.name) as copr_name, MIN("user".username) as user_name, build.copr_id 384 FROM build 385 LEFT OUTER JOIN package 386 ON build.package_id = package.id 387 LEFT OUTER JOIN (SELECT build_chroot.build_id, started_on, ended_on, status_to_order(status) AS st FROM build_chroot) AS statuses 388 ON statuses.build_id=build.id 389 LEFT OUTER JOIN copr 390 ON copr.id = build.copr_id 391 LEFT OUTER JOIN copr_dir 392 ON build.copr_dir_id = copr_dir.id 393 LEFT OUTER JOIN "user" 394 ON copr.user_id = "user".id 395 LEFT OUTER JOIN "group" 396 ON copr.group_id = "group".id 397 WHERE build.copr_id = :copr_id 398 AND (:dirname = '' OR :dirname = copr_dir.name) 399 GROUP BY 400 build.id; 401 """ 402 403 if db.engine.url.drivername == "sqlite": 404 def sqlite_status_to_order(x): 405 if x == 3: 406 return 1 407 elif x == 6: 408 return 2 409 elif x == 7: 410 return 3 411 elif x == 4: 412 return 4 413 elif x == 0: 414 return 5 415 elif x == 1: 416 return 6 417 elif x == 5: 418 return 7 419 elif x == 2: 420 return 8 421 elif x == 8: 422 return 9 423 elif x == 9: 424 return 10 425 return 1000
426 427 def sqlite_order_to_status(x): 428 if x == 1: 429 return 3 430 elif x == 2: 431 return 6 432 elif x == 3: 433 return 7 434 elif x == 4: 435 return 4 436 elif x == 5: 437 return 0 438 elif x == 6: 439 return 1 440 elif x == 7: 441 return 5 442 elif x == 8: 443 return 2 444 elif x == 9: 445 return 8 446 elif x == 10: 447 return 9 448 return 1000
449 450 conn = db.engine.connect() 451 conn.connection.create_function("status_to_order", 1, sqlite_status_to_order) 452 conn.connection.create_function("order_to_status", 1, sqlite_order_to_status) 453 statement = text(query_select) 454 statement.bindparams(bindparam("copr_id", Integer)) 455 statement.bindparams(bindparam("dirname", String)) 456 result = conn.execute(statement, {"copr_id": copr.id, "dirname": dirname}) 457 else: 458 statement = text(query_select) 459 statement.bindparams(bindparam("copr_id", Integer)) 460 statement.bindparams(bindparam("dirname", String)) 461 result = db.engine.execute(statement, {"copr_id": copr.id, "dirname": dirname}) 462 463 return result 464 465 @classmethod
466 - def join_group(cls, query):
467 return query.join(models.Copr).outerjoin(models.Group)
468 469 @classmethod
470 - def get_multiple_by_name(cls, username, coprname):
471 query = cls.get_multiple() 472 return (query.join(models.Build.copr) 473 .options(db.contains_eager(models.Build.copr)) 474 .join(models.Copr.user) 475 .filter(models.Copr.name == coprname) 476 .filter(models.User.username == username))
477 478 @classmethod
479 - def get_by_ids(cls, ids):
480 return models.Build.query.filter(models.Build.id.in_(ids))
481 482 @classmethod
483 - def get_by_id(cls, build_id):
484 return models.Build.query.filter(models.Build.id == build_id)
485 486 @classmethod
487 - def create_new_from_other_build(cls, user, copr, source_build, 488 chroot_names=None, **build_options):
489 skip_import = False 490 git_hashes = {} 491 492 if source_build.source_type == helpers.BuildSourceEnum('upload'): 493 if source_build.repeatable: 494 skip_import = True 495 for chroot in source_build.build_chroots: 496 git_hashes[chroot.name] = chroot.git_hash 497 else: 498 raise UnrepeatableBuildException("Build sources were not fully imported into CoprDistGit.") 499 500 build = cls.create_new(user, copr, source_build.source_type, source_build.source_json, chroot_names, 501 pkgs=source_build.pkgs, git_hashes=git_hashes, skip_import=skip_import, 502 srpm_url=source_build.srpm_url, **build_options) 503 build.package_id = source_build.package_id 504 build.pkg_version = source_build.pkg_version 505 return build
506 507 @classmethod
508 - def create_new_from_url(cls, user, copr, url, 509 chroot_names=None, **build_options):
510 """ 511 :type user: models.User 512 :type copr: models.Copr 513 514 :type chroot_names: List[str] 515 516 :rtype: models.Build 517 """ 518 source_type = helpers.BuildSourceEnum("link") 519 source_json = json.dumps({"url": url}) 520 srpm_url = None if url.endswith('.spec') else url 521 return cls.create_new(user, copr, source_type, source_json, chroot_names, 522 pkgs=url, srpm_url=srpm_url, **build_options)
523 524 @classmethod
525 - def create_new_from_scm(cls, user, copr, scm_type, clone_url, 526 committish='', subdirectory='', spec='', srpm_build_method='rpkg', 527 chroot_names=None, **build_options):
528 """ 529 :type user: models.User 530 :type copr: models.Copr 531 532 :type chroot_names: List[str] 533 534 :rtype: models.Build 535 """ 536 source_type = helpers.BuildSourceEnum("scm") 537 source_json = json.dumps({"type": scm_type, 538 "clone_url": clone_url, 539 "committish": committish, 540 "subdirectory": subdirectory, 541 "spec": spec, 542 "srpm_build_method": srpm_build_method}) 543 return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
544 545 @classmethod
546 - def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, spec_template, 547 python_versions, chroot_names=None, **build_options):
548 """ 549 :type user: models.User 550 :type copr: models.Copr 551 :type package_name: str 552 :type version: str 553 :type python_versions: List[str] 554 555 :type chroot_names: List[str] 556 557 :rtype: models.Build 558 """ 559 source_type = helpers.BuildSourceEnum("pypi") 560 source_json = json.dumps({"pypi_package_name": pypi_package_name, 561 "pypi_package_version": pypi_package_version, 562 "spec_template": spec_template, 563 "python_versions": python_versions}) 564 return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
565 566 @classmethod
567 - def create_new_from_rubygems(cls, user, copr, gem_name, 568 chroot_names=None, **build_options):
569 """ 570 :type user: models.User 571 :type copr: models.Copr 572 :type gem_name: str 573 :type chroot_names: List[str] 574 :rtype: models.Build 575 """ 576 source_type = helpers.BuildSourceEnum("rubygems") 577 source_json = json.dumps({"gem_name": gem_name}) 578 return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
579 580 @classmethod
581 - def create_new_from_custom(cls, user, copr, 582 script, script_chroot=None, script_builddeps=None, 583 script_resultdir=None, chroot_names=None, **kwargs):
584 """ 585 :type user: models.User 586 :type copr: models.Copr 587 :type script: str 588 :type script_chroot: str 589 :type script_builddeps: str 590 :type script_resultdir: str 591 :type chroot_names: List[str] 592 :rtype: models.Build 593 """ 594 source_type = helpers.BuildSourceEnum("custom") 595 source_dict = { 596 'script': script, 597 'chroot': script_chroot, 598 'builddeps': script_builddeps, 599 'resultdir': script_resultdir, 600 } 601 602 return cls.create_new(user, copr, source_type, json.dumps(source_dict), 603 chroot_names, **kwargs)
604 605 @classmethod
606 - def create_new_from_upload(cls, user, copr, f_uploader, orig_filename, 607 chroot_names=None, **build_options):
608 """ 609 :type user: models.User 610 :type copr: models.Copr 611 :param f_uploader(file_path): function which stores data at the given `file_path` 612 :return: 613 """ 614 tmp = tempfile.mkdtemp(dir=app.config["STORAGE_DIR"]) 615 tmp_name = os.path.basename(tmp) 616 filename = secure_filename(orig_filename) 617 file_path = os.path.join(tmp, filename) 618 f_uploader(file_path) 619 620 # make the pkg public 621 pkg_url = "{baseurl}/tmp/{tmp_dir}/{filename}".format( 622 baseurl=app.config["PUBLIC_COPR_BASE_URL"], 623 tmp_dir=tmp_name, 624 filename=filename) 625 626 # create json describing the build source 627 source_type = helpers.BuildSourceEnum("upload") 628 source_json = json.dumps({"url": pkg_url, "pkg": filename, "tmp": tmp_name}) 629 srpm_url = None if pkg_url.endswith('.spec') else pkg_url 630 631 try: 632 build = cls.create_new(user, copr, source_type, source_json, 633 chroot_names, pkgs=pkg_url, srpm_url=srpm_url, **build_options) 634 except Exception: 635 shutil.rmtree(tmp) # todo: maybe we should delete in some cleanup procedure? 636 raise 637 638 return build
639 640 @classmethod
641 - def create_new(cls, user, copr, source_type, source_json, chroot_names=None, pkgs="", 642 git_hashes=None, skip_import=False, background=False, batch=None, 643 srpm_url=None, **build_options):
644 """ 645 :type user: models.User 646 :type copr: models.Copr 647 :type chroot_names: List[str] 648 :type source_type: int value from helpers.BuildSourceEnum 649 :type source_json: str in json format 650 :type pkgs: str 651 :type git_hashes: dict 652 :type skip_import: bool 653 :type background: bool 654 :type batch: models.Batch 655 :rtype: models.Build 656 """ 657 if chroot_names is None: 658 chroots = [c for c in copr.active_chroots] 659 else: 660 chroots = [] 661 for chroot in copr.active_chroots: 662 if chroot.name in chroot_names: 663 chroots.append(chroot) 664 665 build = cls.add( 666 user=user, 667 pkgs=pkgs, 668 copr=copr, 669 chroots=chroots, 670 source_type=source_type, 671 source_json=source_json, 672 enable_net=build_options.get("enable_net", copr.build_enable_net), 673 background=background, 674 git_hashes=git_hashes, 675 skip_import=skip_import, 676 batch=batch, 677 srpm_url=srpm_url, 678 ) 679 680 if user.proven: 681 if "timeout" in build_options: 682 build.timeout = build_options["timeout"] 683 684 return build
685 686 @classmethod
687 - def add(cls, user, pkgs, copr, source_type=None, source_json=None, 688 repos=None, chroots=None, timeout=None, enable_net=True, 689 git_hashes=None, skip_import=False, background=False, batch=None, 690 srpm_url=None):
691 692 if chroots is None: 693 chroots = [] 694 695 coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action( 696 copr, "Can't build while there is an operation in progress: {action}") 697 users_logic.UsersLogic.raise_if_cant_build_in_copr( 698 user, copr, 699 "You don't have permissions to build in this copr.") 700 701 if not repos: 702 repos = copr.repos 703 704 # todo: eliminate pkgs and this check 705 if pkgs and (" " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs): 706 raise exceptions.MalformedArgumentException("Trying to create a build using src_pkg " 707 "with bad characters. Forgot to split?") 708 709 # just temporary to keep compatibility 710 if not source_type or not source_json: 711 source_type = helpers.BuildSourceEnum("link") 712 source_json = json.dumps({"url":pkgs}) 713 714 if skip_import and srpm_url: 715 chroot_status = StatusEnum("pending") 716 source_status = StatusEnum("succeeded") 717 elif srpm_url: 718 chroot_status = StatusEnum("waiting") 719 source_status = StatusEnum("importing") 720 else: 721 chroot_status = StatusEnum("waiting") 722 source_status = StatusEnum("pending") 723 724 build = models.Build( 725 user=user, 726 pkgs=pkgs, 727 copr=copr, 728 repos=repos, 729 source_type=source_type, 730 source_json=source_json, 731 source_status=source_status, 732 submitted_on=int(time.time()), 733 enable_net=bool(enable_net), 734 is_background=bool(background), 735 batch=batch, 736 srpm_url=srpm_url, 737 ) 738 739 if timeout: 740 build.timeout = timeout or DEFAULT_BUILD_TIMEOUT 741 742 db.session.add(build) 743 744 # add BuildChroot object for each active (or selected) chroot 745 # this copr is assigned to 746 if not chroots: 747 chroots = copr.active_chroots 748 749 for chroot in chroots: 750 git_hash = None 751 if git_hashes: 752 git_hash = git_hashes.get(chroot.name) 753 buildchroot = models.BuildChroot( 754 build=build, 755 status=chroot_status, 756 mock_chroot=chroot, 757 git_hash=git_hash, 758 ) 759 db.session.add(buildchroot) 760 761 return build
762 763 @classmethod
764 - def rebuild_package(cls, package, source_dict_update={}, copr_dir=None, update_callback=None, 765 scm_object_type=None, scm_object_id=None, scm_object_url=None):
766 767 source_dict = package.source_json_dict 768 source_dict.update(source_dict_update) 769 source_json = json.dumps(source_dict) 770 771 if not copr_dir: 772 copr_dir = package.copr.main_dir 773 774 build = models.Build( 775 user=None, 776 pkgs=None, 777 package=package, 778 copr=package.copr, 779 repos=package.copr.repos, 780 source_status=StatusEnum("pending"), 781 source_type=package.source_type, 782 source_json=source_json, 783 submitted_on=int(time.time()), 784 enable_net=package.copr.build_enable_net, 785 timeout=DEFAULT_BUILD_TIMEOUT, 786 copr_dir=copr_dir, 787 update_callback=update_callback, 788 scm_object_type=scm_object_type, 789 scm_object_id=scm_object_id, 790 scm_object_url=scm_object_url, 791 ) 792 db.session.add(build) 793 794 chroots = package.copr.active_chroots 795 status = StatusEnum("waiting") 796 for chroot in chroots: 797 buildchroot = models.BuildChroot( 798 build=build, 799 status=status, 800 mock_chroot=chroot, 801 git_hash=None 802 ) 803 db.session.add(buildchroot) 804 805 cls.process_update_callback(build) 806 return build
807 808 809 terminal_states = {StatusEnum("failed"), StatusEnum("succeeded"), StatusEnum("canceled")} 810 811 @classmethod
812 - def get_buildchroots_by_build_id_and_branch(cls, build_id, branch):
813 """ 814 Returns a list of BuildChroots identified by build_id and dist-git 815 branch name. 816 """ 817 return ( 818 models.BuildChroot.query 819 .join(models.MockChroot) 820 .filter(models.BuildChroot.build_id==build_id) 821 .filter(models.MockChroot.distgit_branch_name==branch) 822 ).all()
823 824 825 @classmethod
826 - def delete_local_source(cls, build):
827 """ 828 Deletes the locally stored data for build purposes. This is typically 829 uploaded srpm file, uploaded spec file or webhook POST content. 830 """ 831 # is it hosted on the copr frontend? 832 data = json.loads(build.source_json) 833 if 'tmp' in data: 834 tmp = data["tmp"] 835 storage_path = app.config["STORAGE_DIR"] 836 try: 837 shutil.rmtree(os.path.join(storage_path, tmp)) 838 except: 839 pass
840 841 842 @classmethod
843 - def update_state_from_dict(cls, build, upd_dict):
844 """ 845 :param build: 846 :param upd_dict: 847 example: 848 { 849 "builds":[ 850 { 851 "id": 1, 852 "copr_id": 2, 853 "started_on": 1390866440 854 }, 855 { 856 "id": 2, 857 "copr_id": 1, 858 "status": 0, 859 "chroot": "fedora-18-x86_64", 860 "result_dir": "baz", 861 "ended_on": 1390866440 862 }] 863 } 864 """ 865 log.info("Updating build {} by: {}".format(build.id, upd_dict)) 866 867 # update build 868 for attr in ["built_packages", "srpm_url"]: 869 value = upd_dict.get(attr, None) 870 if value: 871 setattr(build, attr, value) 872 873 # update source build status 874 if upd_dict.get("task_id") == build.task_id: 875 build.result_dir = upd_dict.get("result_dir", "") 876 877 if upd_dict.get("status") == StatusEnum("succeeded"): 878 new_status = StatusEnum("importing") 879 else: 880 new_status = upd_dict.get("status") 881 882 build.source_status = new_status 883 if new_status == StatusEnum("failed") or \ 884 new_status == StatusEnum("skipped"): 885 for ch in build.build_chroots: 886 ch.status = new_status 887 ch.ended_on = upd_dict.get("ended_on") or time.time() 888 db.session.add(ch) 889 890 if new_status == StatusEnum("failed"): 891 build.fail_type = FailTypeEnum("srpm_build_error") 892 893 cls.process_update_callback(build) 894 db.session.add(build) 895 return 896 897 if "chroot" in upd_dict: 898 # update respective chroot status 899 for build_chroot in build.build_chroots: 900 if build_chroot.name == upd_dict["chroot"]: 901 build_chroot.result_dir = upd_dict.get("result_dir", "") 902 903 if "status" in upd_dict and build_chroot.status not in BuildsLogic.terminal_states: 904 build_chroot.status = upd_dict["status"] 905 906 if upd_dict.get("status") in BuildsLogic.terminal_states: 907 build_chroot.ended_on = upd_dict.get("ended_on") or time.time() 908 909 if upd_dict.get("status") == StatusEnum("starting"): 910 build_chroot.started_on = upd_dict.get("started_on") or time.time() 911 912 db.session.add(build_chroot) 913 914 # If the last package of a module was successfully built, 915 # then send an action to create module repodata on backend 916 if (build.module 917 and upd_dict.get("status") == StatusEnum("succeeded") 918 and all(b.status == StatusEnum("succeeded") for b in build.module.builds)): 919 ActionsLogic.send_build_module(build.copr, build.module) 920 921 cls.process_update_callback(build) 922 db.session.add(build)
923 924 @classmethod
925 - def process_update_callback(cls, build):
926 parsed_git_url = helpers.get_parsed_git_url(build.copr.scm_repo_url) 927 if not parsed_git_url: 928 return 929 930 if build.update_callback == 'pagure_flag_pull_request': 931 api_url = 'https://{0}/api/0/{1}/pull-request/{2}/flag'.format( 932 parsed_git_url.netloc, parsed_git_url.path, build.scm_object_id) 933 return cls.pagure_flag(build, api_url) 934 935 elif build.update_callback == 'pagure_flag_commit': 936 api_url = 'https://{0}/api/0/{1}/c/{2}/flag'.format( 937 parsed_git_url.netloc, parsed_git_url.path, build.scm_object_id) 938 return cls.pagure_flag(build, api_url)
939 940 @classmethod
941 - def pagure_flag(cls, build, api_url):
942 headers = { 943 'Authorization': 'token {}'.format(build.copr.scm_api_auth.get('api_key')) 944 } 945 946 if build.srpm_url: 947 progress = 50 948 else: 949 progress = 10 950 951 state_table = { 952 'failed': ('failure', 0), 953 'succeeded': ('success', 100), 954 'canceled': ('canceled', 0), 955 'running': ('pending', progress), 956 'pending': ('pending', progress), 957 'skipped': ('error', 0), 958 'starting': ('pending', progress), 959 'importing': ('pending', progress), 960 'forked': ('error', 0), 961 'waiting': ('pending', progress), 962 'unknown': ('error', 0), 963 } 964 965 build_url = os.path.join( 966 app.config['PUBLIC_COPR_BASE_URL'], 967 'coprs', build.copr.full_name.replace('@', 'g/'), 968 'build', str(build.id) 969 ) 970 971 data = { 972 'username': 'Copr build', 973 'comment': '#{}'.format(build.id), 974 'url': build_url, 975 'status': state_table[build.state][0], 976 'percent': state_table[build.state][1], 977 'uid': str(build.id), 978 } 979 980 log.info('Sending data to Pagure API: %s', pprint.pformat(data)) 981 response = requests.post(api_url, data=data, headers=headers) 982 log.info('Pagure API response: %s', response.text)
983 984 @classmethod
985 - def cancel_build(cls, user, build):
986 if not user.can_build_in(build.copr): 987 raise exceptions.InsufficientRightsException( 988 "You are not allowed to cancel this build.") 989 if not build.cancelable: 990 if build.status == StatusEnum("starting"): 991 # this is not intuitive, that's why we provide more specific message 992 err_msg = "Cannot cancel build {} in state 'starting'".format(build.id) 993 else: 994 err_msg = "Cannot cancel build {}".format(build.id) 995 raise exceptions.RequestCannotBeExecuted(err_msg) 996 997 if build.status == StatusEnum("running"): # otherwise the build is just in frontend 998 ActionsLogic.send_cancel_build(build) 999 1000 build.canceled = True 1001 cls.process_update_callback(build) 1002 1003 for chroot in build.build_chroots: 1004 chroot.status = 2 # canceled 1005 if chroot.ended_on is not None: 1006 chroot.ended_on = time.time()
1007 1008 @classmethod
1009 - def delete_build(cls, user, build, send_delete_action=True):
1010 """ 1011 :type user: models.User 1012 :type build: models.Build 1013 """ 1014 if not user.can_edit(build.copr) or build.persistent: 1015 raise exceptions.InsufficientRightsException( 1016 "You are not allowed to delete build `{}`.".format(build.id)) 1017 1018 if not build.finished: 1019 raise exceptions.ActionInProgressException( 1020 "You can not delete build `{}` which is not finished.".format(build.id), 1021 "Unfinished build") 1022 1023 if send_delete_action: 1024 ActionsLogic.send_delete_build(build) 1025 1026 for build_chroot in build.build_chroots: 1027 db.session.delete(build_chroot) 1028 1029 db.session.delete(build)
1030 1031 @classmethod
1032 - def mark_as_failed(cls, build_id):
1033 """ 1034 Marks build as failed on all its non-finished chroots 1035 """ 1036 build = cls.get(build_id).one() 1037 chroots = filter(lambda x: x.status != StatusEnum("succeeded"), build.build_chroots) 1038 for chroot in chroots: 1039 chroot.status = StatusEnum("failed") 1040 cls.process_update_callback(build) 1041 return build
1042 1043 @classmethod
1044 - def last_modified(cls, copr):
1045 """ Get build datetime (as epoch) of last successful build 1046 1047 :arg copr: object of copr 1048 """ 1049 builds = cls.get_multiple_by_copr(copr) 1050 1051 last_build = ( 1052 builds.join(models.BuildChroot) 1053 .filter((models.BuildChroot.status == StatusEnum("succeeded")) 1054 | (models.BuildChroot.status == StatusEnum("skipped"))) 1055 .filter(models.BuildChroot.ended_on.isnot(None)) 1056 .order_by(models.BuildChroot.ended_on.desc()) 1057 ).first() 1058 if last_build: 1059 return last_build.ended_on 1060 else: 1061 return None
1062 1063 @classmethod
1064 - def filter_is_finished(cls, query, is_finished):
1065 # todo: check that ended_on is set correctly for all cases 1066 # e.g.: failed dist-git import, cancellation 1067 if is_finished: 1068 return query.join(models.BuildChroot).filter(models.BuildChroot.ended_on.isnot(None)) 1069 else: 1070 return query.join(models.BuildChroot).filter(models.BuildChroot.ended_on.is_(None))
1071 1072 @classmethod
1073 - def filter_by_group_name(cls, query, group_name):
1074 return query.filter(models.Group.name == group_name)
1075 1076 @classmethod
1077 - def filter_by_package_name(cls, query, package_name):
1078 return query.join(models.Package).filter(models.Package.name == package_name)
1079
1080 1081 -class BuildChrootsLogic(object):
1082 @classmethod
1083 - def get_by_build_id_and_name(cls, build_id, name):
1084 mc = MockChrootsLogic.get_from_name(name).one() 1085 1086 return ( 1087 BuildChroot.query 1088 .filter(BuildChroot.build_id == build_id) 1089 .filter(BuildChroot.mock_chroot_id == mc.id) 1090 )
1091 1092 @classmethod
1093 - def get_multiply(cls):
1094 query = ( 1095 models.BuildChroot.query 1096 .join(models.BuildChroot.build) 1097 .join(models.BuildChroot.mock_chroot) 1098 .join(models.Build.copr) 1099 .join(models.Copr.user) 1100 .outerjoin(models.Group) 1101 ) 1102 return query
1103 1104 @classmethod
1105 - def filter_by_build_id(cls, query, build_id):
1106 return query.filter(models.Build.id == build_id)
1107 1108 @classmethod
1109 - def filter_by_project_id(cls, query, project_id):
1110 return query.filter(models.Copr.id == project_id)
1111 1112 @classmethod
1113 - def filter_by_project_user_name(cls, query, username):
1114 return query.filter(models.User.username == username)
1115 1116 @classmethod
1117 - def filter_by_state(cls, query, state):
1118 return query.filter(models.BuildChroot.status == StatusEnum(state))
1119 1120 @classmethod
1121 - def filter_by_group_name(cls, query, group_name):
1122 return query.filter(models.Group.name == group_name)
1123
1124 1125 -class BuildsMonitorLogic(object):
1126 @classmethod
1127 - def get_monitor_data(cls, copr):
1128 query = """ 1129 SELECT 1130 package.id as package_id, 1131 package.name AS package_name, 1132 build.id AS build_id, 1133 build_chroot.status AS build_chroot_status, 1134 build.pkg_version AS build_pkg_version, 1135 mock_chroot.id AS mock_chroot_id, 1136 mock_chroot.os_release AS mock_chroot_os_release, 1137 mock_chroot.os_version AS mock_chroot_os_version, 1138 mock_chroot.arch AS mock_chroot_arch 1139 FROM package 1140 JOIN (SELECT 1141 MAX(build.id) AS max_build_id_for_chroot, 1142 build.package_id AS package_id, 1143 build_chroot.mock_chroot_id AS mock_chroot_id 1144 FROM build 1145 JOIN build_chroot 1146 ON build.id = build_chroot.build_id 1147 WHERE build.copr_id = {copr_id} 1148 AND build_chroot.status != 2 1149 GROUP BY build.package_id, 1150 build_chroot.mock_chroot_id) AS max_build_ids_for_a_chroot 1151 ON package.id = max_build_ids_for_a_chroot.package_id 1152 JOIN build 1153 ON build.id = max_build_ids_for_a_chroot.max_build_id_for_chroot 1154 JOIN build_chroot 1155 ON build_chroot.mock_chroot_id = max_build_ids_for_a_chroot.mock_chroot_id 1156 AND build_chroot.build_id = max_build_ids_for_a_chroot.max_build_id_for_chroot 1157 JOIN mock_chroot 1158 ON mock_chroot.id = max_build_ids_for_a_chroot.mock_chroot_id 1159 ORDER BY package.name ASC, package.id ASC, mock_chroot.os_release ASC, mock_chroot.os_version ASC, mock_chroot.arch ASC 1160 """.format(copr_id=copr.id) 1161 rows = db.session.execute(query) 1162 return rows
1163