Package coprs :: Module helpers
[hide private]
[frames] | no frames]

Source Code for Module coprs.helpers

  1  from __future__ import division 
  2   
  3  import math 
  4  import random 
  5  import string 
  6   
  7  from six import with_metaclass 
  8  from six.moves.urllib.parse import urljoin, urlparse 
  9  from textwrap import dedent 
 10  import re 
 11   
 12  import flask 
 13  from flask import url_for 
 14  from dateutil import parser as dt_parser 
 15  from netaddr import IPAddress, IPNetwork 
 16  from redis import StrictRedis 
 17  from sqlalchemy.types import TypeDecorator, VARCHAR 
 18  import json 
 19   
 20  from coprs import constants 
 21  from coprs import app 
22 23 24 -def generate_api_token(size=30):
25 """ Generate a random string used as token to access the API 26 remotely. 27 28 :kwarg: size, the size of the token to generate, defaults to 30 29 chars. 30 :return: a string, the API token for the user. 31 """ 32 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
33 34 35 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}" 36 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 37 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 38 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
39 40 41 -class CounterStatType(object):
42 REPO_DL = "repo_dl"
43
44 45 -class EnumType(type):
46
47 - def __call__(self, attr):
48 if isinstance(attr, int): 49 for k, v in self.vals.items(): 50 if v == attr: 51 return k 52 raise KeyError("num {0} is not mapped".format(attr)) 53 else: 54 return self.vals[attr]
55
56 57 -class PermissionEnum(with_metaclass(EnumType, object)):
58 vals = {"nothing": 0, "request": 1, "approved": 2} 59 60 @classmethod
61 - def choices_list(cls, without=-1):
62 return [(n, k) for k, n in cls.vals.items() if n != without]
63
64 65 -class ActionTypeEnum(with_metaclass(EnumType, object)):
66 vals = { 67 "delete": 0, 68 "rename": 1, 69 "legal-flag": 2, 70 "createrepo": 3, 71 "update_comps": 4, 72 "gen_gpg_key": 5, 73 "rawhide_to_release": 6, 74 "fork": 7, 75 "update_module_md": 8, 76 "build_module": 9, 77 "cancel_build": 10, 78 }
79
80 81 -class BackendResultEnum(with_metaclass(EnumType, object)):
82 vals = {"waiting": 0, "success": 1, "failure": 2}
83
84 85 -class RoleEnum(with_metaclass(EnumType, object)):
86 vals = {"user": 0, "admin": 1}
87
88 89 -class StatusEnum(with_metaclass(EnumType, object)):
90 vals = {"failed": 0, 91 "succeeded": 1, 92 "canceled": 2, 93 "running": 3, 94 "pending": 4, 95 "skipped": 5, # if there was this package built already 96 "starting": 6, # build picked by worker but no VM initialized 97 "importing": 7, # SRPM is being imported to dist-git 98 "forked": 8, # build(-chroot) was forked 99 "unknown": 1000, # order_to_status/status_to_order issue 100 }
101
102 103 -class ModuleStatusEnum(with_metaclass(EnumType, object)):
104 vals = {"pending": 0, "succeeded": 1, "failed": 2}
105
106 107 -class BuildSourceEnum(with_metaclass(EnumType, object)):
108 vals = {"unset": 0, 109 "link": 1, # url 110 "upload": 2, # pkg, tmp 111 "git_and_tito": 3, # git_url, git_dir, git_branch, tito_test 112 "mock_scm": 4, # scm_type, scm_url, spec, scm_branch, scm_subdir 113 "pypi": 5, # package_name, version, python_versions 114 "rubygems": 6, # gem_name 115 "distgit": 7, # url, branch 116 }
117
118 119 # The same enum is also in distgit's helpers.py 120 -class FailTypeEnum(with_metaclass(EnumType, object)):
121 vals = {"unset": 0, 122 # General errors mixed with errors for SRPM URL/upload: 123 "unknown_error": 1, 124 "build_error": 2, 125 "srpm_import_failed": 3, 126 "srpm_download_failed": 4, 127 "srpm_query_failed": 5, 128 "import_timeout_exceeded": 6, 129 # TODO: unused currently 130 "tito_general_error": 30, 131 "git_clone_failed": 31, 132 "git_wrong_directory": 32, 133 "git_checkout_error": 33, 134 "srpm_build_error": 34, 135 }
136
137 138 -class JSONEncodedDict(TypeDecorator):
139 """Represents an immutable structure as a json-encoded string. 140 141 Usage:: 142 143 JSONEncodedDict(255) 144 145 """ 146 147 impl = VARCHAR 148
149 - def process_bind_param(self, value, dialect):
150 if value is not None: 151 value = json.dumps(value) 152 153 return value
154
155 - def process_result_value(self, value, dialect):
156 if value is not None: 157 value = json.loads(value) 158 return value
159
160 -class Paginator(object):
161
162 - def __init__(self, query, total_count, page=1, 163 per_page_override=None, urls_count_override=None, 164 additional_params=None):
165 166 self.query = query 167 self.total_count = total_count 168 self.page = page 169 self.per_page = per_page_override or constants.ITEMS_PER_PAGE 170 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT 171 self.additional_params = additional_params or dict() 172 173 self._sliced_query = None
174
175 - def page_slice(self, page):
176 return (self.per_page * (page - 1), 177 self.per_page * page)
178 179 @property
180 - def sliced_query(self):
181 if not self._sliced_query: 182 self._sliced_query = self.query[slice(*self.page_slice(self.page))] 183 return self._sliced_query
184 185 @property
186 - def pages(self):
187 return int(math.ceil(self.total_count / float(self.per_page)))
188
189 - def border_url(self, request, start):
190 if start: 191 if self.page - 1 > self.urls_count // 2: 192 return self.url_for_other_page(request, 1), 1 193 else: 194 if self.page < self.pages - self.urls_count // 2: 195 return self.url_for_other_page(request, self.pages), self.pages 196 197 return None
198
199 - def get_urls(self, request):
200 left_border = self.page - self.urls_count // 2 201 left_border = 1 if left_border < 1 else left_border 202 right_border = self.page + self.urls_count // 2 203 right_border = self.pages if right_border > self.pages else right_border 204 205 return [(self.url_for_other_page(request, i), i) 206 for i in range(left_border, right_border + 1)]
207
208 - def url_for_other_page(self, request, page):
209 args = request.view_args.copy() 210 args["page"] = page 211 args.update(self.additional_params) 212 return flask.url_for(request.endpoint, **args)
213
214 215 -def chroot_to_branch(chroot):
216 """ 217 Get a git branch name from chroot. Follow the fedora naming standard. 218 """ 219 os, version, arch = chroot.split("-") 220 if os == "fedora": 221 if version == "rawhide": 222 return "master" 223 os = "f" 224 elif os == "epel" and int(version) <= 6: 225 os = "el" 226 elif os == "mageia" and version == "cauldron": 227 os = "cauldron" 228 version = "" 229 elif os == "mageia": 230 os = "mga" 231 return "{}{}".format(os, version)
232
233 234 -def splitFilename(filename):
235 """ 236 Pass in a standard style rpm fullname 237 238 Return a name, version, release, epoch, arch, e.g.:: 239 foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386 240 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64 241 """ 242 243 if filename[-4:] == '.rpm': 244 filename = filename[:-4] 245 246 archIndex = filename.rfind('.') 247 arch = filename[archIndex+1:] 248 249 relIndex = filename[:archIndex].rfind('-') 250 rel = filename[relIndex+1:archIndex] 251 252 verIndex = filename[:relIndex].rfind('-') 253 ver = filename[verIndex+1:relIndex] 254 255 epochIndex = filename.find(':') 256 if epochIndex == -1: 257 epoch = '' 258 else: 259 epoch = filename[:epochIndex] 260 261 name = filename[epochIndex + 1:verIndex] 262 return name, ver, rel, epoch, arch
263
264 265 -def parse_package_name(pkg):
266 """ 267 Parse package name from possibly incomplete nvra string. 268 """ 269 270 if pkg.count(".") >= 3 and pkg.count("-") >= 2: 271 return splitFilename(pkg)[0] 272 273 # doesn"t seem like valid pkg string, try to guess package name 274 result = "" 275 pkg = pkg.replace(".rpm", "").replace(".src", "") 276 277 for delim in ["-", "."]: 278 if delim in pkg: 279 parts = pkg.split(delim) 280 for part in parts: 281 if any(map(lambda x: x.isdigit(), part)): 282 return result[:-1] 283 284 result += part + "-" 285 286 return result[:-1] 287 288 return pkg
289
290 291 -def generate_repo_url(mock_chroot, url):
292 """ Generates url with build results for .repo file. 293 No checks if copr or mock_chroot exists. 294 """ 295 if mock_chroot.os_release == "fedora": 296 if mock_chroot.os_version != "rawhide": 297 mock_chroot.os_version = "$releasever" 298 299 url = urljoin( 300 url, "{0}-{1}-{2}/".format(mock_chroot.os_release, 301 mock_chroot.os_version, "$basearch")) 302 303 return url
304
305 306 -def fix_protocol_for_backend(url):
307 """ 308 Ensure that url either has http or https protocol according to the 309 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL" 310 """ 311 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https": 312 return url.replace("http://", "https://") 313 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http": 314 return url.replace("https://", "http://") 315 else: 316 return url
317
318 319 -def fix_protocol_for_frontend(url):
320 """ 321 Ensure that url either has http or https protocol according to the 322 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL" 323 """ 324 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https": 325 return url.replace("http://", "https://") 326 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http": 327 return url.replace("https://", "http://") 328 else: 329 return url
330
331 332 -class Serializer(object):
333
334 - def to_dict(self, options=None):
335 """ 336 Usage: 337 338 SQLAlchObject.to_dict() => returns a flat dict of the object 339 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object 340 and will include a flat dict of object foo inside of that 341 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns 342 a dict of the object, which will include dict of foo 343 (which will include dict of bar) and dict of spam. 344 345 Options can also contain two special values: __columns_only__ 346 and __columns_except__ 347 348 If present, the first makes only specified fiels appear, 349 the second removes specified fields. Both of these fields 350 must be either strings (only works for one field) or lists 351 (for one and more fields). 352 353 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]}, 354 "__columns_only__": "name"}) => 355 356 The SQLAlchObject will only put its "name" into the resulting dict, 357 while "foo" all of its fields except "id". 358 359 Options can also specify whether to include foo_id when displaying 360 related foo object (__included_ids__, defaults to True). 361 This doesn"t apply when __columns_only__ is specified. 362 """ 363 364 result = {} 365 if options is None: 366 options = {} 367 columns = self.serializable_attributes 368 369 if "__columns_only__" in options: 370 columns = options["__columns_only__"] 371 else: 372 columns = set(columns) 373 if "__columns_except__" in options: 374 columns_except = options["__columns_except__"] 375 if not isinstance(options["__columns_except__"], list): 376 columns_except = [options["__columns_except__"]] 377 378 columns -= set(columns_except) 379 380 if ("__included_ids__" in options and 381 options["__included_ids__"] is False): 382 383 related_objs_ids = [ 384 r + "_id" for r, _ in options.items() 385 if not r.startswith("__")] 386 387 columns -= set(related_objs_ids) 388 389 columns = list(columns) 390 391 for column in columns: 392 result[column] = getattr(self, column) 393 394 for related, values in options.items(): 395 if hasattr(self, related): 396 result[related] = getattr(self, related).to_dict(values) 397 return result
398 399 @property
400 - def serializable_attributes(self):
401 return map(lambda x: x.name, self.__table__.columns)
402
403 404 -class RedisConnectionProvider(object):
405 - def __init__(self, config):
406 self.host = config.get("REDIS_HOST", "127.0.0.1") 407 self.port = int(config.get("REDIS_PORT", "6379"))
408
409 - def get_connection(self):
410 return StrictRedis(host=self.host, port=self.port)
411
412 413 -def get_redis_connection():
414 """ 415 Creates connection to redis, now we use default instance at localhost, no config needed 416 """ 417 return StrictRedis()
418
419 420 -def dt_to_unixtime(dt):
421 """ 422 Converts datetime to unixtime 423 :param dt: DateTime instance 424 :rtype: float 425 """ 426 return float(dt.strftime('%s'))
427
428 429 -def string_dt_to_unixtime(dt_string):
430 """ 431 Converts datetime to unixtime from string 432 :param dt_string: datetime string 433 :rtype: str 434 """ 435 return dt_to_unixtime(dt_parser.parse(dt_string))
436
437 438 -def is_ip_from_builder_net(ip):
439 """ 440 Checks is ip is owned by the builders network 441 :param str ip: IPv4 address 442 :return bool: True 443 """ 444 ip_addr = IPAddress(ip) 445 for subnet in app.config.get("BUILDER_IPS", ["127.0.0.1/24"]): 446 if ip_addr in IPNetwork(subnet): 447 return True 448 449 return False
450
451 452 -def str2bool(v):
453 if v is None: 454 return False 455 return v.lower() in ("yes", "true", "t", "1")
456
457 458 -def copr_url(view, copr, **kwargs):
459 """ 460 Examine given copr and generate proper URL for the `view` 461 462 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters, 463 and therefore you should *not* pass them manually. 464 465 Usage: 466 copr_url("coprs_ns.foo", copr) 467 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz) 468 """ 469 if copr.is_a_group_project: 470 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs) 471 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
472
473 474 -def url_for_copr_view(view, group_view, copr, **kwargs):
475 if copr.is_a_group_project: 476 return url_for(group_view, group_name=copr.group.name, coprname=copr.name, **kwargs) 477 else: 478 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
479
480 481 -def url_for_copr_builds(copr):
482 if copr.is_a_group_project: 483 return url_for("coprs_ns.group_copr_builds", 484 group_name=copr.group.name, coprname=copr.name) 485 else: 486 return url_for("coprs_ns.copr_builds", 487 username=copr.user.username, coprname=copr.name)
488 489 490 from sqlalchemy.engine.default import DefaultDialect 491 from sqlalchemy.sql.sqltypes import String, DateTime, NullType 492 493 # python2/3 compatible. 494 PY3 = str is not bytes 495 text = str if PY3 else unicode 496 int_type = int if PY3 else (int, long) 497 str_type = str if PY3 else (str, unicode)
498 499 500 -class StringLiteral(String):
501 """Teach SA how to literalize various things."""
502 - def literal_processor(self, dialect):
503 super_processor = super(StringLiteral, self).literal_processor(dialect) 504 505 def process(value): 506 if isinstance(value, int_type): 507 return text(value) 508 if not isinstance(value, str_type): 509 value = text(value) 510 result = super_processor(value) 511 if isinstance(result, bytes): 512 result = result.decode(dialect.encoding) 513 return result
514 return process
515
516 517 -class LiteralDialect(DefaultDialect):
518 colspecs = { 519 # prevent various encoding explosions 520 String: StringLiteral, 521 # teach SA about how to literalize a datetime 522 DateTime: StringLiteral, 523 # don't format py2 long integers to NULL 524 NullType: StringLiteral, 525 }
526
527 528 -def literal_query(statement):
529 """NOTE: This is entirely insecure. DO NOT execute the resulting strings.""" 530 import sqlalchemy.orm 531 if isinstance(statement, sqlalchemy.orm.Query): 532 statement = statement.statement 533 return statement.compile( 534 dialect=LiteralDialect(), 535 compile_kwargs={'literal_binds': True}, 536 ).string
537
538 539 -def stream_template(template_name, **context):
540 app.update_template_context(context) 541 t = app.jinja_env.get_template(template_name) 542 rv = t.stream(context) 543 rv.enable_buffering(2) 544 return rv
545
546 547 -def generate_repo_name(repo_url):
548 """ based on url, generate repo name """ 549 repo_url = re.sub("[^a-zA-Z0-9]", '_', repo_url) 550 repo_url = re.sub("(__*)", '_', repo_url) 551 repo_url = re.sub("(_*$)|^_*", '', repo_url) 552 return repo_url
553
554 555 -def pre_process_repo_url(chroot, repo_url):
556 """ 557 Expands variables and sanitize repo url to be used for mock config 558 """ 559 parsed_url = urlparse(repo_url) 560 if parsed_url.scheme == "copr": 561 user = parsed_url.netloc 562 prj = parsed_url.path.split("/")[1] 563 repo_url = "/".join([ 564 flask.current_app.config["BACKEND_BASE_URL"], 565 "results", user, prj, chroot 566 ]) + "/" 567 568 repo_url = repo_url.replace("$chroot", chroot) 569 repo_url = repo_url.replace("$distname", chroot.split("-")[0]) 570 return repo_url
571
572 573 -def generate_build_config(copr, chroot_id):
574 """ Return dict with proper build config contents """ 575 chroot = None 576 for i in copr.copr_chroots: 577 if i.mock_chroot.name == chroot_id: 578 chroot = i 579 if not chroot: 580 return {} 581 582 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs 583 584 repos = [{ 585 "id": "copr_base", 586 "url": copr.repo_url + "/{}/".format(chroot_id), 587 "name": "Copr repository", 588 }] 589 590 if not copr.auto_createrepo: 591 repos.append({ 592 "id": "copr_base_devel", 593 "url": copr.repo_url + "/{}/devel/".format(chroot_id), 594 "name": "Copr buildroot", 595 }) 596 597 for repo in copr.repos_list: 598 repo_view = { 599 "id": generate_repo_name(repo), 600 "url": pre_process_repo_url(chroot_id, repo), 601 "name": "Additional repo " + generate_repo_name(repo), 602 } 603 repos.append(repo_view) 604 for repo in chroot.repos_list: 605 repo_view = { 606 "id": generate_repo_name(repo), 607 "url": pre_process_repo_url(chroot_id, repo), 608 "name": "Additional repo " + generate_repo_name(repo), 609 } 610 repos.append(repo_view) 611 612 return { 613 'project_id': copr.repo_id, 614 'additional_packages': packages.split(), 615 'repos': repos, 616 'chroot': chroot_id, 617 'use_bootstrap_container': copr.use_bootstrap_container 618 }
619