Package coprs :: Module helpers
[hide private]
[frames] | no frames]

Source Code for Module coprs.helpers

  1  import math 
  2  import random 
  3  import string 
  4   
  5  from six import with_metaclass 
  6  from six.moves.urllib.parse import urljoin, urlparse, parse_qs 
  7  from textwrap import dedent 
  8  import re 
  9   
 10  import flask 
 11  import posixpath 
 12  from flask import url_for 
 13  from dateutil import parser as dt_parser 
 14  from netaddr import IPAddress, IPNetwork 
 15  from redis import StrictRedis 
 16  from sqlalchemy.types import TypeDecorator, VARCHAR 
 17  import json 
 18   
 19  from copr_common.enums import EnumType 
 20  from coprs import constants 
 21  from coprs import app 
22 23 24 -def generate_api_token(size=30):
25 """ Generate a random string used as token to access the API 26 remotely. 27 28 :kwarg: size, the size of the token to generate, defaults to 30 29 chars. 30 :return: a string, the API token for the user. 31 """ 32 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
33 34 35 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}" 36 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 37 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 38 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
39 40 41 -class CounterStatType(object):
42 REPO_DL = "repo_dl"
43
44 45 -class PermissionEnum(with_metaclass(EnumType, object)):
46 vals = {"nothing": 0, "request": 1, "approved": 2} 47 48 @classmethod
49 - def choices_list(cls, without=-1):
50 return [(n, k) for k, n in cls.vals.items() if n != without]
51
52 53 -class BuildSourceEnum(with_metaclass(EnumType, object)):
54 vals = {"unset": 0, 55 "link": 1, # url 56 "upload": 2, # pkg, tmp, url 57 "pypi": 5, # package_name, version, python_versions 58 "rubygems": 6, # gem_name 59 "scm": 8, # type, clone_url, committish, subdirectory, spec, srpm_build_method 60 "custom": 9, # user-provided script to build sources 61 }
62
63 64 -class JSONEncodedDict(TypeDecorator):
65 """Represents an immutable structure as a json-encoded string. 66 67 Usage:: 68 69 JSONEncodedDict(255) 70 71 """ 72 73 impl = VARCHAR 74
75 - def process_bind_param(self, value, dialect):
76 if value is not None: 77 value = json.dumps(value) 78 79 return value
80
81 - def process_result_value(self, value, dialect):
82 if value is not None: 83 value = json.loads(value) 84 return value
85
86 87 -class Paginator(object):
88 - def __init__(self, query, total_count, page=1, 89 per_page_override=None, urls_count_override=None, 90 additional_params=None):
91 92 self.query = query 93 self.total_count = total_count 94 self.page = page 95 self.per_page = per_page_override or constants.ITEMS_PER_PAGE 96 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT 97 self.additional_params = additional_params or dict() 98 99 self._sliced_query = None
100
101 - def page_slice(self, page):
102 return (self.per_page * (page - 1), 103 self.per_page * page)
104 105 @property
106 - def sliced_query(self):
107 if not self._sliced_query: 108 self._sliced_query = self.query[slice(*self.page_slice(self.page))] 109 return self._sliced_query
110 111 @property
112 - def pages(self):
113 return int(math.ceil(self.total_count / float(self.per_page)))
114
115 - def border_url(self, request, start):
116 if start: 117 if self.page - 1 > self.urls_count // 2: 118 return self.url_for_other_page(request, 1), 1 119 else: 120 if self.page < self.pages - self.urls_count // 2: 121 return self.url_for_other_page(request, self.pages), self.pages 122 123 return None
124
125 - def get_urls(self, request):
126 left_border = self.page - self.urls_count // 2 127 left_border = 1 if left_border < 1 else left_border 128 right_border = self.page + self.urls_count // 2 129 right_border = self.pages if right_border > self.pages else right_border 130 131 return [(self.url_for_other_page(request, i), i) 132 for i in range(left_border, right_border + 1)]
133
134 - def url_for_other_page(self, request, page):
135 args = request.view_args.copy() 136 args["page"] = page 137 args.update(self.additional_params) 138 return flask.url_for(request.endpoint, **args)
139
140 141 -def chroot_to_branch(chroot):
142 """ 143 Get a git branch name from chroot. Follow the fedora naming standard. 144 """ 145 os, version, arch = chroot.rsplit("-", 2) 146 if os == "fedora": 147 if version == "rawhide": 148 return "master" 149 os = "f" 150 elif os == "epel" and int(version) <= 6: 151 os = "el" 152 elif os == "mageia" and version == "cauldron": 153 os = "cauldron" 154 version = "" 155 elif os == "mageia": 156 os = "mga" 157 return "{}{}".format(os, version)
158
159 160 # TODO: is there something like python-rpm-utils or python-dnf-utils for this? 161 -def splitFilename(filename):
162 """ 163 Pass in a standard style rpm fullname 164 165 Return a name, version, release, epoch, arch, e.g.:: 166 foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386 167 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64 168 """ 169 170 if filename[-4:] == '.rpm': 171 filename = filename[:-4] 172 173 archIndex = filename.rfind('.') 174 arch = filename[archIndex+1:] 175 176 relIndex = filename[:archIndex].rfind('-') 177 rel = filename[relIndex+1:archIndex] 178 179 verIndex = filename[:relIndex].rfind('-') 180 ver = filename[verIndex+1:relIndex] 181 182 epochIndex = filename.find(':') 183 if epochIndex == -1: 184 epoch = '' 185 else: 186 epoch = filename[:epochIndex] 187 188 name = filename[epochIndex + 1:verIndex] 189 return name, ver, rel, epoch, arch
190
191 192 -def parse_package_name(pkg):
193 """ 194 Parse package name from possibly incomplete nvra string. 195 """ 196 197 if pkg.count(".") >= 3 and pkg.count("-") >= 2: 198 return splitFilename(pkg)[0] 199 200 # doesn"t seem like valid pkg string, try to guess package name 201 result = "" 202 pkg = pkg.replace(".rpm", "").replace(".src", "") 203 204 for delim in ["-", "."]: 205 if delim in pkg: 206 parts = pkg.split(delim) 207 for part in parts: 208 if any(map(lambda x: x.isdigit(), part)): 209 return result[:-1] 210 211 result += part + "-" 212 213 return result[:-1] 214 215 return pkg
216
217 218 -def generate_repo_url(mock_chroot, url):
219 """ Generates url with build results for .repo file. 220 No checks if copr or mock_chroot exists. 221 """ 222 os_version = mock_chroot.os_version 223 224 if mock_chroot.os_release == "fedora": 225 if mock_chroot.os_version != "rawhide": 226 os_version = "$releasever" 227 228 if mock_chroot.os_release == "opensuse-leap": 229 os_version = "$releasever" 230 231 if mock_chroot.os_release == "mageia": 232 if mock_chroot.os_version != "cauldron": 233 os_version = "$releasever" 234 235 url = posixpath.join( 236 url, "{0}-{1}-{2}/".format(mock_chroot.os_release, 237 os_version, "$basearch")) 238 239 return url
240
241 242 -def fix_protocol_for_backend(url):
243 """ 244 Ensure that url either has http or https protocol according to the 245 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL" 246 """ 247 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https": 248 return url.replace("http://", "https://") 249 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http": 250 return url.replace("https://", "http://") 251 else: 252 return url
253
254 255 -def fix_protocol_for_frontend(url):
256 """ 257 Ensure that url either has http or https protocol according to the 258 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL" 259 """ 260 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https": 261 return url.replace("http://", "https://") 262 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http": 263 return url.replace("https://", "http://") 264 else: 265 return url
266
267 268 -class Serializer(object):
269
270 - def to_dict(self, options=None):
271 """ 272 Usage: 273 274 SQLAlchObject.to_dict() => returns a flat dict of the object 275 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object 276 and will include a flat dict of object foo inside of that 277 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns 278 a dict of the object, which will include dict of foo 279 (which will include dict of bar) and dict of spam. 280 281 Options can also contain two special values: __columns_only__ 282 and __columns_except__ 283 284 If present, the first makes only specified fields appear, 285 the second removes specified fields. Both of these fields 286 must be either strings (only works for one field) or lists 287 (for one and more fields). 288 289 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]}, 290 "__columns_only__": "name"}) => 291 292 The SQLAlchObject will only put its "name" into the resulting dict, 293 while "foo" all of its fields except "id". 294 295 Options can also specify whether to include foo_id when displaying 296 related foo object (__included_ids__, defaults to True). 297 This doesn"t apply when __columns_only__ is specified. 298 """ 299 300 result = {} 301 if options is None: 302 options = {} 303 columns = self.serializable_attributes 304 305 if "__columns_only__" in options: 306 columns = options["__columns_only__"] 307 else: 308 columns = set(columns) 309 if "__columns_except__" in options: 310 columns_except = options["__columns_except__"] 311 if not isinstance(options["__columns_except__"], list): 312 columns_except = [options["__columns_except__"]] 313 314 columns -= set(columns_except) 315 316 if ("__included_ids__" in options and 317 options["__included_ids__"] is False): 318 319 related_objs_ids = [ 320 r + "_id" for r, _ in options.items() 321 if not r.startswith("__")] 322 323 columns -= set(related_objs_ids) 324 325 columns = list(columns) 326 327 for column in columns: 328 result[column] = getattr(self, column) 329 330 for related, values in options.items(): 331 if hasattr(self, related): 332 result[related] = getattr(self, related).to_dict(values) 333 return result
334 335 @property
336 - def serializable_attributes(self):
337 return map(lambda x: x.name, self.__table__.columns)
338
339 340 -class RedisConnectionProvider(object):
341 - def __init__(self, config):
342 self.host = config.get("REDIS_HOST", "127.0.0.1") 343 self.port = int(config.get("REDIS_PORT", "6379"))
344
345 - def get_connection(self):
346 return StrictRedis(host=self.host, port=self.port)
347
348 349 -def get_redis_connection():
350 """ 351 Creates connection to redis, now we use default instance at localhost, no config needed 352 """ 353 return StrictRedis()
354
355 356 -def dt_to_unixtime(dt):
357 """ 358 Converts datetime to unixtime 359 :param dt: DateTime instance 360 :rtype: float 361 """ 362 return float(dt.strftime('%s'))
363
364 365 -def string_dt_to_unixtime(dt_string):
366 """ 367 Converts datetime to unixtime from string 368 :param dt_string: datetime string 369 :rtype: str 370 """ 371 return dt_to_unixtime(dt_parser.parse(dt_string))
372
373 374 -def is_ip_from_builder_net(ip):
375 """ 376 Checks is ip is owned by the builders network 377 :param str ip: IPv4 address 378 :return bool: True 379 """ 380 ip_addr = IPAddress(ip) 381 for subnet in app.config.get("BUILDER_IPS", ["127.0.0.1/24"]): 382 if ip_addr in IPNetwork(subnet): 383 return True 384 385 return False
386
387 388 -def str2bool(v):
389 if v is None: 390 return False 391 return v.lower() in ("yes", "true", "t", "1")
392
393 394 -def copr_url(view, copr, **kwargs):
395 """ 396 Examine given copr and generate proper URL for the `view` 397 398 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters, 399 and therefore you should *not* pass them manually. 400 401 Usage: 402 copr_url("coprs_ns.foo", copr) 403 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz) 404 """ 405 if copr.is_a_group_project: 406 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs) 407 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
408
409 410 -def url_for_copr_view(view, group_view, copr, **kwargs):
411 if copr.is_a_group_project: 412 return url_for(group_view, group_name=copr.group.name, coprname=copr.name, **kwargs) 413 else: 414 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
415
416 417 -def url_for_copr_builds(copr):
418 return copr_url("coprs_ns.copr_builds", copr)
419 420 421 from sqlalchemy.engine.default import DefaultDialect 422 from sqlalchemy.sql.sqltypes import String, DateTime, NullType 423 424 # python2/3 compatible. 425 PY3 = str is not bytes 426 text = str if PY3 else unicode 427 int_type = int if PY3 else (int, long) 428 str_type = str if PY3 else (str, unicode)
429 430 431 -class StringLiteral(String):
432 """Teach SA how to literalize various things."""
433 - def literal_processor(self, dialect):
434 super_processor = super(StringLiteral, self).literal_processor(dialect) 435 436 def process(value): 437 if isinstance(value, int_type): 438 return text(value) 439 if not isinstance(value, str_type): 440 value = text(value) 441 result = super_processor(value) 442 if isinstance(result, bytes): 443 result = result.decode(dialect.encoding) 444 return result
445 return process
446
447 448 -class LiteralDialect(DefaultDialect):
449 colspecs = { 450 # prevent various encoding explosions 451 String: StringLiteral, 452 # teach SA about how to literalize a datetime 453 DateTime: StringLiteral, 454 # don't format py2 long integers to NULL 455 NullType: StringLiteral, 456 }
457
458 459 -def literal_query(statement):
460 """NOTE: This is entirely insecure. DO NOT execute the resulting strings.""" 461 import sqlalchemy.orm 462 if isinstance(statement, sqlalchemy.orm.Query): 463 statement = statement.statement 464 return statement.compile( 465 dialect=LiteralDialect(), 466 compile_kwargs={'literal_binds': True}, 467 ).string
468
469 470 -def stream_template(template_name, **context):
471 app.update_template_context(context) 472 t = app.jinja_env.get_template(template_name) 473 rv = t.stream(context) 474 rv.enable_buffering(2) 475 return rv
476
477 478 -def generate_repo_name(repo_url):
479 """ based on url, generate repo name """ 480 repo_url = re.sub("[^a-zA-Z0-9]", '_', repo_url) 481 repo_url = re.sub("(__*)", '_', repo_url) 482 repo_url = re.sub("(_*$)|^_*", '', repo_url) 483 return repo_url
484
485 486 -def pre_process_repo_url(chroot, repo_url):
487 """ 488 Expands variables and sanitize repo url to be used for mock config 489 """ 490 parsed_url = urlparse(repo_url) 491 if parsed_url.scheme == "copr": 492 user = parsed_url.netloc 493 prj = parsed_url.path.split("/")[1] 494 repo_url = "/".join([ 495 flask.current_app.config["BACKEND_BASE_URL"], 496 "results", user, prj, chroot 497 ]) + "/" 498 499 repo_url = repo_url.replace("$chroot", chroot) 500 repo_url = repo_url.replace("$distname", chroot.rsplit("-", 2)[0]) 501 return repo_url
502
503 504 -def parse_repo_params(repo, supported_keys=None):
505 """ 506 :param repo: str repo from Copr/CoprChroot/Build/... 507 :param supported_keys list of supported optional parameters 508 :return: dict of optional parameters parsed from the repo URL 509 """ 510 supported_keys = supported_keys or ["priority"] 511 if not repo.startswith("copr://"): 512 return {} 513 514 params = {} 515 qs = parse_qs(urlparse(repo).query) 516 for k, v in qs.items(): 517 if k in supported_keys: 518 # parse_qs returns values as lists, but we allow setting the param only once, 519 # so we can take just first value from it 520 value = int(v[0]) if v[0].isnumeric() else v[0] 521 params[k] = value 522 return params
523
524 525 -def generate_build_config(copr, chroot_id):
526 """ Return dict with proper build config contents """ 527 chroot = None 528 for i in copr.copr_chroots: 529 if i.mock_chroot.name == chroot_id: 530 chroot = i 531 if not chroot: 532 return {} 533 534 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs 535 536 repos = [{ 537 "id": "copr_base", 538 "url": copr.repo_url + "/{}/".format(chroot_id), 539 "name": "Copr repository", 540 }] 541 542 if not copr.auto_createrepo: 543 repos.append({ 544 "id": "copr_base_devel", 545 "url": copr.repo_url + "/{}/devel/".format(chroot_id), 546 "name": "Copr buildroot", 547 }) 548 549 def get_additional_repo_views(repos_list): 550 repos = [] 551 for repo in repos_list: 552 params = parse_repo_params(repo) 553 repo_view = { 554 "id": generate_repo_name(repo), 555 "url": pre_process_repo_url(chroot_id, repo), 556 "name": "Additional repo " + generate_repo_name(repo), 557 } 558 repo_view.update(params) 559 repos.append(repo_view) 560 return repos
561 562 repos.extend(get_additional_repo_views(copr.repos_list)) 563 repos.extend(get_additional_repo_views(chroot.repos_list)) 564 565 return { 566 'project_id': copr.repo_id, 567 'additional_packages': packages.split(), 568 'repos': repos, 569 'chroot': chroot_id, 570 'use_bootstrap_container': copr.use_bootstrap_container, 571 'with_opts': chroot.with_opts.split(), 572 'without_opts': chroot.without_opts.split(), 573 } 574
575 576 -def generate_additional_repos(copr_chroot):
577 base_repo = "copr://{}".format(copr_chroot.copr.full_name) 578 repos = [base_repo] + copr_chroot.repos_list + copr_chroot.copr.repos_list 579 if not copr_chroot.copr.auto_createrepo: 580 repos.append("copr://{}/devel".format(copr_chroot.copr.full_name)) 581 return repos
582
583 584 -def trim_git_url(url):
585 if not url: 586 return None 587 588 return re.sub(r'(\.git)?/*$', '', url)
589
590 591 -def get_parsed_git_url(url):
592 if not url: 593 return False 594 595 url = trim_git_url(url) 596 return urlparse(url)
597