1 from __future__ import division
2
3 import math
4 import random
5 import string
6
7 from six import with_metaclass
8 from six.moves.urllib.parse import urljoin, urlparse
9 from textwrap import dedent
10 import re
11
12 import flask
13 from flask import url_for
14 from dateutil import parser as dt_parser
15 from netaddr import IPAddress, IPNetwork
16 from redis import StrictRedis
17 from sqlalchemy.types import TypeDecorator, VARCHAR
18 import json
19
20 from coprs import constants
21 from coprs import app
25 """ Generate a random string used as token to access the API
26 remotely.
27
28 :kwarg: size, the size of the token to generate, defaults to 30
29 chars.
30 :return: a string, the API token for the user.
31 """
32 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
33
34
35 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}"
36 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
37 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
38 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
43
46
48 if isinstance(attr, int):
49 for k, v in self.vals.items():
50 if v == attr:
51 return k
52 raise KeyError("num {0} is not mapped".format(attr))
53 else:
54 return self.vals[attr]
55
58 vals = {"nothing": 0, "request": 1, "approved": 2}
59
60 @classmethod
62 return [(n, k) for k, n in cls.vals.items() if n != without]
63
66 vals = {
67 "delete": 0,
68 "rename": 1,
69 "legal-flag": 2,
70 "createrepo": 3,
71 "update_comps": 4,
72 "gen_gpg_key": 5,
73 "rawhide_to_release": 6,
74 "fork": 7,
75 "update_module_md": 8,
76 "build_module": 9,
77 "cancel_build": 10,
78 }
79
82 vals = {"waiting": 0, "success": 1, "failure": 2}
83
84
85 -class RoleEnum(with_metaclass(EnumType, object)):
86 vals = {"user": 0, "admin": 1}
87
88
89 -class StatusEnum(with_metaclass(EnumType, object)):
90 vals = {"failed": 0,
91 "succeeded": 1,
92 "canceled": 2,
93 "running": 3,
94 "pending": 4,
95 "skipped": 5,
96 "starting": 6,
97 "importing": 7,
98 "forked": 8,
99 "unknown": 1000,
100 }
101
104 vals = {"pending": 0, "succeeded": 1, "failed": 2}
105
108 vals = {"unset": 0,
109 "link": 1,
110 "upload": 2,
111 "git_and_tito": 3,
112 "mock_scm": 4,
113 "pypi": 5,
114 "rubygems": 6,
115 "distgit": 7,
116 }
117
118
119
120 -class FailTypeEnum(with_metaclass(EnumType, object)):
121 vals = {"unset": 0,
122
123 "unknown_error": 1,
124 "build_error": 2,
125 "srpm_import_failed": 3,
126 "srpm_download_failed": 4,
127 "srpm_query_failed": 5,
128 "import_timeout_exceeded": 6,
129
130 "tito_general_error": 30,
131 "git_clone_failed": 31,
132 "git_wrong_directory": 32,
133 "git_checkout_error": 33,
134 "srpm_build_error": 34,
135 }
136
139 """Represents an immutable structure as a json-encoded string.
140
141 Usage::
142
143 JSONEncodedDict(255)
144
145 """
146
147 impl = VARCHAR
148
150 if value is not None:
151 value = json.dumps(value)
152
153 return value
154
156 if value is not None:
157 value = json.loads(value)
158 return value
159
161
162 - def __init__(self, query, total_count, page=1,
163 per_page_override=None, urls_count_override=None,
164 additional_params=None):
165
166 self.query = query
167 self.total_count = total_count
168 self.page = page
169 self.per_page = per_page_override or constants.ITEMS_PER_PAGE
170 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT
171 self.additional_params = additional_params or dict()
172
173 self._sliced_query = None
174
175 - def page_slice(self, page):
176 return (self.per_page * (page - 1),
177 self.per_page * page)
178
179 @property
181 if not self._sliced_query:
182 self._sliced_query = self.query[slice(*self.page_slice(self.page))]
183 return self._sliced_query
184
185 @property
187 return int(math.ceil(self.total_count / float(self.per_page)))
188
190 if start:
191 if self.page - 1 > self.urls_count // 2:
192 return self.url_for_other_page(request, 1), 1
193 else:
194 if self.page < self.pages - self.urls_count // 2:
195 return self.url_for_other_page(request, self.pages), self.pages
196
197 return None
198
200 left_border = self.page - self.urls_count // 2
201 left_border = 1 if left_border < 1 else left_border
202 right_border = self.page + self.urls_count // 2
203 right_border = self.pages if right_border > self.pages else right_border
204
205 return [(self.url_for_other_page(request, i), i)
206 for i in range(left_border, right_border + 1)]
207
208 - def url_for_other_page(self, request, page):
209 args = request.view_args.copy()
210 args["page"] = page
211 args.update(self.additional_params)
212 return flask.url_for(request.endpoint, **args)
213
216 """
217 Get a git branch name from chroot. Follow the fedora naming standard.
218 """
219 os, version, arch = chroot.split("-")
220 if os == "fedora":
221 if version == "rawhide":
222 return "master"
223 os = "f"
224 elif os == "epel" and int(version) <= 6:
225 os = "el"
226 elif os == "mageia" and version == "cauldron":
227 os = "cauldron"
228 version = ""
229 elif os == "mageia":
230 os = "mga"
231 return "{}{}".format(os, version)
232
235 """
236 Pass in a standard style rpm fullname
237
238 Return a name, version, release, epoch, arch, e.g.::
239 foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386
240 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64
241 """
242
243 if filename[-4:] == '.rpm':
244 filename = filename[:-4]
245
246 archIndex = filename.rfind('.')
247 arch = filename[archIndex+1:]
248
249 relIndex = filename[:archIndex].rfind('-')
250 rel = filename[relIndex+1:archIndex]
251
252 verIndex = filename[:relIndex].rfind('-')
253 ver = filename[verIndex+1:relIndex]
254
255 epochIndex = filename.find(':')
256 if epochIndex == -1:
257 epoch = ''
258 else:
259 epoch = filename[:epochIndex]
260
261 name = filename[epochIndex + 1:verIndex]
262 return name, ver, rel, epoch, arch
263
266 """
267 Parse package name from possibly incomplete nvra string.
268 """
269
270 if pkg.count(".") >= 3 and pkg.count("-") >= 2:
271 return splitFilename(pkg)[0]
272
273
274 result = ""
275 pkg = pkg.replace(".rpm", "").replace(".src", "")
276
277 for delim in ["-", "."]:
278 if delim in pkg:
279 parts = pkg.split(delim)
280 for part in parts:
281 if any(map(lambda x: x.isdigit(), part)):
282 return result[:-1]
283
284 result += part + "-"
285
286 return result[:-1]
287
288 return pkg
289
304
307 """
308 Ensure that url either has http or https protocol according to the
309 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL"
310 """
311 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https":
312 return url.replace("http://", "https://")
313 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http":
314 return url.replace("https://", "http://")
315 else:
316 return url
317
320 """
321 Ensure that url either has http or https protocol according to the
322 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL"
323 """
324 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https":
325 return url.replace("http://", "https://")
326 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http":
327 return url.replace("https://", "http://")
328 else:
329 return url
330
333
335 """
336 Usage:
337
338 SQLAlchObject.to_dict() => returns a flat dict of the object
339 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object
340 and will include a flat dict of object foo inside of that
341 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns
342 a dict of the object, which will include dict of foo
343 (which will include dict of bar) and dict of spam.
344
345 Options can also contain two special values: __columns_only__
346 and __columns_except__
347
348 If present, the first makes only specified fiels appear,
349 the second removes specified fields. Both of these fields
350 must be either strings (only works for one field) or lists
351 (for one and more fields).
352
353 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]},
354 "__columns_only__": "name"}) =>
355
356 The SQLAlchObject will only put its "name" into the resulting dict,
357 while "foo" all of its fields except "id".
358
359 Options can also specify whether to include foo_id when displaying
360 related foo object (__included_ids__, defaults to True).
361 This doesn"t apply when __columns_only__ is specified.
362 """
363
364 result = {}
365 if options is None:
366 options = {}
367 columns = self.serializable_attributes
368
369 if "__columns_only__" in options:
370 columns = options["__columns_only__"]
371 else:
372 columns = set(columns)
373 if "__columns_except__" in options:
374 columns_except = options["__columns_except__"]
375 if not isinstance(options["__columns_except__"], list):
376 columns_except = [options["__columns_except__"]]
377
378 columns -= set(columns_except)
379
380 if ("__included_ids__" in options and
381 options["__included_ids__"] is False):
382
383 related_objs_ids = [
384 r + "_id" for r, _ in options.items()
385 if not r.startswith("__")]
386
387 columns -= set(related_objs_ids)
388
389 columns = list(columns)
390
391 for column in columns:
392 result[column] = getattr(self, column)
393
394 for related, values in options.items():
395 if hasattr(self, related):
396 result[related] = getattr(self, related).to_dict(values)
397 return result
398
399 @property
402
406 self.host = config.get("REDIS_HOST", "127.0.0.1")
407 self.port = int(config.get("REDIS_PORT", "6379"))
408
410 return StrictRedis(host=self.host, port=self.port)
411
414 """
415 Creates connection to redis, now we use default instance at localhost, no config needed
416 """
417 return StrictRedis()
418
421 """
422 Converts datetime to unixtime
423 :param dt: DateTime instance
424 :rtype: float
425 """
426 return float(dt.strftime('%s'))
427
430 """
431 Converts datetime to unixtime from string
432 :param dt_string: datetime string
433 :rtype: str
434 """
435 return dt_to_unixtime(dt_parser.parse(dt_string))
436
439 """
440 Checks is ip is owned by the builders network
441 :param str ip: IPv4 address
442 :return bool: True
443 """
444 ip_addr = IPAddress(ip)
445 for subnet in app.config.get("BUILDER_IPS", ["127.0.0.1/24"]):
446 if ip_addr in IPNetwork(subnet):
447 return True
448
449 return False
450
453 if v is None:
454 return False
455 return v.lower() in ("yes", "true", "t", "1")
456
459 """
460 Examine given copr and generate proper URL for the `view`
461
462 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters,
463 and therefore you should *not* pass them manually.
464
465 Usage:
466 copr_url("coprs_ns.foo", copr)
467 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz)
468 """
469 if copr.is_a_group_project:
470 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs)
471 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
472
479
488
489
490 from sqlalchemy.engine.default import DefaultDialect
491 from sqlalchemy.sql.sqltypes import String, DateTime, NullType
492
493
494 PY3 = str is not bytes
495 text = str if PY3 else unicode
496 int_type = int if PY3 else (int, long)
497 str_type = str if PY3 else (str, unicode)
501 """Teach SA how to literalize various things."""
514 return process
515
526
529 """NOTE: This is entirely insecure. DO NOT execute the resulting strings."""
530 import sqlalchemy.orm
531 if isinstance(statement, sqlalchemy.orm.Query):
532 statement = statement.statement
533 return statement.compile(
534 dialect=LiteralDialect(),
535 compile_kwargs={'literal_binds': True},
536 ).string
537
540 app.update_template_context(context)
541 t = app.jinja_env.get_template(template_name)
542 rv = t.stream(context)
543 rv.enable_buffering(2)
544 return rv
545
553
556 """
557 Expands variables and sanitize repo url to be used for mock config
558 """
559 parsed_url = urlparse(repo_url)
560 if parsed_url.scheme == "copr":
561 user = parsed_url.netloc
562 prj = parsed_url.path.split("/")[1]
563 repo_url = "/".join([
564 flask.current_app.config["BACKEND_BASE_URL"],
565 "results", user, prj, chroot
566 ]) + "/"
567
568 repo_url = repo_url.replace("$chroot", chroot)
569 repo_url = repo_url.replace("$distname", chroot.split("-")[0])
570 return repo_url
571
574 """ Return dict with proper build config contents """
575 chroot = None
576 for i in copr.copr_chroots:
577 if i.mock_chroot.name == chroot_id:
578 chroot = i
579 if not chroot:
580 return {}
581
582 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs
583
584 repos = [{
585 "id": "copr_base",
586 "url": copr.repo_url + "/{}/".format(chroot_id),
587 "name": "Copr repository",
588 }]
589
590 if not copr.auto_createrepo:
591 repos.append({
592 "id": "copr_base_devel",
593 "url": copr.repo_url + "/{}/devel/".format(chroot_id),
594 "name": "Copr buildroot",
595 })
596
597 for repo in copr.repos_list:
598 repo_view = {
599 "id": generate_repo_name(repo),
600 "url": pre_process_repo_url(chroot_id, repo),
601 "name": "Additional repo " + generate_repo_name(repo),
602 }
603 repos.append(repo_view)
604 for repo in chroot.repos_list:
605 repo_view = {
606 "id": generate_repo_name(repo),
607 "url": pre_process_repo_url(chroot_id, repo),
608 "name": "Additional repo " + generate_repo_name(repo),
609 }
610 repos.append(repo_view)
611
612 return {
613 'project_id': copr.repo_id,
614 'additional_packages': packages.split(),
615 'repos': repos,
616 'chroot': chroot_id,
617 'use_bootstrap_container': copr.use_bootstrap_container
618 }
619