Package coprs :: Module helpers
[hide private]
[frames] | no frames]

Source Code for Module coprs.helpers

  1  from __future__ import division 
  2   
  3  import math 
  4  import random 
  5  import string 
  6   
  7  from six import with_metaclass 
  8  from six.moves.urllib.parse import urljoin, urlparse, parse_qs 
  9  from textwrap import dedent 
 10  import re 
 11   
 12  import flask 
 13  import posixpath 
 14  from flask import url_for 
 15  from dateutil import parser as dt_parser 
 16  from netaddr import IPAddress, IPNetwork 
 17  from redis import StrictRedis 
 18  from sqlalchemy.types import TypeDecorator, VARCHAR 
 19  import json 
 20   
 21  from coprs import constants 
 22  from coprs import app 
23 24 25 -def generate_api_token(size=30):
26 """ Generate a random string used as token to access the API 27 remotely. 28 29 :kwarg: size, the size of the token to generate, defaults to 30 30 chars. 31 :return: a string, the API token for the user. 32 """ 33 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
34 35 36 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}" 37 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 38 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 39 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
40 41 42 -class CounterStatType(object):
43 REPO_DL = "repo_dl"
44
45 46 -class EnumType(type):
47
48 - def __call__(self, attr):
49 if isinstance(attr, int): 50 for k, v in self.vals.items(): 51 if v == attr: 52 return k 53 raise KeyError("num {0} is not mapped".format(attr)) 54 else: 55 return self.vals[attr]
56
57 58 -class PermissionEnum(with_metaclass(EnumType, object)):
59 vals = {"nothing": 0, "request": 1, "approved": 2} 60 61 @classmethod
62 - def choices_list(cls, without=-1):
63 return [(n, k) for k, n in cls.vals.items() if n != without]
64
65 66 -class ActionTypeEnum(with_metaclass(EnumType, object)):
67 vals = { 68 "delete": 0, 69 "rename": 1, 70 "legal-flag": 2, 71 "createrepo": 3, 72 "update_comps": 4, 73 "gen_gpg_key": 5, 74 "rawhide_to_release": 6, 75 "fork": 7, 76 "update_module_md": 8, 77 "build_module": 9, 78 "cancel_build": 10, 79 }
80
81 82 -class BackendResultEnum(with_metaclass(EnumType, object)):
83 vals = {"waiting": 0, "success": 1, "failure": 2}
84
85 86 -class RoleEnum(with_metaclass(EnumType, object)):
87 vals = {"user": 0, "admin": 1}
88
89 90 -class StatusEnum(with_metaclass(EnumType, object)):
91 vals = { 92 "failed": 0, # build failed 93 "succeeded": 1, # build succeeded 94 "canceled": 2, # build was canceled 95 "running": 3, # SRPM or RPM build is running 96 "pending": 4, # build(-chroot) is waiting to be picked 97 "skipped": 5, # if there was this package built already 98 "starting": 6, # build was picked by worker but no VM initialized yet 99 "importing": 7, # SRPM is being imported into dist-git 100 "forked": 8, # build(-chroot) was forked 101 "waiting": 9, # build(-chroot) is waiting for something else to finish 102 "unknown": 1000, # undefined 103 }
104
105 106 -class ModuleStatusEnum(with_metaclass(EnumType, object)):
107 vals = {"pending": 0, "succeeded": 1, "failed": 2}
108
109 110 -class BuildSourceEnum(with_metaclass(EnumType, object)):
111 vals = {"unset": 0, 112 "link": 1, # url 113 "upload": 2, # pkg, tmp, url 114 "pypi": 5, # package_name, version, python_versions 115 "rubygems": 6, # gem_name 116 "scm": 8, # type, clone_url, committish, subdirectory, spec, srpm_build_method 117 "custom": 9, # user-provided script to build sources 118 }
119
120 # The same enum is also in distgit's helpers.py 121 -class FailTypeEnum(with_metaclass(EnumType, object)):
122 vals = {"unset": 0, 123 # General errors mixed with errors for SRPM URL/upload: 124 "unknown_error": 1, 125 "build_error": 2, 126 "srpm_import_failed": 3, 127 "srpm_download_failed": 4, 128 "srpm_query_failed": 5, 129 "import_timeout_exceeded": 6, 130 "git_clone_failed": 31, 131 "git_wrong_directory": 32, 132 "git_checkout_error": 33, 133 "srpm_build_error": 34, 134 }
135
136 137 -class JSONEncodedDict(TypeDecorator):
138 """Represents an immutable structure as a json-encoded string. 139 140 Usage:: 141 142 JSONEncodedDict(255) 143 144 """ 145 146 impl = VARCHAR 147
148 - def process_bind_param(self, value, dialect):
149 if value is not None: 150 value = json.dumps(value) 151 152 return value
153
154 - def process_result_value(self, value, dialect):
155 if value is not None: 156 value = json.loads(value) 157 return value
158
159 -class Paginator(object):
160
161 - def __init__(self, query, total_count, page=1, 162 per_page_override=None, urls_count_override=None, 163 additional_params=None):
164 165 self.query = query 166 self.total_count = total_count 167 self.page = page 168 self.per_page = per_page_override or constants.ITEMS_PER_PAGE 169 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT 170 self.additional_params = additional_params or dict() 171 172 self._sliced_query = None
173
174 - def page_slice(self, page):
175 return (self.per_page * (page - 1), 176 self.per_page * page)
177 178 @property
179 - def sliced_query(self):
180 if not self._sliced_query: 181 self._sliced_query = self.query[slice(*self.page_slice(self.page))] 182 return self._sliced_query
183 184 @property
185 - def pages(self):
186 return int(math.ceil(self.total_count / float(self.per_page)))
187
188 - def border_url(self, request, start):
189 if start: 190 if self.page - 1 > self.urls_count // 2: 191 return self.url_for_other_page(request, 1), 1 192 else: 193 if self.page < self.pages - self.urls_count // 2: 194 return self.url_for_other_page(request, self.pages), self.pages 195 196 return None
197
198 - def get_urls(self, request):
199 left_border = self.page - self.urls_count // 2 200 left_border = 1 if left_border < 1 else left_border 201 right_border = self.page + self.urls_count // 2 202 right_border = self.pages if right_border > self.pages else right_border 203 204 return [(self.url_for_other_page(request, i), i) 205 for i in range(left_border, right_border + 1)]
206
207 - def url_for_other_page(self, request, page):
208 args = request.view_args.copy() 209 args["page"] = page 210 args.update(self.additional_params) 211 return flask.url_for(request.endpoint, **args)
212
213 214 -def chroot_to_branch(chroot):
215 """ 216 Get a git branch name from chroot. Follow the fedora naming standard. 217 """ 218 os, version, arch = chroot.split("-") 219 if os == "fedora": 220 if version == "rawhide": 221 return "master" 222 os = "f" 223 elif os == "epel" and int(version) <= 6: 224 os = "el" 225 elif os == "mageia" and version == "cauldron": 226 os = "cauldron" 227 version = "" 228 elif os == "mageia": 229 os = "mga" 230 return "{}{}".format(os, version)
231
232 233 # TODO: is there something like python-rpm-utils or python-dnf-utils for this? 234 -def splitFilename(filename):
235 """ 236 Pass in a standard style rpm fullname 237 238 Return a name, version, release, epoch, arch, e.g.:: 239 foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386 240 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64 241 """ 242 243 if filename[-4:] == '.rpm': 244 filename = filename[:-4] 245 246 archIndex = filename.rfind('.') 247 arch = filename[archIndex+1:] 248 249 relIndex = filename[:archIndex].rfind('-') 250 rel = filename[relIndex+1:archIndex] 251 252 verIndex = filename[:relIndex].rfind('-') 253 ver = filename[verIndex+1:relIndex] 254 255 epochIndex = filename.find(':') 256 if epochIndex == -1: 257 epoch = '' 258 else: 259 epoch = filename[:epochIndex] 260 261 name = filename[epochIndex + 1:verIndex] 262 return name, ver, rel, epoch, arch
263
264 265 -def parse_package_name(pkg):
266 """ 267 Parse package name from possibly incomplete nvra string. 268 """ 269 270 if pkg.count(".") >= 3 and pkg.count("-") >= 2: 271 return splitFilename(pkg)[0] 272 273 # doesn"t seem like valid pkg string, try to guess package name 274 result = "" 275 pkg = pkg.replace(".rpm", "").replace(".src", "") 276 277 for delim in ["-", "."]: 278 if delim in pkg: 279 parts = pkg.split(delim) 280 for part in parts: 281 if any(map(lambda x: x.isdigit(), part)): 282 return result[:-1] 283 284 result += part + "-" 285 286 return result[:-1] 287 288 return pkg
289
290 291 -def generate_repo_url(mock_chroot, url):
292 """ Generates url with build results for .repo file. 293 No checks if copr or mock_chroot exists. 294 """ 295 os_version = mock_chroot.os_version 296 297 if mock_chroot.os_release == "fedora": 298 if mock_chroot.os_version != "rawhide": 299 os_version = "$releasever" 300 301 url = posixpath.join( 302 url, "{0}-{1}-{2}/".format(mock_chroot.os_release, 303 os_version, "$basearch")) 304 305 return url
306
307 308 -def fix_protocol_for_backend(url):
309 """ 310 Ensure that url either has http or https protocol according to the 311 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL" 312 """ 313 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https": 314 return url.replace("http://", "https://") 315 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http": 316 return url.replace("https://", "http://") 317 else: 318 return url
319
320 321 -def fix_protocol_for_frontend(url):
322 """ 323 Ensure that url either has http or https protocol according to the 324 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL" 325 """ 326 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https": 327 return url.replace("http://", "https://") 328 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http": 329 return url.replace("https://", "http://") 330 else: 331 return url
332
333 334 -class Serializer(object):
335
336 - def to_dict(self, options=None):
337 """ 338 Usage: 339 340 SQLAlchObject.to_dict() => returns a flat dict of the object 341 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object 342 and will include a flat dict of object foo inside of that 343 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns 344 a dict of the object, which will include dict of foo 345 (which will include dict of bar) and dict of spam. 346 347 Options can also contain two special values: __columns_only__ 348 and __columns_except__ 349 350 If present, the first makes only specified fields appear, 351 the second removes specified fields. Both of these fields 352 must be either strings (only works for one field) or lists 353 (for one and more fields). 354 355 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]}, 356 "__columns_only__": "name"}) => 357 358 The SQLAlchObject will only put its "name" into the resulting dict, 359 while "foo" all of its fields except "id". 360 361 Options can also specify whether to include foo_id when displaying 362 related foo object (__included_ids__, defaults to True). 363 This doesn"t apply when __columns_only__ is specified. 364 """ 365 366 result = {} 367 if options is None: 368 options = {} 369 columns = self.serializable_attributes 370 371 if "__columns_only__" in options: 372 columns = options["__columns_only__"] 373 else: 374 columns = set(columns) 375 if "__columns_except__" in options: 376 columns_except = options["__columns_except__"] 377 if not isinstance(options["__columns_except__"], list): 378 columns_except = [options["__columns_except__"]] 379 380 columns -= set(columns_except) 381 382 if ("__included_ids__" in options and 383 options["__included_ids__"] is False): 384 385 related_objs_ids = [ 386 r + "_id" for r, _ in options.items() 387 if not r.startswith("__")] 388 389 columns -= set(related_objs_ids) 390 391 columns = list(columns) 392 393 for column in columns: 394 result[column] = getattr(self, column) 395 396 for related, values in options.items(): 397 if hasattr(self, related): 398 result[related] = getattr(self, related).to_dict(values) 399 return result
400 401 @property
402 - def serializable_attributes(self):
403 return map(lambda x: x.name, self.__table__.columns)
404
405 406 -class RedisConnectionProvider(object):
407 - def __init__(self, config):
408 self.host = config.get("REDIS_HOST", "127.0.0.1") 409 self.port = int(config.get("REDIS_PORT", "6379"))
410
411 - def get_connection(self):
412 return StrictRedis(host=self.host, port=self.port)
413
414 415 -def get_redis_connection():
416 """ 417 Creates connection to redis, now we use default instance at localhost, no config needed 418 """ 419 return StrictRedis()
420
421 422 -def dt_to_unixtime(dt):
423 """ 424 Converts datetime to unixtime 425 :param dt: DateTime instance 426 :rtype: float 427 """ 428 return float(dt.strftime('%s'))
429
430 431 -def string_dt_to_unixtime(dt_string):
432 """ 433 Converts datetime to unixtime from string 434 :param dt_string: datetime string 435 :rtype: str 436 """ 437 return dt_to_unixtime(dt_parser.parse(dt_string))
438
439 440 -def is_ip_from_builder_net(ip):
441 """ 442 Checks is ip is owned by the builders network 443 :param str ip: IPv4 address 444 :return bool: True 445 """ 446 ip_addr = IPAddress(ip) 447 for subnet in app.config.get("BUILDER_IPS", ["127.0.0.1/24"]): 448 if ip_addr in IPNetwork(subnet): 449 return True 450 451 return False
452
453 454 -def str2bool(v):
455 if v is None: 456 return False 457 return v.lower() in ("yes", "true", "t", "1")
458
459 460 -def copr_url(view, copr, **kwargs):
461 """ 462 Examine given copr and generate proper URL for the `view` 463 464 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters, 465 and therefore you should *not* pass them manually. 466 467 Usage: 468 copr_url("coprs_ns.foo", copr) 469 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz) 470 """ 471 if copr.is_a_group_project: 472 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs) 473 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
474
475 476 -def url_for_copr_view(view, group_view, copr, **kwargs):
477 if copr.is_a_group_project: 478 return url_for(group_view, group_name=copr.group.name, coprname=copr.name, **kwargs) 479 else: 480 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
481
482 483 -def url_for_copr_builds(copr):
484 if copr.is_a_group_project: 485 return url_for("coprs_ns.group_copr_builds", 486 group_name=copr.group.name, coprname=copr.name) 487 else: 488 return url_for("coprs_ns.copr_builds", 489 username=copr.user.username, coprname=copr.name)
490 491 492 from sqlalchemy.engine.default import DefaultDialect 493 from sqlalchemy.sql.sqltypes import String, DateTime, NullType 494 495 # python2/3 compatible. 496 PY3 = str is not bytes 497 text = str if PY3 else unicode 498 int_type = int if PY3 else (int, long) 499 str_type = str if PY3 else (str, unicode)
500 501 502 -class StringLiteral(String):
503 """Teach SA how to literalize various things."""
504 - def literal_processor(self, dialect):
505 super_processor = super(StringLiteral, self).literal_processor(dialect) 506 507 def process(value): 508 if isinstance(value, int_type): 509 return text(value) 510 if not isinstance(value, str_type): 511 value = text(value) 512 result = super_processor(value) 513 if isinstance(result, bytes): 514 result = result.decode(dialect.encoding) 515 return result
516 return process
517
518 519 -class LiteralDialect(DefaultDialect):
520 colspecs = { 521 # prevent various encoding explosions 522 String: StringLiteral, 523 # teach SA about how to literalize a datetime 524 DateTime: StringLiteral, 525 # don't format py2 long integers to NULL 526 NullType: StringLiteral, 527 }
528
529 530 -def literal_query(statement):
531 """NOTE: This is entirely insecure. DO NOT execute the resulting strings.""" 532 import sqlalchemy.orm 533 if isinstance(statement, sqlalchemy.orm.Query): 534 statement = statement.statement 535 return statement.compile( 536 dialect=LiteralDialect(), 537 compile_kwargs={'literal_binds': True}, 538 ).string
539
540 541 -def stream_template(template_name, **context):
542 app.update_template_context(context) 543 t = app.jinja_env.get_template(template_name) 544 rv = t.stream(context) 545 rv.enable_buffering(2) 546 return rv
547
548 549 -def generate_repo_name(repo_url):
550 """ based on url, generate repo name """ 551 repo_url = re.sub("[^a-zA-Z0-9]", '_', repo_url) 552 repo_url = re.sub("(__*)", '_', repo_url) 553 repo_url = re.sub("(_*$)|^_*", '', repo_url) 554 return repo_url
555
556 557 -def pre_process_repo_url(chroot, repo_url):
558 """ 559 Expands variables and sanitize repo url to be used for mock config 560 """ 561 parsed_url = urlparse(repo_url) 562 if parsed_url.scheme == "copr": 563 user = parsed_url.netloc 564 prj = parsed_url.path.split("/")[1] 565 repo_url = "/".join([ 566 flask.current_app.config["BACKEND_BASE_URL"], 567 "results", user, prj, chroot 568 ]) + "/" 569 570 repo_url = repo_url.replace("$chroot", chroot) 571 repo_url = repo_url.replace("$distname", chroot.split("-")[0]) 572 return repo_url
573
574 575 -def parse_repo_params(repo, supported_keys=None):
576 """ 577 :param repo: str repo from Copr/CoprChroot/Build/... 578 :param supported_keys list of supported optional parameters 579 :return: dict of optional parameters parsed from the repo URL 580 """ 581 supported_keys = supported_keys or ["priority"] 582 if not repo.startswith("copr://"): 583 return {} 584 585 params = {} 586 qs = parse_qs(urlparse(repo).query) 587 for k, v in qs.items(): 588 if k in supported_keys: 589 # parse_qs returns values as lists, but we allow setting the param only once, 590 # so we can take just first value from it 591 value = int(v[0]) if v[0].isnumeric() else v[0] 592 params[k] = value 593 return params
594
595 596 -def generate_build_config(copr, chroot_id):
597 """ Return dict with proper build config contents """ 598 chroot = None 599 for i in copr.copr_chroots: 600 if i.mock_chroot.name == chroot_id: 601 chroot = i 602 if not chroot: 603 return {} 604 605 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs 606 607 repos = [{ 608 "id": "copr_base", 609 "url": copr.repo_url + "/{}/".format(chroot_id), 610 "name": "Copr repository", 611 }] 612 613 if not copr.auto_createrepo: 614 repos.append({ 615 "id": "copr_base_devel", 616 "url": copr.repo_url + "/{}/devel/".format(chroot_id), 617 "name": "Copr buildroot", 618 }) 619 620 def get_additional_repo_views(repos_list): 621 repos = [] 622 for repo in repos_list: 623 params = parse_repo_params(repo) 624 repo_view = { 625 "id": generate_repo_name(repo), 626 "url": pre_process_repo_url(chroot_id, repo), 627 "name": "Additional repo " + generate_repo_name(repo), 628 } 629 repo_view.update(params) 630 repos.append(repo_view) 631 return repos
632 633 repos.extend(get_additional_repo_views(copr.repos_list)) 634 repos.extend(get_additional_repo_views(chroot.repos_list)) 635 636 return { 637 'project_id': copr.repo_id, 638 'additional_packages': packages.split(), 639 'repos': repos, 640 'chroot': chroot_id, 641 'use_bootstrap_container': copr.use_bootstrap_container 642 } 643