1 import tempfile
2 import shutil
3 import json
4 import os
5 import pprint
6 import time
7 import flask
8 import sqlite3
9 from sqlalchemy.sql import text
10 from sqlalchemy import or_
11 from sqlalchemy import and_
12 from sqlalchemy.orm import joinedload
13 from sqlalchemy.orm.exc import NoResultFound
14 from sqlalchemy.sql import false,true
15 from werkzeug.utils import secure_filename
16 from sqlalchemy import desc,asc, bindparam, Integer
17 from collections import defaultdict
18
19 from coprs import app
20 from coprs import db
21 from coprs import exceptions
22 from coprs import models
23 from coprs import helpers
24 from coprs.constants import DEFAULT_BUILD_TIMEOUT, MAX_BUILD_TIMEOUT
25 from coprs.exceptions import MalformedArgumentException, ActionInProgressException, InsufficientRightsException, UnrepeatableBuildException
26 from coprs.helpers import StatusEnum
27
28 from coprs.logic import coprs_logic
29 from coprs.logic import users_logic
30 from coprs.logic.actions_logic import ActionsLogic
31 from coprs.models import BuildChroot,Build,Package,MockChroot
32 from .coprs_logic import MockChrootsLogic
33
34 log = app.logger
38 @classmethod
39 - def get(cls, build_id):
41
42 @classmethod
53
54 @classmethod
65
66 @classmethod
85
86 @classmethod
94
95 @classmethod
97 end = int(time.time())
98 start = end - 86399
99 step = 3600
100 tasks = cls.get_tasks_by_time(start, end)
101 steps = int(round((end - start) / step + 0.5))
102 current_step = 0
103
104 data = [[0] * (steps + 1)]
105 data[0][0] = ''
106 for t in tasks:
107 task = t.to_dict()
108 while task['started_on'] > start + step * (current_step + 1):
109 current_step += 1
110 data[0][current_step + 1] += 1
111 return data
112
113 @classmethod
125
126 @classmethod
135
136 @classmethod
152
153 @classmethod
162
163 @classmethod
166
167 @classmethod
170
171 @classmethod
176
177 @classmethod
184
185 @classmethod
187 if db.engine.url.drivername == "sqlite":
188 return
189
190 status_to_order = """
191 CREATE OR REPLACE FUNCTION status_to_order (x integer)
192 RETURNS integer AS $$ BEGIN
193 RETURN CASE WHEN x = 3 THEN 1
194 WHEN x = 6 THEN 2
195 WHEN x = 7 THEN 3
196 WHEN x = 4 THEN 4
197 WHEN x = 0 THEN 5
198 WHEN x = 1 THEN 6
199 WHEN x = 5 THEN 7
200 WHEN x = 2 THEN 8
201 WHEN x = 8 THEN 9
202 WHEN x = 9 THEN 10
203 ELSE x
204 END; END;
205 $$ LANGUAGE plpgsql;
206 """
207
208 order_to_status = """
209 CREATE OR REPLACE FUNCTION order_to_status (x integer)
210 RETURNS integer AS $$ BEGIN
211 RETURN CASE WHEN x = 1 THEN 3
212 WHEN x = 2 THEN 6
213 WHEN x = 3 THEN 7
214 WHEN x = 4 THEN 4
215 WHEN x = 5 THEN 0
216 WHEN x = 6 THEN 1
217 WHEN x = 7 THEN 5
218 WHEN x = 8 THEN 2
219 WHEN x = 9 THEN 8
220 WHEN x = 10 THEN 9
221 ELSE x
222 END; END;
223 $$ LANGUAGE plpgsql;
224 """
225
226 db.engine.connect()
227 db.engine.execute(status_to_order)
228 db.engine.execute(order_to_status)
229
230 @classmethod
232 query_select = """
233 SELECT build.id, build.source_status, MAX(package.name) AS pkg_name, build.pkg_version, build.submitted_on,
234 MIN(statuses.started_on) AS started_on, MAX(statuses.ended_on) AS ended_on, order_to_status(MIN(statuses.st)) AS status,
235 build.canceled, MIN("group".name) AS group_name, MIN(copr.name) as copr_name, MIN("user".username) as user_name
236 FROM build
237 LEFT OUTER JOIN package
238 ON build.package_id = package.id
239 LEFT OUTER JOIN (SELECT build_chroot.build_id, started_on, ended_on, status_to_order(status) AS st FROM build_chroot) AS statuses
240 ON statuses.build_id=build.id
241 LEFT OUTER JOIN copr
242 ON copr.id = build.copr_id
243 LEFT OUTER JOIN "user"
244 ON copr.user_id = "user".id
245 LEFT OUTER JOIN "group"
246 ON copr.group_id = "group".id
247 WHERE build.copr_id = :copr_id
248 GROUP BY
249 build.id;
250 """
251
252 if db.engine.url.drivername == "sqlite":
253 def sqlite_status_to_order(x):
254 if x == 3:
255 return 1
256 elif x == 6:
257 return 2
258 elif x == 7:
259 return 3
260 elif x == 4:
261 return 4
262 elif x == 0:
263 return 5
264 elif x == 1:
265 return 6
266 elif x == 5:
267 return 7
268 elif x == 2:
269 return 8
270 elif x == 8:
271 return 9
272 elif x == 9:
273 return 10
274 return 1000
275
276 def sqlite_order_to_status(x):
277 if x == 1:
278 return 3
279 elif x == 2:
280 return 6
281 elif x == 3:
282 return 7
283 elif x == 4:
284 return 4
285 elif x == 5:
286 return 0
287 elif x == 6:
288 return 1
289 elif x == 7:
290 return 5
291 elif x == 8:
292 return 2
293 elif x == 9:
294 return 8
295 elif x == 10:
296 return 9
297 return 1000
298
299 conn = db.engine.connect()
300 conn.connection.create_function("status_to_order", 1, sqlite_status_to_order)
301 conn.connection.create_function("order_to_status", 1, sqlite_order_to_status)
302 statement = text(query_select)
303 statement.bindparams(bindparam("copr_id", Integer))
304 result = conn.execute(statement, {"copr_id": copr.id})
305 else:
306 statement = text(query_select)
307 statement.bindparams(bindparam("copr_id", Integer))
308 result = db.engine.execute(statement, {"copr_id": copr.id})
309
310 return result
311
312 @classmethod
315
316 @classmethod
324
325 @classmethod
328
329 @classmethod
332
333 @classmethod
353
354 @classmethod
370
371 @classmethod
372 - def create_new_from_scm(cls, user, copr, scm_type, clone_url,
373 committish='', subdirectory='', spec='', srpm_build_method='rpkg',
374 chroot_names=None, **build_options):
375 """
376 :type user: models.User
377 :type copr: models.Copr
378
379 :type chroot_names: List[str]
380
381 :rtype: models.Build
382 """
383 source_type = helpers.BuildSourceEnum("scm")
384 source_json = json.dumps({"type": scm_type,
385 "clone_url": clone_url,
386 "committish": committish,
387 "subdirectory": subdirectory,
388 "spec": spec,
389 "srpm_build_method": srpm_build_method})
390 return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
391
392 @classmethod
393 - def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, python_versions,
394 chroot_names=None, **build_options):
411
412 @classmethod
425
426 @classmethod
427 - def create_new_from_custom(cls, user, copr,
428 script, script_chroot=None, script_builddeps=None,
429 script_resultdir=None, chroot_names=None, **kwargs):
430 """
431 :type user: models.User
432 :type copr: models.Copr
433 :type script: str
434 :type script_chroot: str
435 :type script_builddeps: str
436 :type script_resultdir: str
437 :type chroot_names: List[str]
438 :rtype: models.Build
439 """
440 source_type = helpers.BuildSourceEnum("custom")
441 source_dict = {
442 'script': script,
443 'chroot': script_chroot,
444 'builddeps': script_builddeps,
445 'resultdir': script_resultdir,
446 }
447
448 return cls.create_new(user, copr, source_type, json.dumps(source_dict),
449 chroot_names, **kwargs)
450
451 @classmethod
452 - def create_new_from_upload(cls, user, copr, f_uploader, orig_filename,
453 chroot_names=None, **build_options):
454 """
455 :type user: models.User
456 :type copr: models.Copr
457 :param f_uploader(file_path): function which stores data at the given `file_path`
458 :return:
459 """
460 tmp = tempfile.mkdtemp(dir=app.config["STORAGE_DIR"])
461 tmp_name = os.path.basename(tmp)
462 filename = secure_filename(orig_filename)
463 file_path = os.path.join(tmp, filename)
464 f_uploader(file_path)
465
466
467 pkg_url = "{baseurl}/tmp/{tmp_dir}/{filename}".format(
468 baseurl=app.config["PUBLIC_COPR_BASE_URL"],
469 tmp_dir=tmp_name,
470 filename=filename)
471
472
473 source_type = helpers.BuildSourceEnum("upload")
474 source_json = json.dumps({"url": pkg_url, "pkg": filename, "tmp": tmp_name})
475 srpm_url = None if pkg_url.endswith('.spec') else pkg_url
476
477 try:
478 build = cls.create_new(user, copr, source_type, source_json,
479 chroot_names, pkgs=pkg_url, srpm_url=srpm_url, **build_options)
480 except Exception:
481 shutil.rmtree(tmp)
482 raise
483
484 return build
485
486 @classmethod
487 - def create_new(cls, user, copr, source_type, source_json, chroot_names=None, pkgs="",
488 git_hashes=None, skip_import=False, background=False, batch=None,
489 srpm_url=None, **build_options):
490 """
491 :type user: models.User
492 :type copr: models.Copr
493 :type chroot_names: List[str]
494 :type source_type: int value from helpers.BuildSourceEnum
495 :type source_json: str in json format
496 :type pkgs: str
497 :type git_hashes: dict
498 :type skip_import: bool
499 :type background: bool
500 :type batch: models.Batch
501 :rtype: models.Build
502 """
503 if chroot_names is None:
504 chroots = [c for c in copr.active_chroots]
505 else:
506 chroots = []
507 for chroot in copr.active_chroots:
508 if chroot.name in chroot_names:
509 chroots.append(chroot)
510
511 build = cls.add(
512 user=user,
513 pkgs=pkgs,
514 copr=copr,
515 chroots=chroots,
516 source_type=source_type,
517 source_json=source_json,
518 enable_net=build_options.get("enable_net", copr.build_enable_net),
519 background=background,
520 git_hashes=git_hashes,
521 skip_import=skip_import,
522 batch=batch,
523 srpm_url=srpm_url,
524 )
525
526 if user.proven:
527 if "timeout" in build_options:
528 build.timeout = build_options["timeout"]
529
530 return build
531
532 @classmethod
533 - def add(cls, user, pkgs, copr, source_type=None, source_json=None,
534 repos=None, chroots=None, timeout=None, enable_net=True,
535 git_hashes=None, skip_import=False, background=False, batch=None,
536 srpm_url=None):
537
538 if chroots is None:
539 chroots = []
540
541 coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action(
542 copr, "Can't build while there is an operation in progress: {action}")
543 users_logic.UsersLogic.raise_if_cant_build_in_copr(
544 user, copr,
545 "You don't have permissions to build in this copr.")
546
547 if not repos:
548 repos = copr.repos
549
550
551 if pkgs and (" " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs):
552 raise exceptions.MalformedArgumentException("Trying to create a build using src_pkg "
553 "with bad characters. Forgot to split?")
554
555
556 if not source_type or not source_json:
557 source_type = helpers.BuildSourceEnum("link")
558 source_json = json.dumps({"url":pkgs})
559
560 if skip_import and srpm_url:
561 chroot_status = StatusEnum("pending")
562 source_status = StatusEnum("succeeded")
563 elif srpm_url:
564 chroot_status = StatusEnum("waiting")
565 source_status = StatusEnum("importing")
566 else:
567 chroot_status = StatusEnum("waiting")
568 source_status = StatusEnum("pending")
569
570 build = models.Build(
571 user=user,
572 pkgs=pkgs,
573 copr=copr,
574 repos=repos,
575 source_type=source_type,
576 source_json=source_json,
577 source_status=source_status,
578 submitted_on=int(time.time()),
579 enable_net=bool(enable_net),
580 is_background=bool(background),
581 batch=batch,
582 srpm_url=srpm_url,
583 )
584
585 if timeout:
586 build.timeout = timeout or DEFAULT_BUILD_TIMEOUT
587
588 db.session.add(build)
589
590
591
592 if not chroots:
593 chroots = copr.active_chroots
594
595 for chroot in chroots:
596 git_hash = None
597 if git_hashes:
598 git_hash = git_hashes.get(chroot.name)
599 buildchroot = models.BuildChroot(
600 build=build,
601 status=chroot_status,
602 mock_chroot=chroot,
603 git_hash=git_hash,
604 )
605 db.session.add(buildchroot)
606
607 return build
608
609 @classmethod
611 source_dict = package.source_json_dict
612 source_dict.update(source_dict_update)
613 source_json = json.dumps(source_dict)
614
615 build = models.Build(
616 user=None,
617 pkgs=None,
618 package_id=package.id,
619 copr=package.copr,
620 repos=package.copr.repos,
621 source_status=helpers.StatusEnum("pending"),
622 source_type=package.source_type,
623 source_json=source_json,
624 submitted_on=int(time.time()),
625 enable_net=package.copr.build_enable_net,
626 timeout=DEFAULT_BUILD_TIMEOUT
627 )
628
629 db.session.add(build)
630
631 chroots = package.copr.active_chroots
632
633 status = helpers.StatusEnum("waiting")
634
635 for chroot in chroots:
636 buildchroot = models.BuildChroot(
637 build=build,
638 status=status,
639 mock_chroot=chroot,
640 git_hash=None
641 )
642
643 db.session.add(buildchroot)
644
645 return build
646
647
648 terminal_states = {StatusEnum("failed"), StatusEnum("succeeded"), StatusEnum("canceled")}
649
650 @classmethod
662
663
664 @classmethod
666 """
667 Deletes the locally stored data for build purposes. This is typically
668 uploaded srpm file, uploaded spec file or webhook POST content.
669 """
670
671 data = json.loads(build.source_json)
672 if 'tmp' in data:
673 tmp = data["tmp"]
674 storage_path = app.config["STORAGE_DIR"]
675 try:
676 shutil.rmtree(os.path.join(storage_path, tmp))
677 except:
678 pass
679
680
681 @classmethod
683 """
684 :param build:
685 :param upd_dict:
686 example:
687 {
688 "builds":[
689 {
690 "id": 1,
691 "copr_id": 2,
692 "started_on": 139086644000
693 },
694 {
695 "id": 2,
696 "copr_id": 1,
697 "status": 0,
698 "chroot": "fedora-18-x86_64",
699 "results": "http://server/results/foo/bar/",
700 "ended_on": 139086644000
701 }]
702 }
703 """
704 log.info("Updating build {} by: {}".format(build.id, upd_dict))
705
706
707 for attr in ["results", "built_packages", "srpm_url"]:
708 value = upd_dict.get(attr, None)
709 if value:
710 setattr(build, attr, value)
711
712
713 if upd_dict.get("task_id") == build.task_id:
714 if upd_dict.get("status") == StatusEnum("succeeded"):
715 new_status = StatusEnum("importing")
716 else:
717 new_status = upd_dict.get("status")
718
719 build.source_status = new_status
720
721 if new_status == StatusEnum("failed") or \
722 new_status == StatusEnum("skipped"):
723 for ch in build.build_chroots:
724 ch.status = new_status
725 ch.ended_on = upd_dict.get("ended_on") or time.time()
726 db.session.add(ch)
727
728 if new_status == StatusEnum("failed"):
729 build.fail_type = helpers.FailTypeEnum("srpm_build_error")
730
731 db.session.add(build)
732 return
733
734 if "chroot" in upd_dict:
735
736 for build_chroot in build.build_chroots:
737 if build_chroot.name == upd_dict["chroot"]:
738 if "status" in upd_dict and build_chroot.status not in BuildsLogic.terminal_states:
739 build_chroot.status = upd_dict["status"]
740
741 if upd_dict.get("status") in BuildsLogic.terminal_states:
742 build_chroot.ended_on = upd_dict.get("ended_on") or time.time()
743
744 if upd_dict.get("status") == StatusEnum("starting"):
745 build_chroot.started_on = upd_dict.get("started_on") or time.time()
746
747 db.session.add(build_chroot)
748
749
750
751 if (build.module
752 and upd_dict.get("status") == StatusEnum("succeeded")
753 and all(b.status == StatusEnum("succeeded") for b in build.module.builds)):
754 ActionsLogic.send_build_module(build.copr, build.module)
755
756 db.session.add(build)
757
758 @classmethod
779
780 @classmethod
781 - def delete_build(cls, user, build, send_delete_action=True):
802
803 @classmethod
813
814 @classmethod
833
834 @classmethod
842
843 @classmethod
846
849 @classmethod
858
859 @classmethod
870
871 @classmethod
874
875 @classmethod
878
879 @classmethod
882
883 @classmethod
886
887 @classmethod
890
893 @classmethod
895 query = """
896 SELECT
897 package.id as package_id,
898 package.name AS package_name,
899 build.id AS build_id,
900 build_chroot.status AS build_chroot_status,
901 build.pkg_version AS build_pkg_version,
902 mock_chroot.id AS mock_chroot_id,
903 mock_chroot.os_release AS mock_chroot_os_release,
904 mock_chroot.os_version AS mock_chroot_os_version,
905 mock_chroot.arch AS mock_chroot_arch
906 FROM package
907 JOIN (SELECT
908 MAX(build.id) AS max_build_id_for_chroot,
909 build.package_id AS package_id,
910 build_chroot.mock_chroot_id AS mock_chroot_id
911 FROM build
912 JOIN build_chroot
913 ON build.id = build_chroot.build_id
914 WHERE build.copr_id = {copr_id}
915 AND build_chroot.status != 2
916 GROUP BY build.package_id,
917 build_chroot.mock_chroot_id) AS max_build_ids_for_a_chroot
918 ON package.id = max_build_ids_for_a_chroot.package_id
919 JOIN build
920 ON build.id = max_build_ids_for_a_chroot.max_build_id_for_chroot
921 JOIN build_chroot
922 ON build_chroot.mock_chroot_id = max_build_ids_for_a_chroot.mock_chroot_id
923 AND build_chroot.build_id = max_build_ids_for_a_chroot.max_build_id_for_chroot
924 JOIN mock_chroot
925 ON mock_chroot.id = max_build_ids_for_a_chroot.mock_chroot_id
926 ORDER BY package.name ASC, package.id ASC, mock_chroot.os_release ASC, mock_chroot.os_version ASC, mock_chroot.arch ASC
927 """.format(copr_id=copr.id)
928 rows = db.session.execute(query)
929 return rows
930