Package coprs :: Package logic :: Module builds_logic
[hide private]
[frames] | no frames]

Source Code for Module coprs.logic.builds_logic

  1  import tempfile 
  2  import shutil 
  3  import json 
  4  import os 
  5  import pprint 
  6  import time 
  7  import flask 
  8  import sqlite3 
  9  from sqlalchemy.sql import text 
 10  from sqlalchemy import or_ 
 11  from sqlalchemy import and_ 
 12  from sqlalchemy.orm import joinedload 
 13  from sqlalchemy.orm.exc import NoResultFound 
 14  from sqlalchemy.sql import false,true 
 15  from werkzeug.utils import secure_filename 
 16  from sqlalchemy import desc,asc, bindparam, Integer 
 17  from collections import defaultdict 
 18   
 19  from coprs import app 
 20  from coprs import db 
 21  from coprs import exceptions 
 22  from coprs import models 
 23  from coprs import helpers 
 24  from coprs.constants import DEFAULT_BUILD_TIMEOUT, MAX_BUILD_TIMEOUT, MAX_PRIO 
 25  from coprs.exceptions import MalformedArgumentException, ActionInProgressException, InsufficientRightsException 
 26  from coprs.helpers import StatusEnum 
 27   
 28  from coprs.logic import coprs_logic 
 29  from coprs.logic import users_logic 
 30  from coprs.logic.actions_logic import ActionsLogic 
 31  from coprs.models import BuildChroot,Build,Package,MockChroot 
 32  from .coprs_logic import MockChrootsLogic 
 33   
 34  log = app.logger 
35 36 37 -class BuildsLogic(object):
38 @classmethod
39 - def get(cls, build_id):
40 return models.Build.query.filter(models.Build.id == build_id)
41 42 # todo: move methods operating with BuildChroot to BuildChrootLogic 43 @classmethod
44 - def get_build_tasks(cls, status, background=None):
45 """ Returns tasks with given status. If background is specified then 46 returns normal jobs (false) or background jobs (true) 47 """ 48 result = models.BuildChroot.query.join(models.Build)\ 49 .filter(models.BuildChroot.status == status)\ 50 .order_by(models.BuildChroot.build_id.asc()) 51 if background is not None: 52 result = result.filter(models.Build.is_background == (true() if background else false())) 53 return result
54 55 @classmethod
56 - def get_recent_tasks(cls, user=None, limit=None):
57 if not limit: 58 limit = 100 59 60 query = models.Build.query 61 if user is not None: 62 query = query.filter(models.Build.user_id == user.id) 63 64 query = query.join( 65 models.BuildChroot.query 66 .filter(models.BuildChroot.ended_on.isnot(None)) 67 .order_by(models.BuildChroot.ended_on.desc()) 68 .subquery() 69 ).order_by(models.Build.id.desc()) 70 71 # Workaround - otherwise it could take less records than `limit`even though there are more of them. 72 query = query.limit(limit if limit > 100 else 100) 73 return list(query.all()[:4])
74 75 @classmethod
77 """ 78 Returns Builds which are waiting to be uploaded to dist git 79 """ 80 query = (models.Build.query.join(models.BuildChroot) 81 .filter(models.Build.canceled == false()) 82 .filter(models.BuildChroot.status == helpers.StatusEnum("importing")) 83 .filter(models.Build.srpm_url.isnot(None)) 84 ) 85 query = query.order_by(models.BuildChroot.build_id.asc()) 86 return query
87 88 @classmethod
89 - def get_build_task_queue(cls, is_background=False): # deprecated
90 """ 91 Returns BuildChroots which are - waiting to be built or 92 - older than 2 hours and unfinished 93 """ 94 # todo: filter out build without package 95 query = (models.BuildChroot.query.join(models.Build) 96 .filter(models.Build.canceled == false()) 97 .filter(models.Build.is_background == (true() if is_background else false())) 98 .filter(or_( 99 models.BuildChroot.status == helpers.StatusEnum("pending"), 100 models.BuildChroot.status == helpers.StatusEnum("starting"), 101 and_( 102 # We are moving ended_on to the BuildChroot, now it should be reliable, 103 # so we don't want to reschedule failed chroots 104 # models.BuildChroot.status.in_([ 105 # # Bug 1206562 - Cannot delete Copr because it incorrectly thinks 106 # # there are unfinished builds. Solution: `failed` but unfinished 107 # # (ended_on is null) builds should be rescheduled. 108 # # todo: we need to be sure that correct `failed` set is set together with `ended_on` 109 # helpers.StatusEnum("running"), 110 # helpers.StatusEnum("failed") 111 #]), 112 models.BuildChroot.status == helpers.StatusEnum("running"), 113 models.BuildChroot.started_on < int(time.time() - 1.1 * MAX_BUILD_TIMEOUT), 114 models.BuildChroot.ended_on.is_(None) 115 )) 116 )) 117 query = query.order_by(models.BuildChroot.build_id.asc()) 118 return query
119 120 121 @classmethod
122 - def get_select_srpm_build_tasks_query(cls, background=False):
123 return (models.Build.query.join(models.BuildChroot) 124 .filter(models.Build.srpm_url.is_(None)) 125 .filter(models.Build.canceled == false()) 126 .filter(models.BuildChroot.status == helpers.StatusEnum("importing")) 127 .filter(models.Build.is_background == (true() if background else false())))
128 129 @classmethod
130 - def select_srpm_build_task(cls, background=False):
131 return (cls.get_select_srpm_build_tasks_query(background) 132 .order_by(models.Build.priority.asc(), 133 models.Build.id.asc()) 134 .first())
135 136 @classmethod
137 - def get_srpm_build_task_lowest_priority(cls, background=False):
138 lowest_prio_task = (cls.get_select_srpm_build_tasks_query(background) 139 .order_by(models.Build.priority.desc()).first()) 140 return (lowest_prio_task.priority if lowest_prio_task else 0)
141 142 @classmethod
143 - def get_select_build_tasks_query(cls, background=False):
144 return (models.BuildChroot.query.join(models.Build) 145 .filter(models.Build.is_background == (true() if background else false())) 146 .filter(models.Build.canceled == false()) 147 .filter(or_( 148 models.BuildChroot.status == helpers.StatusEnum("pending"), 149 and_( 150 models.BuildChroot.status == helpers.StatusEnum("running"), 151 models.BuildChroot.started_on < int(time.time() - 1.1 * MAX_BUILD_TIMEOUT), 152 models.BuildChroot.ended_on.is_(None) 153 ) 154 )))
155 156 @classmethod
157 - def select_build_task(cls, background=False):
158 return (cls.get_select_build_tasks_query(background) 159 .order_by(models.BuildChroot.priority.asc(), 160 models.BuildChroot.build_id.asc()) 161 .first())
162 163 @classmethod
164 - def get_build_task_lowest_priority(cls, background=False):
165 lowest_prio_task = (cls.get_select_build_tasks_query(background) 166 .order_by(models.BuildChroot.priority.desc()).first()) 167 return (lowest_prio_task.priority if lowest_prio_task else 0)
168 169 @classmethod
170 - def get_task_lowest_priority(cls, background=False):
171 prio1 = cls.get_build_task_lowest_priority(background) 172 prio2 = cls.get_srpm_build_task_lowest_priority(background) 173 return max(prio1, prio2)
174 175 @classmethod
176 - def get_build_task(cls, task_id):
177 try: 178 build_id, chroot_name = task_id.split("-", 1) 179 except ValueError: 180 raise MalformedArgumentException("Invalid task_id {}".format(task_id)) 181 182 build_chroot = BuildChrootsLogic.get_by_build_id_and_name(build_id, chroot_name) 183 return build_chroot.join(models.Build).first()
184 185 @classmethod
186 - def get_srpm_build_task(cls, build_id):
187 return BuildsLogic.get_by_id(build_id).first()
188 189 @classmethod
190 - def defer(cls, build_id):
191 build = cls.get(build_id).first() 192 if not build: 193 return None 194 build.priority = (BuildsLogic.get_task_lowest_priority(build.is_background)+1)%MAX_PRIO 195 db.session.add(build) 196 return build
197 198 @classmethod
199 - def get_multiple(cls):
200 return models.Build.query.order_by(models.Build.id.desc())
201 202 @classmethod
203 - def get_multiple_by_copr(cls, copr):
204 """ Get collection of builds in copr sorted by build_id descending 205 """ 206 return cls.get_multiple().filter(models.Build.copr == copr)
207 208 @classmethod
209 - def get_multiple_by_user(cls, user):
210 """ Get collection of builds in copr sorted by build_id descending 211 form the copr belonging to `user` 212 """ 213 return cls.get_multiple().join(models.Build.copr).filter( 214 models.Copr.user == user)
215 216 217 @classmethod
218 - def init_db(cls):
219 if db.engine.url.drivername == "sqlite": 220 return 221 222 status_to_order = """ 223 CREATE OR REPLACE FUNCTION status_to_order (x integer) 224 RETURNS integer AS $$ BEGIN 225 RETURN CASE WHEN x = 0 THEN 0 226 WHEN x = 3 THEN 1 227 WHEN x = 6 THEN 2 228 WHEN x = 7 THEN 3 229 WHEN x = 4 THEN 4 230 WHEN x = 1 THEN 5 231 WHEN x = 5 THEN 6 232 ELSE 1000 233 END; END; 234 $$ LANGUAGE plpgsql; 235 """ 236 237 order_to_status = """ 238 CREATE OR REPLACE FUNCTION order_to_status (x integer) 239 RETURNS integer AS $$ BEGIN 240 RETURN CASE WHEN x = 0 THEN 0 241 WHEN x = 1 THEN 3 242 WHEN x = 2 THEN 6 243 WHEN x = 3 THEN 7 244 WHEN x = 4 THEN 4 245 WHEN x = 5 THEN 1 246 WHEN x = 6 THEN 5 247 ELSE 1000 248 END; END; 249 $$ LANGUAGE plpgsql; 250 """ 251 252 db.engine.connect() 253 db.engine.execute(status_to_order) 254 db.engine.execute(order_to_status)
255 256 @classmethod
257 - def get_copr_builds_list(cls, copr):
258 query_select = """ 259 SELECT build.id, MAX(package.name) AS pkg_name, build.pkg_version, build.submitted_on, 260 MIN(statuses.started_on) AS started_on, MAX(statuses.ended_on) AS ended_on, order_to_status(MIN(statuses.st)) AS status, 261 build.canceled, MIN("group".name) AS group_name, MIN(copr.name) as copr_name, MIN("user".username) as user_name 262 FROM build 263 LEFT OUTER JOIN package 264 ON build.package_id = package.id 265 LEFT OUTER JOIN (SELECT build_chroot.build_id, started_on, ended_on, status_to_order(status) AS st FROM build_chroot) AS statuses 266 ON statuses.build_id=build.id 267 LEFT OUTER JOIN copr 268 ON copr.id = build.copr_id 269 LEFT OUTER JOIN "user" 270 ON copr.user_id = "user".id 271 LEFT OUTER JOIN "group" 272 ON copr.group_id = "group".id 273 WHERE build.copr_id = :copr_id 274 GROUP BY 275 build.id; 276 """ 277 278 if db.engine.url.drivername == "sqlite": 279 def sqlite_status_to_order(x): 280 if x == 3: 281 return 1 282 elif x == 6: 283 return 2 284 elif x == 7: 285 return 3 286 elif x == 4: 287 return 4 288 elif x == 0: 289 return 5 290 elif x == 1: 291 return 6 292 elif x == 5: 293 return 7 294 elif x == 8: 295 return 8 296 return 1000
297 298 def sqlite_order_to_status(x): 299 if x == 1: 300 return 3 301 elif x == 2: 302 return 6 303 elif x == 3: 304 return 7 305 elif x == 4: 306 return 4 307 elif x == 5: 308 return 0 309 elif x == 6: 310 return 1 311 elif x == 7: 312 return 5 313 elif x == 8: 314 return 8 315 return 1000 316 317 conn = db.engine.connect() 318 conn.connection.create_function("status_to_order", 1, sqlite_status_to_order) 319 conn.connection.create_function("order_to_status", 1, sqlite_order_to_status) 320 statement = text(query_select) 321 statement.bindparams(bindparam("copr_id", Integer)) 322 result = conn.execute(statement, {"copr_id": copr.id}) 323 else: 324 statement = text(query_select) 325 statement.bindparams(bindparam("copr_id", Integer)) 326 result = db.engine.execute(statement, {"copr_id": copr.id}) 327 328 return result 329 330 @classmethod
331 - def join_group(cls, query):
332 return query.join(models.Copr).outerjoin(models.Group)
333 334 @classmethod
335 - def get_multiple_by_name(cls, username, coprname):
336 query = cls.get_multiple() 337 return (query.join(models.Build.copr) 338 .options(db.contains_eager(models.Build.copr)) 339 .join(models.Copr.user) 340 .filter(models.Copr.name == coprname) 341 .filter(models.User.username == username))
342 343 @classmethod
344 - def get_importing(cls):
345 """ 346 Return builds that are waiting for dist git to import the sources. 347 """ 348 query = (models.Build.query.join(models.Build.copr) 349 .join(models.User) 350 .join(models.BuildChroot) 351 .options(db.contains_eager(models.Build.copr)) 352 .options(db.contains_eager("copr.user")) 353 .filter((models.BuildChroot.started_on == None) 354 | (models.BuildChroot.started_on < int(time.time() - 7200))) 355 .filter(models.BuildChroot.ended_on == None) 356 .filter(models.Build.canceled == False) 357 .order_by(models.Build.submitted_on.asc())) 358 return query
359 360 @classmethod
361 - def get_waiting(cls):
362 """ 363 Return builds that aren't both started and finished 364 (if build start submission fails, we still want to mark 365 the build as non-waiting, if it ended) 366 this has very different goal then get_multiple, so implement it alone 367 """ 368 369 query = (models.Build.query.join(models.Build.copr) 370 .join(models.User).join(models.BuildChroot) 371 .options(db.contains_eager(models.Build.copr)) 372 .options(db.contains_eager("copr.user")) 373 .filter((models.BuildChroot.started_on.is_(None)) 374 | (models.BuildChroot.started_on < int(time.time() - 7200))) 375 .filter(models.BuildChroot.ended_on.is_(None)) 376 .filter(models.Build.canceled == false()) 377 .order_by(models.Build.submitted_on.asc())) 378 return query
379 380 @classmethod
381 - def get_by_ids(cls, ids):
382 return models.Build.query.filter(models.Build.id.in_(ids))
383 384 @classmethod
385 - def get_by_id(cls, build_id):
386 return models.Build.query.filter(models.Build.id == build_id)
387 388 @classmethod
389 - def create_new_from_other_build(cls, user, copr, source_build, 390 chroot_names=None, **build_options):
391 skip_import = False 392 git_hashes = {} 393 394 if source_build.source_type == helpers.BuildSourceEnum('upload'): 395 # I don't have the source 396 # so I don't want to import anything, just rebuild what's in dist git 397 skip_import = True 398 399 for chroot in source_build.build_chroots: 400 if not chroot.git_hash: 401 # I got an old build from time we didn't use dist git 402 # So I'll submit it as a new build using it's link 403 skip_import = False 404 git_hashes = None 405 flask.flash("This build is not in Dist Git. Trying to import the package again.") 406 break 407 git_hashes[chroot.name] = chroot.git_hash 408 409 build = cls.create_new(user, copr, source_build.source_type, source_build.source_json, chroot_names, 410 pkgs=source_build.pkgs, git_hashes=git_hashes, skip_import=skip_import, 411 srpm_url=source_build.srpm_url, **build_options) 412 build.package_id = source_build.package_id 413 build.pkg_version = source_build.pkg_version 414 return build
415 416 @classmethod
417 - def create_new_from_url(cls, user, copr, url, 418 chroot_names=None, **build_options):
419 """ 420 :type user: models.User 421 :type copr: models.Copr 422 423 :type chroot_names: List[str] 424 425 :rtype: models.Build 426 """ 427 source_type = helpers.BuildSourceEnum("link") 428 source_json = json.dumps({"url": url}) 429 srpm_url = None if url.endswith('.spec') else url 430 return cls.create_new(user, copr, source_type, source_json, chroot_names, 431 pkgs=url, srpm_url=srpm_url, **build_options)
432 433 @classmethod
434 - def create_new_from_scm(cls, user, copr, scm_type, clone_url, 435 committish='', subdirectory='', spec='', srpm_build_method='rpkg', 436 chroot_names=None, **build_options):
437 """ 438 :type user: models.User 439 :type copr: models.Copr 440 441 :type chroot_names: List[str] 442 443 :rtype: models.Build 444 """ 445 source_type = helpers.BuildSourceEnum("scm") 446 source_json = json.dumps({"type": scm_type, 447 "clone_url": clone_url, 448 "committish": committish, 449 "subdirectory": subdirectory, 450 "spec": spec, 451 "srpm_build_method": srpm_build_method}) 452 return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
453 454 @classmethod
455 - def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, python_versions, 456 chroot_names=None, **build_options):
457 """ 458 :type user: models.User 459 :type copr: models.Copr 460 :type package_name: str 461 :type version: str 462 :type python_versions: List[str] 463 464 :type chroot_names: List[str] 465 466 :rtype: models.Build 467 """ 468 source_type = helpers.BuildSourceEnum("pypi") 469 source_json = json.dumps({"pypi_package_name": pypi_package_name, 470 "pypi_package_version": pypi_package_version, 471 "python_versions": python_versions}) 472 return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
473 474 @classmethod
475 - def create_new_from_rubygems(cls, user, copr, gem_name, 476 chroot_names=None, **build_options):
477 """ 478 :type user: models.User 479 :type copr: models.Copr 480 :type gem_name: str 481 :type chroot_names: List[str] 482 :rtype: models.Build 483 """ 484 source_type = helpers.BuildSourceEnum("rubygems") 485 source_json = json.dumps({"gem_name": gem_name}) 486 return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
487 488 @classmethod
489 - def create_new_from_upload(cls, user, copr, f_uploader, orig_filename, 490 chroot_names=None, **build_options):
491 """ 492 :type user: models.User 493 :type copr: models.Copr 494 :param f_uploader(file_path): function which stores data at the given `file_path` 495 :return: 496 """ 497 tmp = tempfile.mkdtemp(dir=app.config["SRPM_STORAGE_DIR"]) 498 tmp_name = os.path.basename(tmp) 499 filename = secure_filename(orig_filename) 500 file_path = os.path.join(tmp, filename) 501 f_uploader(file_path) 502 503 # make the pkg public 504 pkg_url = "{baseurl}/tmp/{tmp_dir}/{filename}".format( 505 baseurl=app.config["PUBLIC_COPR_BASE_URL"], 506 tmp_dir=tmp_name, 507 filename=filename) 508 509 # create json describing the build source 510 source_type = helpers.BuildSourceEnum("upload") 511 source_json = json.dumps({"url": pkg_url, "pkg": filename, "tmp": tmp_name}) 512 srpm_url = None if pkg_url.endswith('.spec') else pkg_url 513 514 try: 515 build = cls.create_new(user, copr, source_type, source_json, 516 chroot_names, pkgs=pkg_url, srpm_url=srpm_url, **build_options) 517 except Exception: 518 shutil.rmtree(tmp) # todo: maybe we should delete in some cleanup procedure? 519 raise 520 521 return build
522 523 @classmethod
524 - def create_new(cls, user, copr, source_type, source_json, chroot_names=None, pkgs="", 525 git_hashes=None, skip_import=False, background=False, batch=None, 526 srpm_url=None, **build_options):
527 """ 528 :type user: models.User 529 :type copr: models.Copr 530 :type chroot_names: List[str] 531 :type source_type: int value from helpers.BuildSourceEnum 532 :type source_json: str in json format 533 :type pkgs: str 534 :type git_hashes: dict 535 :type skip_import: bool 536 :type background: bool 537 :type batch: models.Batch 538 :rtype: models.Build 539 """ 540 if chroot_names is None: 541 chroots = [c for c in copr.active_chroots] 542 else: 543 chroots = [] 544 for chroot in copr.active_chroots: 545 if chroot.name in chroot_names: 546 chroots.append(chroot) 547 548 build = cls.add( 549 user=user, 550 pkgs=pkgs, 551 copr=copr, 552 chroots=chroots, 553 source_type=source_type, 554 source_json=source_json, 555 enable_net=build_options.get("enable_net", copr.build_enable_net), 556 background=background, 557 git_hashes=git_hashes, 558 skip_import=skip_import, 559 batch=batch, 560 srpm_url=srpm_url, 561 ) 562 563 if user.proven: 564 if "timeout" in build_options: 565 build.timeout = build_options["timeout"] 566 567 return build
568 569 @classmethod
570 - def add(cls, user, pkgs, copr, source_type=None, source_json=None, 571 repos=None, chroots=None, timeout=None, enable_net=True, 572 git_hashes=None, skip_import=False, background=False, batch=None, 573 srpm_url=None):
574 if chroots is None: 575 chroots = [] 576 577 coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action( 578 copr, "Can't build while there is an operation in progress: {action}") 579 users_logic.UsersLogic.raise_if_cant_build_in_copr( 580 user, copr, 581 "You don't have permissions to build in this copr.") 582 583 if not repos: 584 repos = copr.repos 585 586 # todo: eliminate pkgs and this check 587 if pkgs and (" " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs): 588 raise exceptions.MalformedArgumentException("Trying to create a build using src_pkg " 589 "with bad characters. Forgot to split?") 590 591 # just temporary to keep compatibility 592 if not source_type or not source_json: 593 source_type = helpers.BuildSourceEnum("link") 594 source_json = json.dumps({"url":pkgs}) 595 596 build = models.Build( 597 user=user, 598 pkgs=pkgs, 599 copr=copr, 600 repos=repos, 601 source_type=source_type, 602 source_json=source_json, 603 submitted_on=int(time.time()), 604 enable_net=bool(enable_net), 605 is_background=bool(background), 606 batch=batch, 607 srpm_url=srpm_url, 608 priority=(cls.get_task_lowest_priority(bool(background))+1)%MAX_PRIO 609 ) 610 611 if timeout: 612 build.timeout = timeout or DEFAULT_BUILD_TIMEOUT 613 614 db.session.add(build) 615 616 # add BuildChroot object for each active (or selected) chroot 617 # this copr is assigned to 618 if not chroots: 619 chroots = copr.active_chroots 620 621 if skip_import: 622 status = StatusEnum("pending") 623 priority = (cls.get_task_lowest_priority(bool(background))+1)%MAX_PRIO 624 else: 625 status = StatusEnum("importing") 626 priority = 0 627 628 for chroot in chroots: 629 git_hash = None 630 if git_hashes: 631 git_hash = git_hashes.get(chroot.name) 632 buildchroot = models.BuildChroot( 633 build=build, 634 status=status, 635 mock_chroot=chroot, 636 git_hash=git_hash, 637 priority=priority 638 ) 639 db.session.add(buildchroot) 640 641 return build
642 643 @classmethod
644 - def rebuild_package(cls, package, source_dict_update={}):
645 source_dict = package.source_json_dict 646 source_dict.update(source_dict_update) 647 source_json = json.dumps(source_dict) 648 649 build = models.Build( 650 user=None, 651 pkgs=None, 652 package_id=package.id, 653 copr=package.copr, 654 repos=package.copr.repos, 655 source_type=package.source_type, 656 source_json=source_json, 657 submitted_on=int(time.time()), 658 enable_net=package.copr.build_enable_net, 659 timeout=DEFAULT_BUILD_TIMEOUT 660 ) 661 662 db.session.add(build) 663 664 chroots = package.copr.active_chroots 665 666 status = helpers.StatusEnum("importing") 667 668 for chroot in chroots: 669 buildchroot = models.BuildChroot( 670 build=build, 671 status=status, 672 mock_chroot=chroot, 673 git_hash=None 674 ) 675 676 db.session.add(buildchroot) 677 678 return build
679 680 681 terminal_states = {StatusEnum("failed"), StatusEnum("succeeded"), StatusEnum("canceled")} 682 683 @classmethod
684 - def get_buildchroots_by_build_id_and_branch(cls, build_id, branch):
685 """ 686 Returns a list of BuildChroots identified by build_id and dist-git 687 branch name. 688 """ 689 return ( 690 models.BuildChroot.query 691 .join(models.MockChroot) 692 .filter(models.BuildChroot.build_id==build_id) 693 .filter(models.MockChroot.distgit_branch_name==branch) 694 ).all()
695 696 697 @classmethod
698 - def delete_local_source(cls, build):
699 """ 700 Deletes the source (rpm or .spec) locally stored for upload (if exists) 701 """ 702 # is it hosted on the copr frontend? 703 if build.source_type == helpers.BuildSourceEnum("upload"): 704 data = json.loads(build.source_json) 705 tmp = data["tmp"] 706 storage_path = app.config["SRPM_STORAGE_DIR"] 707 try: 708 shutil.rmtree(os.path.join(storage_path, tmp)) 709 except: 710 pass
711 712 713 @classmethod
714 - def update_state_from_dict(cls, build, upd_dict):
715 """ 716 :param build: 717 :param upd_dict: 718 example: 719 { 720 "builds":[ 721 { 722 "id": 1, 723 "copr_id": 2, 724 "started_on": 139086644000 725 }, 726 { 727 "id": 2, 728 "copr_id": 1, 729 "status": 0, 730 "chroot": "fedora-18-x86_64", 731 "results": "http://server/results/foo/bar/", 732 "ended_on": 139086644000 733 }] 734 } 735 """ 736 log.info("Updating build: {} by: {}".format(build.id, upd_dict)) 737 if "chroot" in upd_dict: 738 if upd_dict["chroot"] == "srpm-builds": 739 if upd_dict.get("status") == StatusEnum("failed") and not build.canceled: 740 build.fail_type = helpers.FailTypeEnum("srpm_build_error") 741 for ch in build.build_chroots: 742 ch.status = helpers.StatusEnum("failed") 743 ch.ended_on = upd_dict.get("ended_on") or time.time() 744 db.session.add(ch) 745 746 # update respective chroot status 747 for build_chroot in build.build_chroots: 748 if build_chroot.name == upd_dict["chroot"]: 749 750 if "status" in upd_dict and build_chroot.status not in BuildsLogic.terminal_states: 751 build_chroot.status = upd_dict["status"] 752 753 if upd_dict.get("status") in BuildsLogic.terminal_states: 754 build_chroot.ended_on = upd_dict.get("ended_on") or time.time() 755 756 if upd_dict.get("status") == StatusEnum("starting"): 757 build_chroot.started_on = upd_dict.get("started_on") or time.time() 758 759 db.session.add(build_chroot) 760 761 for attr in ["results", "built_packages", "srpm_url"]: 762 value = upd_dict.get(attr, None) 763 if value: 764 setattr(build, attr, value) 765 766 db.session.add(build)
767 768 @classmethod
769 - def cancel_build(cls, user, build):
770 if not user.can_build_in(build.copr): 771 raise exceptions.InsufficientRightsException( 772 "You are not allowed to cancel this build.") 773 if not build.cancelable: 774 if build.status == StatusEnum("starting"): 775 err_msg = "Cannot cancel build {} in state 'starting'".format(build.id) 776 else: 777 err_msg = "Cannot cancel build {}".format(build.id) 778 raise exceptions.RequestCannotBeExecuted(err_msg) 779 780 if build.status == StatusEnum("running"): # otherwise the build is just in frontend 781 ActionsLogic.send_cancel_build(build) 782 783 build.canceled = True 784 for chroot in build.build_chroots: 785 chroot.status = 2 # canceled 786 if chroot.ended_on is not None: 787 chroot.ended_on = time.time()
788 789 @classmethod
790 - def delete_build(cls, user, build, send_delete_action=True):
791 """ 792 :type user: models.User 793 :type build: models.Build 794 """ 795 if not user.can_edit(build.copr) or build.persistent: 796 raise exceptions.InsufficientRightsException( 797 "You are not allowed to delete build `{}`.".format(build.id)) 798 799 if not build.finished: 800 # from celery.contrib import rdb; rdb.set_trace() 801 raise exceptions.ActionInProgressException( 802 "You can not delete build `{}` which is not finished.".format(build.id), 803 "Unfinished build") 804 805 if send_delete_action: 806 ActionsLogic.send_delete_build(build) 807 808 for build_chroot in build.build_chroots: 809 db.session.delete(build_chroot) 810 db.session.delete(build)
811 812 @classmethod
813 - def mark_as_failed(cls, build_id):
814 """ 815 Marks build as failed on all its non-finished chroots 816 """ 817 build = cls.get(build_id).one() 818 chroots = filter(lambda x: x.status != helpers.StatusEnum("succeeded"), build.build_chroots) 819 for chroot in chroots: 820 chroot.status = helpers.StatusEnum("failed") 821 return build
822 823 @classmethod
824 - def last_modified(cls, copr):
825 """ Get build datetime (as epoch) of last successful build 826 827 :arg copr: object of copr 828 """ 829 builds = cls.get_multiple_by_copr(copr) 830 831 last_build = ( 832 builds.join(models.BuildChroot) 833 .filter((models.BuildChroot.status == helpers.StatusEnum("succeeded")) 834 | (models.BuildChroot.status == helpers.StatusEnum("skipped"))) 835 .filter(models.BuildChroot.ended_on.isnot(None)) 836 .order_by(models.BuildChroot.ended_on.desc()) 837 ).first() 838 if last_build: 839 return last_build.ended_on 840 else: 841 return None
842 843 @classmethod
844 - def filter_is_finished(cls, query, is_finished):
845 # todo: check that ended_on is set correctly for all cases 846 # e.g.: failed dist-git import, cancellation 847 if is_finished: 848 return query.join(models.BuildChroot).filter(models.BuildChroot.ended_on.isnot(None)) 849 else: 850 return query.join(models.BuildChroot).filter(models.BuildChroot.ended_on.is_(None))
851 852 @classmethod
853 - def filter_by_group_name(cls, query, group_name):
854 return query.filter(models.Group.name == group_name)
855
856 857 -class BuildChrootsLogic(object):
858 @classmethod
859 - def get_by_build_id_and_name(cls, build_id, name):
860 mc = MockChrootsLogic.get_from_name(name).one() 861 862 return ( 863 BuildChroot.query 864 .filter(BuildChroot.build_id == build_id) 865 .filter(BuildChroot.mock_chroot_id == mc.id) 866 )
867 868 @classmethod
869 - def defer(cls, build_id, name):
870 chroot = cls.get_by_build_id_and_name(build_id, name).first() 871 if not chroot: 872 return None 873 chroot.priority = (BuildsLogic.get_task_lowest_priority(chroot.build.is_background)+1)%MAX_PRIO 874 db.session.add(chroot) 875 return chroot
876 877 @classmethod
878 - def get_multiply(cls):
879 query = ( 880 models.BuildChroot.query 881 .join(models.BuildChroot.build) 882 .join(models.BuildChroot.mock_chroot) 883 .join(models.Build.copr) 884 .join(models.Copr.user) 885 .outerjoin(models.Group) 886 ) 887 return query
888 889 @classmethod
890 - def filter_by_build_id(cls, query, build_id):
891 return query.filter(models.Build.id == build_id)
892 893 @classmethod
894 - def filter_by_project_id(cls, query, project_id):
895 return query.filter(models.Copr.id == project_id)
896 897 @classmethod
898 - def filter_by_project_user_name(cls, query, username):
899 return query.filter(models.User.username == username)
900 901 @classmethod
902 - def filter_by_state(cls, query, state):
904 905 @classmethod
906 - def filter_by_group_name(cls, query, group_name):
907 return query.filter(models.Group.name == group_name)
908
909 910 -class BuildsMonitorLogic(object):
911 @classmethod
912 - def get_monitor_data(cls, copr):
913 query = """ 914 SELECT 915 package.id as package_id, 916 package.name AS package_name, 917 build.id AS build_id, 918 build_chroot.status AS build_chroot_status, 919 build.pkg_version AS build_pkg_version, 920 mock_chroot.id AS mock_chroot_id, 921 mock_chroot.os_release AS mock_chroot_os_release, 922 mock_chroot.os_version AS mock_chroot_os_version, 923 mock_chroot.arch AS mock_chroot_arch 924 FROM package 925 JOIN (SELECT 926 MAX(build.id) AS max_build_id_for_chroot, 927 build.package_id AS package_id, 928 build_chroot.mock_chroot_id AS mock_chroot_id 929 FROM build 930 JOIN build_chroot 931 ON build.id = build_chroot.build_id 932 WHERE build.copr_id = {copr_id} 933 AND build_chroot.status != 2 934 GROUP BY build.package_id, 935 build_chroot.mock_chroot_id) AS max_build_ids_for_a_chroot 936 ON package.id = max_build_ids_for_a_chroot.package_id 937 JOIN build 938 ON build.id = max_build_ids_for_a_chroot.max_build_id_for_chroot 939 JOIN build_chroot 940 ON build_chroot.mock_chroot_id = max_build_ids_for_a_chroot.mock_chroot_id 941 AND build_chroot.build_id = max_build_ids_for_a_chroot.max_build_id_for_chroot 942 JOIN mock_chroot 943 ON mock_chroot.id = max_build_ids_for_a_chroot.mock_chroot_id 944 ORDER BY package.name ASC, package.id ASC, mock_chroot.os_release ASC, mock_chroot.os_version ASC, mock_chroot.arch ASC 945 """.format(copr_id=copr.id) 946 rows = db.session.execute(query) 947 return rows
948