Module manage
[hide private]
[frames] | no frames]

Source Code for Module manage

  1  #!/usr/bin/python3 
  2   
  3  import argparse 
  4  import os 
  5  import subprocess 
  6  import datetime 
  7  import sqlalchemy 
  8  import time 
  9   
 10  import flask 
 11  from flask_script import Manager, Command, Option, Group 
 12  from flask_whooshee import Whooshee 
 13   
 14  from coprs import app 
 15  from coprs import db 
 16  from coprs import exceptions 
 17  from coprs import models 
 18  from coprs.logic import coprs_logic, packages_logic, actions_logic, builds_logic 
 19  from coprs.views.misc import create_user_wrapper 
 20  from coprs.whoosheers import CoprWhoosheer 
 21  from sqlalchemy import or_ 
 22  from coprs.helpers import chroot_to_branch,StatusEnum 
 23   
 24   
25 -class TestCommand(Command):
26
27 - def run(self, test_args):
28 os.environ["COPRS_ENVIRON_UNITTEST"] = "1" 29 if not (("COPR_CONFIG" in os.environ) and os.environ["COPR_CONFIG"]): 30 os.environ["COPR_CONFIG"] = "/etc/copr/copr_unit_test.conf" 31 os.environ["PYTHONPATH"] = "." 32 return subprocess.call(["/usr/bin/python3", "-m", "pytest"] + (test_args or []))
33 34 option_list = ( 35 Option("-a", 36 dest="test_args", 37 nargs=argparse.REMAINDER), 38 )
39 40
41 -class CreateSqliteFileCommand(Command):
42 43 """ 44 Create the sqlite DB file (not the tables). 45 Used for alembic, "create_db" does this automatically. 46 """ 47
48 - def run(self):
49 if flask.current_app.config["SQLALCHEMY_DATABASE_URI"].startswith("sqlite"): 50 # strip sqlite:/// 51 datadir_name = os.path.dirname( 52 flask.current_app.config["SQLALCHEMY_DATABASE_URI"][10:]) 53 if not os.path.exists(datadir_name): 54 os.makedirs(datadir_name)
55 56
57 -class CreateDBCommand(Command):
58 59 """ 60 Create the DB schema 61 """ 62
63 - def run(self, alembic_ini=None):
64 CreateSqliteFileCommand().run() 65 db.create_all() 66 67 # load the Alembic configuration and generate the 68 # version table, "stamping" it with the most recent rev: 69 from alembic.config import Config 70 from alembic import command 71 alembic_cfg = Config(alembic_ini) 72 command.stamp(alembic_cfg, "head") 73 74 # Functions are not covered by models.py, and no migrations are run 75 # by command.stamp() above. Create functions explicitly: 76 builds_logic.BuildsLogic.init_db()
77 78 option_list = ( 79 Option("--alembic", 80 "-f", 81 dest="alembic_ini", 82 help="Path to the alembic configuration file (alembic.ini)", 83 required=True), 84 )
85 86
87 -class DropDBCommand(Command):
88 89 """ 90 Delete DB 91 """ 92
93 - def run(self):
94 db.drop_all()
95 96
97 -class ChrootCommand(Command):
98
99 - def print_invalid_format(self, chroot_name):
100 print( 101 "{0} - invalid chroot format, must be '{release}-{version}-{arch}'." 102 .format(chroot_name))
103
104 - def print_already_exists(self, chroot_name):
105 print("{0} - already exists.".format(chroot_name))
106
107 - def print_doesnt_exist(self, chroot_name):
108 print("{0} - chroot doesn\"t exist.".format(chroot_name))
109 110 option_list = ( 111 Option("chroot_names", 112 help="Chroot name, e.g. fedora-18-x86_64.", 113 nargs="+"), 114 )
115 116
117 -class CreateChrootCommand(ChrootCommand):
118 119 "Creates a mock chroot in DB" 120
121 - def __init__(self):
122 self.option_list += Option( 123 "--dist-git-branch", 124 "-b", 125 dest="branch", 126 help="Branch name for this set of new chroots"),
127
128 - def run(self, chroot_names, branch=None):
141 142
143 -class RawhideToReleaseCommand(Command):
144 145 option_list = ( 146 Option("rawhide_chroot", help="Rawhide chroot name, e.g. fedora-rawhide-x86_64."), 147 Option("dest_chroot", help="Destination chroot, e.g. fedora-24-x86_64."), 148 ) 149
150 - def run(self, rawhide_chroot, dest_chroot):
151 mock_chroot = coprs_logic.MockChrootsLogic.get_from_name(dest_chroot).first() 152 if not mock_chroot: 153 print("Given chroot does not exist. Please run:") 154 print(" sudo python3 manage.py create_chroot {}".format(dest_chroot)) 155 return 156 157 mock_rawhide_chroot = coprs_logic.MockChrootsLogic.get_from_name(rawhide_chroot).first() 158 if not mock_rawhide_chroot: 159 print("Given rawhide chroot does not exist. Didnt you mistyped?:") 160 print(" {}".format(rawhide_chroot)) 161 return 162 163 for copr in coprs_logic.CoprsLogic.get_all(): 164 if not self.has_rawhide(copr) or not copr.follow_fedora_branching: 165 continue 166 167 self.turn_on_the_chroot_for_copr(copr, rawhide_chroot, mock_chroot) 168 169 data = {"copr": copr.name, 170 "user": copr.user.name, 171 "rawhide_chroot": rawhide_chroot, 172 "dest_chroot": dest_chroot, 173 "builds": []} 174 175 for build in builds_logic.BuildsLogic.get_multiple_by_copr(copr): 176 # rbc means rawhide_build_chroot (we needed short variable) 177 rbc = builds_logic.BuildChrootsLogic.get_by_build_id_and_name(build.id, rawhide_chroot).first() 178 dbc = builds_logic.BuildChrootsLogic.get_by_build_id_and_name(build.id, dest_chroot).first() 179 180 if not rbc or rbc.status != StatusEnum("succeeded"): 181 continue 182 data["builds"].append(build.result_dir_name) 183 184 if rbc and not dbc: 185 dest_build_chroot = models.BuildChroot(**rbc.to_dict()) 186 dest_build_chroot.mock_chroot_id = mock_chroot.id 187 dest_build_chroot.mock_chroot = mock_chroot 188 dest_build_chroot.status = StatusEnum("forked") 189 db.session.add(dest_build_chroot) 190 191 if len(data["builds"]): 192 actions_logic.ActionsLogic.send_rawhide_to_release(data) 193 194 db.session.commit()
195
196 - def turn_on_the_chroot_for_copr(self, copr, rawhide_name, mock_chroot):
197 rawhide_chroot = coprs_logic.CoprChrootsLogic.get_by_name_safe(copr, rawhide_name) 198 dest_chroot = coprs_logic.CoprChrootsLogic.get_by_name_safe(copr, mock_chroot.name) 199 200 if not rawhide_chroot or dest_chroot: 201 return 202 203 create_kwargs = { 204 "buildroot_pkgs": rawhide_chroot.buildroot_pkgs, 205 "comps": rawhide_chroot.comps, 206 "comps_name": rawhide_chroot.comps_name, 207 } 208 coprs_logic.CoprChrootsLogic.create_chroot(copr.user, copr, mock_chroot, **create_kwargs)
209
210 - def has_rawhide(self, copr):
211 return any(filter(lambda ch: ch.os_version == "rawhide", copr.mock_chroots))
212 213
214 -class BackendRawhideToReleaseCommand(RawhideToReleaseCommand):
215 216 "Copy backend data of the latest successful rawhide builds into a new chroot" 217
218 - def run(self, rawhide_chroot, dest_chroot):
219 for copr in coprs_logic.CoprsLogic.get_all(): 220 if not self.has_rawhide(copr): 221 continue 222 223 data = {"copr": copr.name, 224 "user": copr.owner_name, 225 "rawhide_chroot": rawhide_chroot, 226 "dest_chroot": dest_chroot, 227 "builds": []} 228 229 for package in packages_logic.PackagesLogic.get_all(copr.id): 230 last_build = package.last_build(successful=True) 231 if last_build: 232 data["builds"].append(last_build.result_dir_name) 233 234 if len(data["builds"]): 235 actions_logic.ActionsLogic.send_rawhide_to_release(data) 236 print("Created copy action from {}/{} to {}/{}" 237 .format(copr.full_name, rawhide_chroot, copr.full_name, dest_chroot)) 238 239 db.session.commit()
240 241
242 -class AlterChrootCommand(ChrootCommand):
243 244 "Activates or deactivates a chroot" 245
246 - def run(self, chroot_names, action):
247 activate = (action == "activate") 248 for chroot_name in chroot_names: 249 try: 250 coprs_logic.MockChrootsLogic.edit_by_name( 251 chroot_name, activate) 252 db.session.commit() 253 except exceptions.MalformedArgumentException: 254 self.print_invalid_format(chroot_name) 255 except exceptions.NotFoundException: 256 self.print_doesnt_exist(chroot_name)
257 258 option_list = ChrootCommand.option_list + ( 259 Option("--action", 260 "-a", 261 dest="action", 262 help="Action to take - currently activate or deactivate", 263 choices=["activate", "deactivate"], 264 required=True), 265 )
266 267
268 -class DropChrootCommand(ChrootCommand):
269 270 "Activates or deactivates a chroot" 271
272 - def run(self, chroot_names):
281 282
283 -class DisplayChrootsCommand(Command):
284 285 "Displays current mock chroots" 286
287 - def run(self, active_only):
288 for ch in coprs_logic.MockChrootsLogic.get_multiple( 289 active_only=active_only).all(): 290 291 print(ch.name)
292 293 option_list = ( 294 Option("--active-only", 295 "-a", 296 dest="active_only", 297 help="Display only active chroots", 298 required=False, 299 action="store_true", 300 default=False), 301 )
302 303
304 -class AddUserCommand(Command):
305 306 """ 307 You should not use regularly as that user will not be related to FAS account. 308 This should be used only for testing or adding special accounts e.g. proxy user. 309 """ 310
311 - def run(self, name, mail, **kwargs):
312 user = models.User.query.filter(models.User.username == name).first() 313 if user: 314 print("User named {0} already exists.".format(name)) 315 return 316 317 user = create_user_wrapper(name, mail) 318 if kwargs["api_token"]: 319 user.api_token = kwargs["api_token"] 320 if kwargs["api_login"]: 321 user.api_token = kwargs["api_login"] 322 323 db.session.add(user) 324 db.session.commit()
325 326 option_list = ( 327 Option("name"), 328 Option("mail"), 329 Option("--api_token", default=None, required=False), 330 Option("--api_login", default=None, required=False), 331 )
332 333
334 -class AlterUserCommand(Command):
335
336 - def run(self, name, **kwargs):
337 user = models.User.query.filter( 338 models.User.username == name).first() 339 if not user: 340 print("No user named {0}.".format(name)) 341 return 342 343 if kwargs["admin"]: 344 user.admin = True 345 if kwargs["no_admin"]: 346 user.admin = False 347 if kwargs["proven"]: 348 user.proven = True 349 if kwargs["no_proven"]: 350 user.proven = False 351 if kwargs["proxy"]: 352 user.proxy = True 353 if kwargs["no_proxy"]: 354 user.proxy = False 355 356 db.session.add(user) 357 db.session.commit()
358 359 option_list = ( 360 Option("name"), 361 Group( 362 Option("--admin", 363 action="store_true"), 364 Option("--no-admin", 365 action="store_true"), 366 exclusive=True 367 ), 368 Group( 369 Option("--proven", 370 action="store_true"), 371 Option("--no-proven", 372 action="store_true"), 373 exclusive=True 374 ), 375 Group( 376 Option("--proxy", 377 action="store_true"), 378 Option("--no-proxy", 379 action="store_true"), 380 exclusive=True 381 ) 382 )
383 384
385 -class FailBuildCommand(Command):
386 387 """ 388 Marks build as failed on all its non-finished chroots 389 """ 390 391 option_list = [Option("build_id")] 392
393 - def run(self, build_id, **kwargs):
394 try: 395 builds_logic.BuildsLogic.mark_as_failed(build_id) 396 print("Marking non-finished chroots of build {} as failed".format(build_id)) 397 db.session.commit() 398 399 except (sqlalchemy.exc.DataError, sqlalchemy.orm.exc.NoResultFound) as e: 400 print("Error: No such build {}".format(build_id)) 401 return 1
402 403
404 -class UpdateIndexesCommand(Command):
405 """ 406 recreates whoosh indexes for all projects 407 """ 408
409 - def run(self):
410 index = Whooshee.get_or_create_index(app, CoprWhoosheer) 411 412 writer = index.writer() 413 for copr in coprs_logic.CoprsLogic.get_all(): 414 CoprWhoosheer.delete_copr(writer, copr) 415 writer.commit(optimize=True) 416 417 writer = index.writer() 418 writer.schema = CoprWhoosheer.schema 419 writer.commit(optimize=True) 420 421 writer = index.writer() 422 for copr in coprs_logic.CoprsLogic.get_all(): 423 CoprWhoosheer.insert_copr(writer, copr) 424 writer.commit(optimize=True)
425 426
427 -class UpdateIndexesQuickCommand(Command):
428 """ 429 Recreates whoosh indexes for projects for which 430 indexed data were updated in last n minutes. 431 Doesn't update schema. 432 """ 433 434 option_list = [Option("minutes_passed")] 435
436 - def run(self, minutes_passed):
437 index = Whooshee.get_or_create_index(app, CoprWhoosheer) 438 439 writer = index.writer() 440 query = db.session.query(models.Copr).filter( 441 models.Copr.latest_indexed_data_update >= time.time()-int(minutes_passed)*60 442 ) 443 for copr in query.all(): 444 CoprWhoosheer.update_copr(writer, copr) 445 writer.commit()
446 447 448 manager = Manager(app) 449 manager.add_command("test", TestCommand()) 450 manager.add_command("create_sqlite_file", CreateSqliteFileCommand()) 451 manager.add_command("create_db", CreateDBCommand()) 452 manager.add_command("drop_db", DropDBCommand()) 453 manager.add_command("create_chroot", CreateChrootCommand()) 454 manager.add_command("alter_chroot", AlterChrootCommand()) 455 manager.add_command("display_chroots", DisplayChrootsCommand()) 456 manager.add_command("drop_chroot", DropChrootCommand()) 457 manager.add_command("alter_user", AlterUserCommand()) 458 manager.add_command("add_user", AddUserCommand()) 459 manager.add_command("fail_build", FailBuildCommand()) 460 manager.add_command("update_indexes", UpdateIndexesCommand()) 461 manager.add_command("update_indexes_quick", UpdateIndexesQuickCommand()) 462 manager.add_command("rawhide_to_release", RawhideToReleaseCommand()) 463 manager.add_command("backend_rawhide_to_release", BackendRawhideToReleaseCommand()) 464 465 if __name__ == "__main__": 466 manager.run() 467