Module manage
[hide private]
[frames] | no frames]

Source Code for Module manage

  1  #!/usr/bin/python3 
  2   
  3  import argparse 
  4  import os 
  5  import subprocess 
  6  import datetime 
  7  import sqlalchemy 
  8  import time 
  9   
 10  import flask 
 11  from flask_script import Manager, Command, Option, Group 
 12  from flask_whooshee import Whooshee 
 13   
 14  from copr_common.enums import StatusEnum 
 15  from coprs import app 
 16  from coprs import db 
 17  from coprs import exceptions 
 18  from coprs import models 
 19  from coprs.logic import coprs_logic, packages_logic, actions_logic, builds_logic, users_logic 
 20  from coprs.views.misc import create_user_wrapper 
 21  from coprs.whoosheers import CoprWhoosheer 
 22  from sqlalchemy import and_, or_ 
 23  from coprs.helpers import chroot_to_branch 
 24   
 25   
26 -class TestCommand(Command):
27
28 - def run(self, coverage, test_args):
29 os.environ["COPRS_ENVIRON_UNITTEST"] = "1" 30 if not (("COPR_CONFIG" in os.environ) and os.environ["COPR_CONFIG"]): 31 os.environ["COPR_CONFIG"] = "/etc/copr/copr_unit_test.conf" 32 33 if 'PYTHONPATH' in os.environ: 34 os.environ['PYTHONPATH'] = os.environ['PYTHONPATH'] + ':.' 35 else: 36 os.environ['PYTHONPATH'] = '.' 37 38 additional_args = [] 39 40 if coverage: 41 additional_args.extend([ 42 '--cov-report', 'term-missing', '--cov', 'coprs' 43 ]) 44 45 return subprocess.call(["/usr/bin/python3", "-m", "pytest"] + additional_args)
46 47 option_list = ( 48 Option("-a", 49 dest="test_args", 50 nargs=argparse.REMAINDER), 51 Option("--coverage", 52 dest="coverage", 53 required=False, 54 action='store_true', 55 default=False), 56 )
57 58
59 -class CreateSqliteFileCommand(Command):
60 61 """ 62 Create the sqlite DB file (not the tables). 63 Used for alembic, "create_db" does this automatically. 64 """ 65
66 - def run(self):
67 if flask.current_app.config["SQLALCHEMY_DATABASE_URI"].startswith("sqlite"): 68 # strip sqlite:/// 69 datadir_name = os.path.dirname( 70 flask.current_app.config["SQLALCHEMY_DATABASE_URI"][10:]) 71 if not os.path.exists(datadir_name): 72 os.makedirs(datadir_name)
73 74
75 -class CreateDBCommand(Command):
76 77 """ 78 Create the DB schema 79 """ 80
81 - def run(self, alembic_ini=None):
82 CreateSqliteFileCommand().run() 83 db.create_all() 84 85 # load the Alembic configuration and generate the 86 # version table, "stamping" it with the most recent rev: 87 from alembic.config import Config 88 from alembic import command 89 alembic_cfg = Config(alembic_ini) 90 command.stamp(alembic_cfg, "head") 91 92 # Functions are not covered by models.py, and no migrations are run 93 # by command.stamp() above. Create functions explicitly: 94 builds_logic.BuildsLogic.init_db()
95 96 option_list = ( 97 Option("--alembic", 98 "-f", 99 dest="alembic_ini", 100 help="Path to the alembic configuration file (alembic.ini)", 101 required=True), 102 )
103 104
105 -class DropDBCommand(Command):
106 107 """ 108 Delete DB 109 """ 110
111 - def run(self):
112 db.drop_all()
113 114
115 -class ChrootCommand(Command):
116
117 - def print_invalid_format(self, chroot_name):
118 print( 119 "{0} - invalid chroot format, must be '{release}-{version}-{arch}'." 120 .format(chroot_name))
121
122 - def print_already_exists(self, chroot_name):
123 print("{0} - already exists.".format(chroot_name))
124
125 - def print_doesnt_exist(self, chroot_name):
126 print("{0} - chroot doesn\"t exist.".format(chroot_name))
127 128 option_list = ( 129 Option("chroot_names", 130 help="Chroot name, e.g. fedora-18-x86_64.", 131 nargs="+"), 132 )
133 134
135 -class CreateChrootCommand(ChrootCommand):
136 137 "Creates a mock chroot in DB" 138
139 - def __init__(self):
140 self.option_list += Option( 141 "--dist-git-branch", 142 "-b", 143 dest="branch", 144 help="Branch name for this set of new chroots"),
145
146 - def run(self, chroot_names, branch=None):
159 160
161 -class RawhideToReleaseCommand(Command):
162 163 option_list = ( 164 Option("rawhide_chroot", help="Rawhide chroot name, e.g. fedora-rawhide-x86_64."), 165 Option("dest_chroot", help="Destination chroot, e.g. fedora-24-x86_64."), 166 ) 167
168 - def run(self, rawhide_chroot, dest_chroot):
169 mock_chroot = coprs_logic.MockChrootsLogic.get_from_name(dest_chroot).first() 170 if not mock_chroot: 171 print("Given chroot does not exist. Please run:") 172 print(" sudo python3 manage.py create_chroot {}".format(dest_chroot)) 173 return 174 175 mock_rawhide_chroot = coprs_logic.MockChrootsLogic.get_from_name(rawhide_chroot).first() 176 if not mock_rawhide_chroot: 177 print("Given rawhide chroot does not exist. Didnt you mistyped?:") 178 print(" {}".format(rawhide_chroot)) 179 return 180 181 for copr in coprs_logic.CoprsLogic.get_all(): 182 if not self.has_rawhide(copr) or not copr.follow_fedora_branching: 183 continue 184 185 self.turn_on_the_chroot_for_copr(copr, rawhide_chroot, mock_chroot) 186 187 data = {"projectname": copr.name, 188 "ownername": copr.owner_name, 189 "rawhide_chroot": rawhide_chroot, 190 "dest_chroot": dest_chroot, 191 "builds": []} 192 193 for build in builds_logic.BuildsLogic.get_multiple_by_copr(copr): 194 # rbc means rawhide_build_chroot (we needed short variable) 195 rbc = builds_logic.BuildChrootsLogic.get_by_build_id_and_name(build.id, rawhide_chroot).first() 196 dbc = builds_logic.BuildChrootsLogic.get_by_build_id_and_name(build.id, dest_chroot).first() 197 198 if not rbc or rbc.status != StatusEnum("succeeded"): 199 continue 200 201 data["builds"].append(rbc.result_dir) 202 203 if rbc and not dbc: 204 dest_build_chroot = models.BuildChroot(**rbc.to_dict()) 205 dest_build_chroot.mock_chroot_id = mock_chroot.id 206 dest_build_chroot.mock_chroot = mock_chroot 207 dest_build_chroot.status = StatusEnum("forked") 208 db.session.add(dest_build_chroot) 209 210 if len(data["builds"]): 211 actions_logic.ActionsLogic.send_rawhide_to_release(data) 212 213 db.session.commit()
214
215 - def turn_on_the_chroot_for_copr(self, copr, rawhide_name, mock_chroot):
216 rawhide_chroot = coprs_logic.CoprChrootsLogic.get_by_name_safe(copr, rawhide_name) 217 dest_chroot = coprs_logic.CoprChrootsLogic.get_by_name_safe(copr, mock_chroot.name) 218 219 if not rawhide_chroot or dest_chroot: 220 return 221 222 create_kwargs = { 223 "buildroot_pkgs": rawhide_chroot.buildroot_pkgs, 224 "comps": rawhide_chroot.comps, 225 "comps_name": rawhide_chroot.comps_name, 226 } 227 coprs_logic.CoprChrootsLogic.create_chroot(copr.user, copr, mock_chroot, **create_kwargs)
228
229 - def has_rawhide(self, copr):
230 return any(filter(lambda ch: ch.os_version == "rawhide", copr.mock_chroots))
231 232
233 -class BackendRawhideToReleaseCommand(RawhideToReleaseCommand):
234 235 "Copy backend data of the latest successful rawhide builds into a new chroot" 236
237 - def run(self, rawhide_chroot, dest_chroot):
238 for copr in coprs_logic.CoprsLogic.get_all(): 239 if not self.has_rawhide(copr): 240 continue 241 242 data = {"copr": copr.name, 243 "user": copr.owner_name, 244 "rawhide_chroot": rawhide_chroot, 245 "dest_chroot": dest_chroot, 246 "builds": []} 247 248 for package in packages_logic.PackagesLogic.get_all(copr.id): 249 last_build = package.last_build(successful=True) 250 if last_build: 251 data["builds"].append(last_build.result_dir) 252 253 if len(data["builds"]): 254 actions_logic.ActionsLogic.send_rawhide_to_release(data) 255 print("Created copy action from {}/{} to {}/{}" 256 .format(copr.full_name, rawhide_chroot, copr.full_name, dest_chroot)) 257 258 db.session.commit()
259 260
261 -class AlterChrootCommand(ChrootCommand):
262 263 "Activates or deactivates a chroot" 264
265 - def run(self, chroot_names, action):
266 activate = (action == "activate") 267 for chroot_name in chroot_names: 268 try: 269 coprs_logic.MockChrootsLogic.edit_by_name( 270 chroot_name, activate) 271 db.session.commit() 272 except exceptions.MalformedArgumentException: 273 self.print_invalid_format(chroot_name) 274 except exceptions.NotFoundException: 275 self.print_doesnt_exist(chroot_name)
276 277 option_list = ChrootCommand.option_list + ( 278 Option("--action", 279 "-a", 280 dest="action", 281 help="Action to take - currently activate or deactivate", 282 choices=["activate", "deactivate"], 283 required=True), 284 )
285 286
287 -class DropChrootCommand(ChrootCommand):
288 289 "Activates or deactivates a chroot" 290
291 - def run(self, chroot_names):
300 301
302 -class DisplayChrootsCommand(Command):
303 304 "Displays current mock chroots" 305
306 - def run(self, active_only):
307 for ch in coprs_logic.MockChrootsLogic.get_multiple( 308 active_only=active_only).all(): 309 310 print(ch.name)
311 312 option_list = ( 313 Option("--active-only", 314 "-a", 315 dest="active_only", 316 help="Display only active chroots", 317 required=False, 318 action="store_true", 319 default=False), 320 )
321 322
323 -class AddUserCommand(Command):
324 325 """ 326 You should not use regularly as that user will not be related to FAS account. 327 This should be used only for testing or adding special accounts e.g. proxy user. 328 """ 329
330 - def run(self, name, mail, **kwargs):
331 user = models.User.query.filter(models.User.username == name).first() 332 if user: 333 print("User named {0} already exists.".format(name)) 334 return 335 336 user = create_user_wrapper(name, mail) 337 if kwargs["api_token"]: 338 user.api_token = kwargs["api_token"] 339 if kwargs["api_login"]: 340 user.api_token = kwargs["api_login"] 341 342 db.session.add(user) 343 db.session.commit()
344 345 option_list = ( 346 Option("name"), 347 Option("mail"), 348 Option("--api_token", default=None, required=False), 349 Option("--api_login", default=None, required=False), 350 )
351 352
353 -class DumpUserCommand(Command):
354
355 - def run(self, username):
356 user = models.User.query.filter(models.User.username == username).first() 357 if not user: 358 print("There is no user named {0}.".format(username)) 359 return 1 360 dumper = users_logic.UserDataDumper(user) 361 print(dumper.dumps(pretty=True))
362 363 option_list = ( 364 Option("username"), 365 )
366 367
368 -class AlterUserCommand(Command):
369
370 - def run(self, name, **kwargs):
371 user = models.User.query.filter( 372 models.User.username == name).first() 373 if not user: 374 print("No user named {0}.".format(name)) 375 return 376 377 if kwargs["admin"]: 378 user.admin = True 379 if kwargs["no_admin"]: 380 user.admin = False 381 if kwargs["proven"]: 382 user.proven = True 383 if kwargs["no_proven"]: 384 user.proven = False 385 if kwargs["proxy"]: 386 user.proxy = True 387 if kwargs["no_proxy"]: 388 user.proxy = False 389 390 db.session.add(user) 391 db.session.commit()
392 393 option_list = ( 394 Option("name"), 395 Group( 396 Option("--admin", 397 action="store_true"), 398 Option("--no-admin", 399 action="store_true"), 400 exclusive=True 401 ), 402 Group( 403 Option("--proven", 404 action="store_true"), 405 Option("--no-proven", 406 action="store_true"), 407 exclusive=True 408 ), 409 Group( 410 Option("--proxy", 411 action="store_true"), 412 Option("--no-proxy", 413 action="store_true"), 414 exclusive=True 415 ) 416 )
417 418
419 -class FailBuildCommand(Command):
420 421 """ 422 Marks build as failed on all its non-finished chroots 423 """ 424 425 option_list = [Option("build_id")] 426
427 - def run(self, build_id, **kwargs):
428 try: 429 builds_logic.BuildsLogic.mark_as_failed(build_id) 430 print("Marking non-finished chroots of build {} as failed".format(build_id)) 431 db.session.commit() 432 433 except (sqlalchemy.exc.DataError, sqlalchemy.orm.exc.NoResultFound) as e: 434 print("Error: No such build {}".format(build_id)) 435 return 1
436 437
438 -class UpdateIndexesCommand(Command):
439 """ 440 recreates whoosh indexes for all projects 441 """ 442
443 - def run(self):
444 index = Whooshee.get_or_create_index(app, CoprWhoosheer) 445 446 writer = index.writer() 447 for copr in coprs_logic.CoprsLogic.get_all(): 448 CoprWhoosheer.delete_copr(writer, copr) 449 writer.commit(optimize=True) 450 451 writer = index.writer() 452 writer.schema = CoprWhoosheer.schema 453 writer.commit(optimize=True) 454 455 writer = index.writer() 456 for copr in coprs_logic.CoprsLogic.get_all(): 457 CoprWhoosheer.insert_copr(writer, copr) 458 writer.commit(optimize=True)
459 460
461 -class UpdateIndexesQuickCommand(Command):
462 """ 463 Recreates whoosh indexes for projects for which 464 indexed data were updated in last n minutes. 465 Doesn't update schema. 466 """ 467 468 option_list = [Option("minutes_passed")] 469
470 - def run(self, minutes_passed):
471 index = Whooshee.get_or_create_index(app, CoprWhoosheer) 472 473 writer = index.writer() 474 query = db.session.query(models.Copr).filter( 475 models.Copr.latest_indexed_data_update >= time.time()-int(minutes_passed)*60 476 ) 477 for copr in query.all(): 478 CoprWhoosheer.update_copr(writer, copr) 479 writer.commit()
480 481
482 -class UpdateGraphsDataCommand(Command):
483 """ 484 Generates newest graph data. 485 """ 486
487 - def run(self):
491 492
493 -class RemoveGraphsDataCommand(Command):
494 """ 495 Removes old cached graph data that is no longer used. 496 """ 497
498 - def run(self):
499 curr_time = int(time.time()) 500 models.BuildsStatistics.query.filter(or_( 501 and_(models.BuildsStatistics.time < curr_time - 91 * 86400, 502 models.BuildsStatistics.stat_type == '24h'), 503 and_(models.BuildsStatistics.time < curr_time - 87000, 504 models.BuildsStatistics.stat_type == '30min'), 505 and_(models.BuildsStatistics.time < curr_time - 87000, 506 models.BuildsStatistics.stat_type == '10min') 507 )).delete() 508 db.session.commit()
509 510 511 manager = Manager(app) 512 manager.add_command("test", TestCommand()) 513 manager.add_command("create_sqlite_file", CreateSqliteFileCommand()) 514 manager.add_command("create_db", CreateDBCommand()) 515 manager.add_command("drop_db", DropDBCommand()) 516 manager.add_command("create_chroot", CreateChrootCommand()) 517 manager.add_command("alter_chroot", AlterChrootCommand()) 518 manager.add_command("display_chroots", DisplayChrootsCommand()) 519 manager.add_command("drop_chroot", DropChrootCommand()) 520 manager.add_command("alter_user", AlterUserCommand()) 521 manager.add_command("add_user", AddUserCommand()) 522 manager.add_command("dump_user", DumpUserCommand()) 523 manager.add_command("fail_build", FailBuildCommand()) 524 manager.add_command("update_indexes", UpdateIndexesCommand()) 525 manager.add_command("update_indexes_quick", UpdateIndexesQuickCommand()) 526 manager.add_command("rawhide_to_release", RawhideToReleaseCommand()) 527 manager.add_command("backend_rawhide_to_release", BackendRawhideToReleaseCommand()) 528 manager.add_command("update_graphs", UpdateGraphsDataCommand()) 529 manager.add_command("vacuum_graphs", RemoveGraphsDataCommand()) 530 531 if __name__ == "__main__": 532 manager.run() 533