Package run :: Module check_for_anitya_version_updates
[hide private]
[frames] | no frames]

Source Code for Module run.check_for_anitya_version_updates

  1  #!/usr/bin/env python2 
  2   
  3  import subprocess 
  4  import argparse 
  5  import sys 
  6  import os 
  7  import json 
  8  import time 
  9  import re 
 10  import logging 
 11  from flask_sqlalchemy import SQLAlchemy 
 12  from sqlalchemy.sql import text 
 13   
 14  sys.path.append( 
 15      os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 
 16  ) 
 17   
 18  from coprs import db, app, helpers, models 
 19  from coprs.logic.builds_logic import BuildsLogic 
 20  from coprs.logic.coprs_logic import CoprsLogic 
 21   
 22  logging.basicConfig( 
 23      filename="{0}/check_for_anitya_version_updates.log".format(app.config.get("LOG_DIR")), 
 24      format='[%(asctime)s][%(levelname)6s]: %(message)s', 
 25      level=logging.DEBUG) 
 26  log = logging.getLogger(__name__) 
 27   
 28  parser = argparse.ArgumentParser(description='Fetch package version updates by using datagrepper log of anitya emitted messages and issue rebuilds of the respective COPR packages for each such update. Requires httpie package.') 
 29   
 30  parser.add_argument('--backend', action='store', default='pypi', choices=['pypi', 'rubygems'], 
 31                     help='only check for updates from backend BACKEND, default pypi') 
 32  parser.add_argument('--delta', action='store', type=int, metavar='SECONDS', default=86400, 
 33                     help='ignore updates older than SECONDS, default 86400') 
 34  parser.add_argument('-v', '--version', action='version', version='1.0', 
 35                     help='print program version and exit') 
 36   
 37  args = parser.parse_args() 
 38   
 39   
40 -def logdebug(msg):
41 print msg 42 log.debug(msg)
43
44 -def loginfo(msg):
45 print msg 46 log.info(msg)
47
48 -def logerror(msg):
49 print >> sys.stderr, msg 50 log.error(msg)
51
52 -def logexception(msg):
53 print >> sys.stderr, msg 54 log.exception(msg)
55
56 -def run_cmd(cmd):
57 """ 58 Run given command in a subprocess 59 """ 60 loginfo('Executing: '+' '.join(cmd)) 61 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 62 (stdout, stderr) = process.communicate() 63 if process.returncode != 0: 64 logerror(stderr) 65 sys.exit(1) 66 return stdout
67
68 -def to_json(data):
69 try: 70 data_json = json.loads(data) 71 except Exception as e: 72 loginfo(data) 73 logexception(str(e)) 74 return data_json
75
76 -def get_updates_messages():
77 cmd_binary = 'curl' 78 url_template = 'https://apps.fedoraproject.org/datagrepper/raw?category=anitya&delta={delta}&topic=org.release-monitoring.prod.anitya.project.version.update&rows_per_page=64&order=asc&page={page}' 79 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=1)] 80 result_json = to_json(run_cmd(get_updates_cmd)) 81 messages = result_json['raw_messages'] 82 pages = result_json['pages'] 83 84 for p in range(2, pages+1): 85 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=p)] 86 result_json = to_json(run_cmd(get_updates_cmd)) 87 messages += result_json['raw_messages'] 88 89 return messages
90
91 -def get_updated_packages(updates_messages):
92 updated_packages = {} 93 for message in updates_messages: 94 update = message['msg'] 95 project = update['project'] 96 if args.backend.lower() != project['backend'].lower(): 97 continue 98 updated_packages[project['name'].lower()] = project['version'] 99 return updated_packages
100
101 -def get_copr_package_info_rows(updated_packages):
102 pkg_name_pattern = '(' + '|'.join(updated_packages.keys()) + ')' 103 source_type = helpers.BuildSourceEnum(args.backend.lower()) 104 if db.engine.url.drivername == "sqlite": 105 placeholder = '?' 106 true = '1' 107 else: 108 placeholder = '%s' 109 true = 'true' 110 rows = db.engine.execute( 111 """ 112 SELECT package.id AS package_id, package.source_json AS source_json, build.pkg_version AS pkg_version, package.copr_id AS copr_id 113 FROM package 114 LEFT OUTER JOIN build ON build.package_id = package.id 115 WHERE package.source_type = {placeholder} AND 116 package.source_json ~* '{pkg_name_pattern}' AND 117 package.webhook_rebuild = {true} AND 118 (build.id is NULL OR build.id = (SELECT MAX(build.id) FROM build WHERE build.package_id = package.id)); 119 """.format(placeholder=placeholder, pkg_name_pattern=pkg_name_pattern, true=true), source_type 120 ) 121 return rows
122
123 -class RubyGemsPackage(object):
124 - def __init__(self, source_json):
125 self.name = source_json['gem_name'].lower()
126
127 - def build(self, copr, new_update_version):
128 return BuildsLogic.create_new_from_rubygems(copr.user, copr, self.name, chroot_names=None)
129 130
131 -class PyPIPackage(object):
132 - def __init__(self, source_json):
133 self.name = source_json['pypi_package_name'].lower() 134 self.python_versions = source_json['python_versions']
135
136 - def build(self, copr, new_updated_version):
137 return BuildsLogic.create_new_from_pypi(copr.user, copr, self.name, new_updated_version, self.python_versions, chroot_names=None)
138 139
140 -def package_from_source(backend, source_json):
141 try: 142 return { 143 'pypi': PyPIPackage, 144 'rubygems': RubyGemsPackage, 145 }[backend](source_json) 146 except KeyError: 147 raise Exception('Unsupported backend {0} passed as command-line argument'.format(args.backend))
148 149
150 -def main():
151 updated_packages = get_updated_packages(get_updates_messages()) 152 loginfo('Updated packages according to datagrepper: {0}'.format(updated_packages)) 153 154 for row in get_copr_package_info_rows(updated_packages): 155 source_json = json.loads(row.source_json) 156 package = package_from_source(args.backend.lower(), source_json) 157 158 latest_build_version = row.pkg_version 159 loginfo('candidate package for rebuild: {0}, package_id: {1}, copr_id: {2}'.format(package.name, row.package_id, row.copr_id)) 160 if package.name in updated_packages: 161 new_updated_version = updated_packages[package.name] 162 logdebug('name: {0}, latest_build_version: {1}, new_updated_version {2}'.format(package.name, latest_build_version, new_updated_version)) 163 164 # if the last build's package version is "different" from new remote package version, rebuild 165 if not latest_build_version or not re.match(new_updated_version, latest_build_version): 166 try: 167 copr = CoprsLogic.get_by_id(row.copr_id)[0] 168 except Exception as e: 169 logexception(e) 170 continue 171 172 loginfo('Launching {} build for package of source name: {}, package_id: {}, copr_id: {}, user_id: {}' 173 .format(args.backend.lower(), package.name, row.package_id, copr.id, copr.user.id)) 174 build = package.build(copr, new_updated_version) 175 db.session.commit() 176 loginfo('Launched build id {0}'.format(build.id))
177 178 if __name__ == '__main__': 179 try: 180 main() 181 except Exception as e: 182 logexception(str(e)) 183