Package run :: Module build_on_pagure_commit
[hide private]
[frames] | no frames]

Source Code for Module run.build_on_pagure_commit

  1  #!/usr/bin/env python3 
  2   
  3  import json 
  4  import pprint 
  5  import zmq 
  6  import sys 
  7  import os 
  8  import logging 
  9  import requests 
 10  import re 
 11   
 12  sys.path.append( 
 13      os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 
 14  ) 
 15   
 16  from coprs import db, app 
 17  from coprs.logic.coprs_logic import CoprsLogic 
 18  from coprs.logic.builds_logic import BuildsLogic 
 19  from coprs.logic.complex_logic import ComplexLogic 
 20  from coprs import helpers 
 21   
 22  try: 
 23      from urllib.parse import urlparse 
 24  except ImportError: 
 25      from urlparse import urlparse 
 26   
 27  SCM_SOURCE_TYPE = helpers.BuildSourceEnum("scm") 
 28   
 29  logging.basicConfig( 
 30      filename="{0}/build_on_pagure_commit.log".format(app.config.get("LOG_DIR")), 
 31      format='[%(asctime)s][%(levelname)6s]: %(message)s', 
 32      level=logging.DEBUG) 
 33   
 34  log = logging.getLogger(__name__) 
 35  log.addHandler(logging.StreamHandler(sys.stdout)) 
 36   
 37  PAGURE_BASE_URL = "https://pagure.io/" 
 38  SRCFPO_BASE_URL = "https://src.fedoraproject.org/" 
 39   
 40  ENDPOINT = 'tcp://hub.fedoraproject.org:9940' 
 41   
 42  TOPICS = ['io.pagure.prod.pagure.git.receive', 
 43            'org.fedoraproject.prod.pagure.git.receive', 
 44            'org.fedoraproject.prod.git.receive'] 
45 46 -class ScmPackage(object):
47 - def __init__(self, db_row):
48 self.pkg_id = db_row.package_id 49 self.copr_id = db_row.copr_id 50 51 self.source_json_dict = json.loads(db_row.source_json) 52 self.clone_url = self.source_json_dict.get('clone_url') or '' 53 self.committish = self.source_json_dict.get('committish') or '' 54 self.subdirectory = self.source_json_dict.get('subdirectory') or '' 55 56 self.package = ComplexLogic.get_package_by_id_safe(self.pkg_id)
57
58 - def build(self, branch):
59 BuildsLogic.rebuild_package(self.package, {'committish': branch}) 60 db.session.commit()
61 62 @classmethod
63 - def get_candidates_for_rebuild(cls, clone_url):
64 if db.engine.url.drivername == "sqlite": 65 placeholder = '?' 66 true = '1' 67 else: 68 placeholder = '%s' 69 true = 'true' 70 71 rows = db.engine.execute( 72 """ 73 SELECT package.id AS package_id, package.source_json AS source_json, package.copr_id AS copr_id 74 FROM package 75 WHERE package.source_type = {0} AND 76 package.webhook_rebuild = {1} AND 77 package.source_json ILIKE {placeholder} 78 """.format(SCM_SOURCE_TYPE, true, placeholder=placeholder), '%'+clone_url+'%' 79 ) 80 return [ScmPackage(row) for row in rows]
81
82 - def is_dir_in_commit(self, raw_commit_text):
83 if not self.subdirectory or not raw_commit_text: 84 return True 85 86 for line in raw_commit_text.split('\n'): 87 match = re.search(r'^(\+\+\+|---) [ab]/(\w*)/?.*$', line) 88 if match and match.group(2).lower() == self.subdirectory.strip('./').lower(): 89 return True 90 91 return False
92
93 94 -def build_on_fedmsg_loop():
95 log.debug("Setting up poller...") 96 97 ctx = zmq.Context() 98 s = ctx.socket(zmq.SUB) 99 s.connect(ENDPOINT) 100 101 for topic in TOPICS: 102 s.setsockopt_string(zmq.SUBSCRIBE, topic) 103 104 poller = zmq.Poller() 105 poller.register(s, zmq.POLLIN) 106 107 while True: 108 log.debug("Polling...") 109 evts = poller.poll(10000) 110 if not evts: 111 continue 112 113 log.debug("Receiving...") 114 _, msg_bytes = s.recv_multipart() 115 msg = msg_bytes.decode("utf-8") 116 117 log.debug("Parsing...") 118 data = json.loads(msg) 119 120 log.info("Got topic: {}".format(data['topic'])) 121 122 if data['topic'] == 'org.fedoraproject.prod.git.receive': 123 project_url_path = data['msg']['commit']['namespace'] + '/' + data['msg']['commit']['repo'] 124 clone_url_path = project_url_path 125 branch = data['msg']['commit']['branch'] 126 start_commit = data['msg']['commit']['rev'] 127 end_commit = data['msg']['commit']['rev'] 128 else: 129 project_url_path = data['msg']['repo']['url_path'] 130 clone_url_path = data['msg']['repo']['fullname'] 131 branch = data['msg']['branch'] 132 start_commit = data['msg']['start_commit'] 133 end_commit = data['msg']['end_commit'] 134 135 if data['topic'] == 'org.fedoraproject.prod.git.receive' or \ 136 data['topic'] == 'org.fedoraproject.prod.pagure.git.receive': 137 base_url = SRCFPO_BASE_URL 138 elif data['topic'] == 'io.pagure.prod.pagure.git.receive': 139 base_url = PAGURE_BASE_URL 140 else: 141 log.error("Unknown topic {} received.".format(data['topic'])) 142 continue 143 144 clone_url = base_url + clone_url_path 145 log.info("\tclone_url = {}".format(clone_url)) 146 log.info("\tbranch = {}".format(branch)) 147 148 candidates = ScmPackage.get_candidates_for_rebuild(clone_url) 149 raw_commit_text = None 150 if candidates: 151 # if start_commit != end_commit 152 # then more than one commit and no means to iterate over :( 153 raw_commit_url = base_url + project_url_path + '/raw/' + end_commit 154 r = requests.get(raw_commit_url) 155 if r.status_code == requests.codes.ok: 156 raw_commit_text = r.text 157 else: 158 log.error("Bad http status {0} from url {1}".format(r.status_code, raw_commit_url)) 159 160 for pkg in candidates: 161 log.info("Considering pkg id: {}, source_json: {}".format(pkg.pkg_id, pkg.source_json_dict)) 162 if (pkg.clone_url == clone_url or pkg.clone_url == clone_url+'.git') \ 163 and (not pkg.committish or branch.endswith(pkg.committish)) \ 164 and pkg.is_dir_in_commit(raw_commit_text): 165 log.info("\t -> rebuilding.") 166 pkg.build(end_commit) 167 else: 168 log.info("\t -> skipping.")
169 170 171 if __name__ == '__main__': 172 while True: 173 try: 174 build_on_fedmsg_loop() 175 except KeyboardInterrupt: 176 sys.exit(1) 177 except: 178 log.exception('Error in fedmsg loop. Restarting it.') 179