1
2
3 import json
4 import pprint
5 import zmq
6 import sys
7 import os
8 import logging
9 import requests
10 import re
11
12 sys.path.append(
13 os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
14 )
15
16 from coprs import db, app
17 from coprs.logic.coprs_logic import CoprsLogic
18 from coprs.logic.builds_logic import BuildsLogic
19 from coprs.logic.complex_logic import ComplexLogic
20 from coprs import helpers
21
22 try:
23 from urllib.parse import urlparse
24 except ImportError:
25 from urlparse import urlparse
26
27 SCM_SOURCE_TYPE = helpers.BuildSourceEnum("scm")
28
29 logging.basicConfig(
30 filename="{0}/build_on_pagure_commit.log".format(app.config.get("LOG_DIR")),
31 format='[%(asctime)s][%(levelname)6s]: %(message)s',
32 level=logging.DEBUG)
33
34 log = logging.getLogger(__name__)
35 log.addHandler(logging.StreamHandler(sys.stdout))
36
37 PAGURE_BASE_URL = "https://pagure.io/"
38 SRCFPO_BASE_URL = "https://src.fedoraproject.org/"
39
40 ENDPOINT = 'tcp://hub.fedoraproject.org:9940'
41
42 TOPICS = ['io.pagure.prod.pagure.git.receive',
43 'org.fedoraproject.prod.pagure.git.receive',
44 'org.fedoraproject.prod.git.receive']
57
61
62 @classmethod
64 if db.engine.url.drivername == "sqlite":
65 placeholder = '?'
66 true = '1'
67 else:
68 placeholder = '%s'
69 true = 'true'
70
71 rows = db.engine.execute(
72 """
73 SELECT package.id AS package_id, package.source_json AS source_json, package.copr_id AS copr_id
74 FROM package
75 WHERE package.source_type = {0} AND
76 package.webhook_rebuild = {1} AND
77 package.source_json ILIKE {placeholder}
78 """.format(SCM_SOURCE_TYPE, true, placeholder=placeholder), '%'+clone_url+'%'
79 )
80 return [ScmPackage(row) for row in rows]
81
83 if not self.subdirectory or not raw_commit_text:
84 return True
85
86 for line in raw_commit_text.split('\n'):
87 match = re.search(r'^(\+\+\+|---) [ab]/(\w*)/?.*$', line)
88 if match and match.group(2).lower() == self.subdirectory.strip('./').lower():
89 return True
90
91 return False
92
95 log.debug("Setting up poller...")
96
97 ctx = zmq.Context()
98 s = ctx.socket(zmq.SUB)
99 s.connect(ENDPOINT)
100
101 for topic in TOPICS:
102 s.setsockopt_string(zmq.SUBSCRIBE, topic)
103
104 poller = zmq.Poller()
105 poller.register(s, zmq.POLLIN)
106
107 while True:
108 log.debug("Polling...")
109 evts = poller.poll(10000)
110 if not evts:
111 continue
112
113 log.debug("Receiving...")
114 _, msg_bytes = s.recv_multipart()
115 msg = msg_bytes.decode("utf-8")
116
117 log.debug("Parsing...")
118 data = json.loads(msg)
119
120 log.info("Got topic: {}".format(data['topic']))
121
122 if data['topic'] == 'org.fedoraproject.prod.git.receive':
123 project_url_path = data['msg']['commit']['namespace'] + '/' + data['msg']['commit']['repo']
124 clone_url_path = project_url_path
125 branch = data['msg']['commit']['branch']
126 start_commit = data['msg']['commit']['rev']
127 end_commit = data['msg']['commit']['rev']
128 else:
129 project_url_path = data['msg']['repo']['url_path']
130 clone_url_path = data['msg']['repo']['fullname']
131 branch = data['msg']['branch']
132 start_commit = data['msg']['start_commit']
133 end_commit = data['msg']['end_commit']
134
135 if data['topic'] == 'org.fedoraproject.prod.git.receive' or \
136 data['topic'] == 'org.fedoraproject.prod.pagure.git.receive':
137 base_url = SRCFPO_BASE_URL
138 elif data['topic'] == 'io.pagure.prod.pagure.git.receive':
139 base_url = PAGURE_BASE_URL
140 else:
141 log.error("Unknown topic {} received.".format(data['topic']))
142 continue
143
144 clone_url = base_url + clone_url_path
145 log.info("\tclone_url = {}".format(clone_url))
146 log.info("\tbranch = {}".format(branch))
147
148 candidates = ScmPackage.get_candidates_for_rebuild(clone_url)
149 raw_commit_text = None
150 if candidates:
151
152
153 raw_commit_url = base_url + project_url_path + '/raw/' + end_commit
154 r = requests.get(raw_commit_url)
155 if r.status_code == requests.codes.ok:
156 raw_commit_text = r.text
157 else:
158 log.error("Bad http status {0} from url {1}".format(r.status_code, raw_commit_url))
159
160 for pkg in candidates:
161 log.info("Considering pkg id: {}, source_json: {}".format(pkg.pkg_id, pkg.source_json_dict))
162 if (pkg.clone_url == clone_url or pkg.clone_url == clone_url+'.git') \
163 and (not pkg.committish or branch.endswith(pkg.committish)) \
164 and pkg.is_dir_in_commit(raw_commit_text):
165 log.info("\t -> rebuilding.")
166 pkg.build(end_commit)
167 else:
168 log.info("\t -> skipping.")
169
170
171 if __name__ == '__main__':
172 while True:
173 try:
174 build_on_fedmsg_loop()
175 except KeyboardInterrupt:
176 sys.exit(1)
177 except:
178 log.exception('Error in fedmsg loop. Restarting it.')
179