1
2
3 import json
4 import pprint
5 import zmq
6 import sys
7 import os
8 import logging
9 import requests
10 import re
11 import munch
12
13 sys.path.append(
14 os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
15 )
16
17 from coprs import db, app, models
18 from coprs.logic.coprs_logic import CoprsLogic, CoprDirsLogic
19 from coprs.logic.builds_logic import BuildsLogic
20 from coprs.logic.complex_logic import ComplexLogic
21 from coprs.logic.packages_logic import PackagesLogic
22 from coprs import helpers
23
24 from urllib.parse import urlparse
25
26 SCM_SOURCE_TYPE = helpers.BuildSourceEnum("scm")
27
28 logging.basicConfig(
29 filename='{0}/pagure-events.log'.format(app.config.get('LOG_DIR')),
30 format='[%(asctime)s][%(levelname)6s]: %(message)s',
31 level=logging.DEBUG)
32
33 log = logging.getLogger(__name__)
34 log.addHandler(logging.StreamHandler(sys.stdout))
35
36 if os.getenv('PAGURE_EVENTS_TESTONLY'):
37 ENDPOINT = 'tcp://stg.pagure.io:9940'
38 else:
39 ENDPOINT = 'tcp://hub.fedoraproject.org:9940'
40
41 log.info("ENDPOINT = {}".format(ENDPOINT))
42
43 TOPICS = {
44 'io.pagure.prod.pagure.git.receive': 'https://pagure.io/',
45 'io.pagure.prod.pagure.pull-request.new': 'https://pagure.io/',
46 'io.pagure.prod.pagure.pull-request.comment.added': 'https://pagure.io/',
47 'org.fedoraproject.prod.pagure.git.receive': 'https://src.fedoraproject.org/',
48 'org.fedoraproject.prod.pagure.pull-request.new': 'https://src.fedoraproject.org/',
49 'org.fedoraproject.prod.pagure.pull-request.comment.added': 'https://src.fedoraproject.org/',
50 'io.pagure.stg.pagure.git.receive': 'https://stg.pagure.io/',
51 'io.pagure.stg.pagure.pull-request.new': 'https://stg.pagure.io/',
52 'io.pagure.stg.pagure.pull-request.comment.added': 'https://stg.pagure.io/',
53 }
68
69 - def build(self, source_dict_update, copr_dir, update_callback,
70 scm_object_type, scm_object_id, scm_object_url):
82
83 @classmethod
85 if db.engine.url.drivername == 'sqlite':
86 placeholder = '?'
87 true = '1'
88 else:
89 placeholder = '%s'
90 true = 'true'
91
92 rows = db.engine.execute(
93 """
94 SELECT package.id AS package_id, package.source_json AS source_json, package.copr_id AS copr_id
95 FROM package JOIN copr_dir ON package.copr_dir_id = copr_dir.id
96 WHERE package.source_type = {0} AND
97 package.webhook_rebuild = {1} AND
98 copr_dir.main = {2} AND
99 package.source_json ILIKE {placeholder}
100 """.format(SCM_SOURCE_TYPE, true, true, placeholder=placeholder), '%'+clone_url+'%'
101 )
102 return [ScmPackage(row) for row in rows]
103
105 if not self.subdirectory or not raw_commit_text:
106 return True
107
108 for line in raw_commit_text.split('\n'):
109 match = re.search(r'^(\+\+\+|---) [ab]/(\w*)/?.*$', line)
110 if match and match.group(2).lower() == self.subdirectory.strip('./').lower():
111 return True
112
113 return False
114
117 """
118 Message handler for updated pull-request opened in pagure.
119 Topic: ``*.pagure.pull-request.comment.added``
120 """
121 if data['msg']['pullrequest']['status'] != 'Open':
122 log.info('Pull-request not open, discarding.')
123 return False
124
125 if not data['msg']['pullrequest']['comments']:
126 log.info('This is most odd, we\'re not seeing comments.')
127 return False
128
129 last_comment = data['msg']['pullrequest']['comments'][-1]
130 if not last_comment or last_comment['notification'] is False:
131 log.info('Comment was not a notification, discarding.')
132 return False
133
134 return munch.Munch({
135 'object_id': data['msg']['pullrequest']['id'],
136 'object_type': 'pull-request',
137 'base_project_url_path': data['msg']['pullrequest']['project']['url_path'],
138 'base_clone_url_path': data['msg']['pullrequest']['project']['fullname'],
139 'base_clone_url': base_url + data['msg']['pullrequest']['project']['fullname'],
140 'project_url_path': data['msg']['pullrequest']['repo_from']['url_path'],
141 'clone_url_path': data['msg']['pullrequest']['repo_from']['fullname'],
142 'clone_url': base_url + data['msg']['pullrequest']['repo_from']['fullname'],
143 'branch_from': data['msg']['pullrequest']['branch_from'],
144 'branch_to': data['msg']['pullrequest']['branch'],
145 'start_commit': data['msg']['pullrequest']['commit_start'],
146 'end_commit': data['msg']['pullrequest']['commit_stop'],
147 })
148
151 """
152 Message handler for new pull-request opened in pagure.
153 Topic: ``*.pagure.pull-request.new``
154 """
155 return munch.Munch({
156 'object_id': data['msg']['pullrequest']['id'],
157 'object_type': 'pull-request',
158 'base_project_url_path': data['msg']['pullrequest']['project']['url_path'],
159 'base_clone_url_path': data['msg']['pullrequest']['project']['fullname'],
160 'base_clone_url': base_url + data['msg']['pullrequest']['project']['fullname'],
161 'project_url_path': data['msg']['pullrequest']['repo_from']['url_path'],
162 'clone_url_path': data['msg']['pullrequest']['repo_from']['fullname'],
163 'clone_url': base_url + data['msg']['pullrequest']['repo_from']['fullname'],
164 'branch_from': data['msg']['pullrequest']['branch_from'],
165 'branch_to': data['msg']['pullrequest']['branch'],
166 'start_commit': data['msg']['pullrequest']['commit_start'],
167 'end_commit': data['msg']['pullrequest']['commit_stop'],
168 })
169
172 """
173 Message handler for push event in pagure.
174 Topic: ``*.pagure.git.receive``
175 """
176 return munch.Munch({
177 'object_id': data['msg']['end_commit'],
178 'object_type': 'commit',
179 'base_project_url_path': data['msg']['repo']['url_path'],
180 'base_clone_url_path': data['msg']['repo']['fullname'],
181 'base_clone_url': base_url + data['msg']['repo']['fullname'],
182 'project_url_path': data['msg']['repo']['url_path'],
183 'clone_url_path': data['msg']['repo']['fullname'],
184 'clone_url': base_url + data['msg']['repo']['fullname'],
185 'branch_from': data['msg']['branch'],
186 'branch_to': data['msg']['branch'],
187 'start_commit': data['msg']['start_commit'],
188 'end_commit': data['msg']['end_commit'],
189 })
190
193 url1 = re.sub(r'(\.git)?/*$', '', str(url1))
194 url2 = re.sub(r'(\.git)?/*$', '', str(url2))
195 o1 = urlparse(url1)
196 o2 = urlparse(url2)
197 return (o1.netloc == o2.netloc and o1.path == o2.path)
198
201 log.debug("Setting up poller...")
202 pp = pprint.PrettyPrinter(width=120)
203
204 ctx = zmq.Context()
205 s = ctx.socket(zmq.SUB)
206 s.connect(ENDPOINT)
207
208 for topic in TOPICS:
209 s.setsockopt_string(zmq.SUBSCRIBE, topic)
210
211 poller = zmq.Poller()
212 poller.register(s, zmq.POLLIN)
213
214 while True:
215 log.debug('Polling...')
216 evts = poller.poll(10000)
217 if not evts:
218 continue
219
220 log.debug('Receiving...')
221 _, msg_bytes = s.recv_multipart()
222 msg = msg_bytes.decode('utf-8')
223
224 log.debug('Parsing...')
225 data = json.loads(msg)
226
227 log.info('Got topic: {}'.format(data['topic']))
228 base_url = TOPICS.get(data['topic'])
229 if not base_url:
230 log.error('Unknown topic {} received. Continuing.')
231 continue
232
233 if re.match(r'^.*.pull-request.new$', data['topic']):
234 event_info = event_info_from_new_pr(data, base_url)
235 elif re.match(r'^.*.pull-request.comment.added$', data['topic']):
236 event_info = event_info_from_pr_update(data, base_url)
237 else:
238 event_info = event_info_from_push(data, base_url)
239
240 log.info('event_info = {}'.format(pp.pformat(event_info)))
241
242 if not event_info:
243 log.info('Received event was discarded. Continuing.')
244 continue
245
246 candidates = ScmPackage.get_candidates_for_rebuild(event_info.base_clone_url)
247 raw_commit_text = None
248 if candidates:
249
250
251 raw_commit_url = base_url + event_info.project_url_path + '/raw/' + event_info.end_commit
252 r = requests.get(raw_commit_url)
253 if r.status_code == requests.codes.ok:
254 raw_commit_text = r.text
255 else:
256 log.error('Bad http status {0} from url {1}'.format(r.status_code, raw_commit_url))
257
258 for pkg in candidates:
259 log.info('Considering pkg id: {}, source_json: {}'.format(pkg.pkg_id, pkg.source_json_dict))
260
261 if (git_compare_urls(pkg.clone_url, event_info.base_clone_url)
262 and (not pkg.committish or event_info.branch_to.endswith(pkg.committish))
263 and pkg.is_dir_in_commit(raw_commit_text)):
264
265 log.info('\t -> rebuilding.')
266
267 if event_info.object_type == 'pull-request':
268 dirname = pkg.copr.name + ':pr:' + str(event_info.object_id)
269 copr_dir = CoprDirsLogic.get_or_create(pkg.copr, dirname)
270 update_callback = 'pagure_flag_pull_request'
271 scm_object_url = os.path.join(base_url, event_info.base_project_url_path,
272 'pull-request', str(event_info.object_id))
273 else:
274 copr_dir = pkg.copr.main_dir
275 update_callback = 'pagure_flag_commit'
276 scm_object_url = os.path.join(base_url, event_info.base_project_url_path,
277 'c', str(event_info.object_id))
278
279 if not git_compare_urls(pkg.copr.scm_repo_url, event_info.base_clone_url):
280 update_callback = ''
281
282 source_dict_update = {
283 'clone_url': event_info.clone_url,
284 'committish': event_info.end_commit,
285 }
286
287 build = pkg.build(
288 source_dict_update,
289 copr_dir,
290 update_callback,
291 event_info.object_type,
292 event_info.object_id,
293 scm_object_url
294 )
295 log.info('\t -> {}'.format(build.to_dict()))
296 db.session.commit()
297 else:
298 log.info('\t -> skipping.')
299
300
301 if __name__ == '__main__':
302 while True:
303 try:
304 build_on_fedmsg_loop()
305 except KeyboardInterrupt:
306 sys.exit(1)
307 except:
308 log.exception('Error in fedmsg loop. Restarting it.')
309