Package run :: Module build_on_pagure_commit
[hide private]
[frames] | no frames]

Source Code for Module run.build_on_pagure_commit

  1  #!/usr/bin/env python 
  2   
  3  import json 
  4  import pprint 
  5  import zmq 
  6  import sys 
  7  import os 
  8  import logging 
  9  import requests 
 10  import re 
 11   
 12  sys.path.append( 
 13      os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 
 14  ) 
 15   
 16  from coprs import db, app 
 17  from coprs.logic.coprs_logic import CoprsLogic 
 18  from coprs.logic.builds_logic import BuildsLogic 
 19  from coprs.logic.complex_logic import ComplexLogic 
 20  from coprs import helpers 
 21   
 22  try: 
 23      from urllib.parse import urlparse 
 24  except ImportError: 
 25      from urlparse import urlparse 
 26   
 27  SCM_SOURCE_TYPE = helpers.BuildSourceEnum("scm") 
 28   
 29  logging.basicConfig( 
 30      filename="{0}/build_on_pagure_commit.log".format(app.config.get("LOG_DIR")), 
 31      format='[%(asctime)s][%(levelname)6s]: %(message)s', 
 32      level=logging.DEBUG) 
 33   
 34  log = logging.getLogger(__name__) 
 35  log.addHandler(logging.StreamHandler(sys.stdout)) 
 36   
 37  PAGURE_BASE_URL = "https://pagure.io/" 
 38  SRCFPO_BASE_URL = "https://src.fedoraproject.org/" 
 39   
 40  ENDPOINT = 'tcp://hub.fedoraproject.org:9940' 
 41  TOPICS = ['io.pagure.prod.pagure.git.receive', 'org.fedoraproject.prod.git.receive'] 
42 43 -class ScmPackage(object):
44 - def __init__(self, db_row):
45 self.pkg_id = db_row.package_id 46 self.copr_id = db_row.copr_id 47 48 self.source_json_dict = json.loads(db_row.source_json) 49 self.clone_url = self.source_json_dict.get('clone_url') or '' 50 self.committish = self.source_json_dict.get('committish') or '' 51 self.subdirectory = self.source_json_dict.get('subdirectory') or '' 52 53 self.package = ComplexLogic.get_package_by_id_safe(self.pkg_id)
54
55 - def build(self, branch):
56 BuildsLogic.rebuild_package(self.package, {'committish': branch}) 57 db.session.commit()
58 59 @classmethod
60 - def get_candidates_for_rebuild(cls, clone_url):
61 if db.engine.url.drivername == "sqlite": 62 placeholder = '?' 63 true = '1' 64 else: 65 placeholder = '%s' 66 true = 'true' 67 68 rows = db.engine.execute( 69 """ 70 SELECT package.id AS package_id, package.source_json AS source_json, package.copr_id AS copr_id 71 FROM package 72 WHERE package.source_type = {0} AND 73 package.webhook_rebuild = {1} AND 74 package.source_json ILIKE {placeholder} 75 """.format(SCM_SOURCE_TYPE, true, placeholder=placeholder), '%'+clone_url+'%' 76 ) 77 return [ScmPackage(row) for row in rows]
78
79 - def is_dir_in_commit(self, raw_commit_text):
80 if not self.subdirectory or not raw_commit_text: 81 return True 82 83 for line in raw_commit_text.split('\n'): 84 match = re.search(r'^(\+\+\+|---) [ab]/(\w*)/?.*$', line) 85 if match and match.group(2).lower() == self.subdirectory.strip('./').lower(): 86 return True 87 88 return False
89
90 91 -def build_on_fedmsg_loop():
92 log.debug("Setting up poller...") 93 94 ctx = zmq.Context() 95 s = ctx.socket(zmq.SUB) 96 s.connect(ENDPOINT) 97 98 for topic in TOPICS: 99 s.setsockopt(zmq.SUBSCRIBE, topic) 100 101 poller = zmq.Poller() 102 poller.register(s, zmq.POLLIN) 103 104 while True: 105 log.debug("Polling...") 106 evts = poller.poll(10000) 107 if not evts: 108 continue 109 110 log.debug("Receiving...") 111 topic, msg = s.recv_multipart() 112 113 log.debug("Parsing...") 114 data = json.loads(msg) 115 116 if data['topic'] == 'io.pagure.prod.pagure.git.receive': 117 project_url_path = data['msg']['repo']['url_path'] 118 clone_url_path = data['msg']['repo']['fullname'] 119 branch = data['msg']['branch'] 120 start_commit = data['msg']['start_commit'] 121 end_commit = data['msg']['end_commit'] 122 base_url = PAGURE_BASE_URL 123 clone_url = base_url + clone_url_path 124 else: 125 # TODO: make the following work for forks 126 # when the messages are actually emitted 127 project_url_path = data['msg']['commit']['namespace'] + '/' + data['msg']['commit']['repo'] 128 clone_url_path = project_url_path 129 branch = data['msg']['commit']['branch'] 130 start_commit = data['msg']['commit']['rev'] 131 end_commit = data['msg']['commit']['rev'] 132 base_url = SRCFPO_BASE_URL 133 clone_url = base_url + clone_url_path 134 135 log.info("MSG:") 136 log.info("\tclone_url_path = {}".format(clone_url_path)) 137 log.info("\tbranch = {}".format(branch)) 138 139 candidates = ScmPackage.get_candidates_for_rebuild(clone_url) 140 raw_commit_text = None 141 if candidates: 142 # if start_commit != end_commit 143 # then more than one commit and no means to iterate over :( 144 raw_commit_url = base_url + project_url_path + '/raw/' + end_commit 145 r = requests.get(raw_commit_url) 146 if r.status_code == requests.codes.ok: 147 raw_commit_text = r.text 148 else: 149 log.error("Bad http status {0} from url {1}".format(r.status_code, raw_commit_url)) 150 151 for pkg in candidates: 152 log.info("Considering pkg id: {}, source_json: {}".format(pkg.pkg_id, pkg.source_json_dict)) 153 if (pkg.clone_url == clone_url or pkg.clone_url == clone_url+'.git') \ 154 and (not pkg.committish or branch.endswith(pkg.committish)) \ 155 and pkg.is_dir_in_commit(raw_commit_text): 156 log.info("\t -> rebuilding.") 157 pkg.build(branch) 158 else: 159 log.info("\t -> skipping.")
160 161 162 if __name__ == '__main__': 163 while True: 164 try: 165 build_on_fedmsg_loop() 166 except KeyboardInterrupt: 167 sys.exit(1) 168 except: 169 log.exception('Error in fedmsg loop. Restarting it.') 170