1
2
3 import subprocess
4 import argparse
5 import sys
6 import os
7 import json
8 import time
9 import re
10 import logging
11 from flask_sqlalchemy import SQLAlchemy
12 from sqlalchemy.sql import text
13
14 sys.path.append(
15 os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
16 )
17
18 from coprs import db, app, helpers, models
19 from coprs.logic.builds_logic import BuildsLogic
20 from coprs.logic.coprs_logic import CoprsLogic
21
22 logging.basicConfig(
23 filename="{0}/check_for_anitya_version_updates.log".format(app.config.get("LOG_DIR")),
24 format='[%(asctime)s][%(levelname)6s]: %(message)s',
25 level=logging.DEBUG)
26 log = logging.getLogger(__name__)
27
28 parser = argparse.ArgumentParser(description='Fetch package version updates by using datagrepper log of anitya emitted messages and issue rebuilds of the respective COPR packages for each such update. Requires httpie package.')
29
30 parser.add_argument('--backend', action='store', default='pypi', choices=['pypi', 'rubygems'],
31 help='only check for updates from backend BACKEND, default pypi')
32 parser.add_argument('--delta', action='store', type=int, metavar='SECONDS', default=86400,
33 help='ignore updates older than SECONDS, default 86400')
34 parser.add_argument('-v', '--version', action='version', version='1.0',
35 help='print program version and exit')
36
37 args = parser.parse_args()
38
39
41 print msg
42 log.debug(msg)
43
45 print msg
46 log.info(msg)
47
49 print >> sys.stderr, msg
50 log.error(msg)
51
53 print >> sys.stderr, msg
54 log.exception(msg)
55
57 """
58 Run given command in a subprocess
59 """
60 loginfo('Executing: '+' '.join(cmd))
61 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
62 (stdout, stderr) = process.communicate()
63 if process.returncode != 0:
64 logerror(stderr)
65 sys.exit(1)
66 return stdout
67
75
77 cmd_binary = 'curl'
78 url_template = 'https://apps.fedoraproject.org/datagrepper/raw?category=anitya&delta={delta}&topic=org.release-monitoring.prod.anitya.project.version.update&rows_per_page=64&order=asc&page={page}'
79 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=1)]
80 result_json = to_json(run_cmd(get_updates_cmd))
81 messages = result_json['raw_messages']
82 pages = result_json['pages']
83
84 for p in range(2, pages+1):
85 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=p)]
86 result_json = to_json(run_cmd(get_updates_cmd))
87 messages += result_json['raw_messages']
88
89 return messages
90
100
102 pkg_name_pattern = '(' + '|'.join(updated_packages.keys()) + ')'
103 source_type = helpers.BuildSourceEnum(args.backend.lower())
104 if db.engine.url.drivername == "sqlite":
105 placeholder = '?'
106 true = '1'
107 else:
108 placeholder = '%s'
109 true = 'true'
110 rows = db.engine.execute(
111 """
112 SELECT package.id AS package_id, package.source_json AS source_json, build.pkg_version AS pkg_version, package.copr_id AS copr_id
113 FROM package
114 LEFT OUTER JOIN build ON build.package_id = package.id
115 WHERE package.source_type = {placeholder} AND
116 package.source_json ~* '{pkg_name_pattern}' AND
117 package.webhook_rebuild = {true} AND
118 (build.id is NULL OR build.id = (SELECT MAX(build.id) FROM build WHERE build.package_id = package.id));
119 """.format(placeholder=placeholder, pkg_name_pattern=pkg_name_pattern, true=true), source_type
120 )
121 return rows
122
126
127 - def build(self, copr, new_update_version):
129
130
135
136 - def build(self, copr, new_updated_version):
138
139
141 try:
142 return {
143 'pypi': PyPIPackage,
144 'rubygems': RubyGemsPackage,
145 }[backend](source_json)
146 except KeyError:
147 raise Exception('Unsupported backend {0} passed as command-line argument'.format(args.backend))
148
149
151 updated_packages = get_updated_packages(get_updates_messages())
152 loginfo('Updated packages according to datagrepper: {0}'.format(updated_packages))
153
154 for row in get_copr_package_info_rows(updated_packages):
155 source_json = json.loads(row.source_json)
156 package = package_from_source(args.backend.lower(), source_json)
157
158 latest_build_version = row.pkg_version
159 loginfo('candidate package for rebuild: {0}, package_id: {1}, copr_id: {2}'.format(package.name, row.package_id, row.copr_id))
160 if package.name in updated_packages:
161 new_updated_version = updated_packages[package.name]
162 logdebug('name: {0}, latest_build_version: {1}, new_updated_version {2}'.format(package.name, latest_build_version, new_updated_version))
163
164
165 if not latest_build_version or not re.match(new_updated_version, latest_build_version):
166 try:
167 copr = CoprsLogic.get_by_id(row.copr_id)[0]
168 except Exception as e:
169 logexception(e)
170 continue
171
172 loginfo('Launching {} build for package of source name: {}, package_id: {}, copr_id: {}, user_id: {}'
173 .format(args.backend.lower(), package.name, row.package_id, copr.id, copr.user.id))
174 build = package.build(copr, new_updated_version)
175 db.session.commit()
176 loginfo('Launched build id {0}'.format(build.id))
177
178 if __name__ == '__main__':
179 try:
180 main()
181 except Exception as e:
182 logexception(str(e))
183