Package coprs :: Package views :: Package backend_ns :: Module backend_general
[hide private]
[frames] | no frames]

Source Code for Module coprs.views.backend_ns.backend_general

  1  import flask 
  2  import time 
  3  import sqlalchemy 
  4   
  5  from coprs import db, app 
  6  from coprs import helpers 
  7  from coprs import models 
  8  from coprs import exceptions 
  9  from coprs.helpers import StatusEnum 
 10  from coprs.logic import actions_logic 
 11  from coprs.logic.builds_logic import BuildsLogic 
 12  from coprs.logic.complex_logic import ComplexLogic 
 13  from coprs.logic.coprs_logic import CoprChrootsLogic 
 14  from coprs.logic.packages_logic import PackagesLogic 
 15   
 16  from coprs.views import misc 
 17  from coprs.views.backend_ns import backend_ns 
 18  from sqlalchemy.sql import false, true 
 19   
 20  import json 
 21  import logging 
 22  log = logging.getLogger(__name__) 
23 24 25 @backend_ns.route("/importing/") 26 # FIXME I'm commented 27 #@misc.backend_authenticated 28 -def dist_git_importing_queue():
29 """ 30 Return list of builds that are waiting for dist git to import the sources. 31 """ 32 builds_list = [] 33 builds_for_import = BuildsLogic.get_build_importing_queue().filter(models.Build.is_background == false()).limit(200).all() 34 if not builds_for_import: 35 builds_for_import = BuildsLogic.get_build_importing_queue().filter(models.Build.is_background == true()).limit(30) 36 37 for task in builds_for_import: 38 copr = task.copr 39 branches = set() 40 for b_ch in task.build_chroots: 41 branches.add(b_ch.mock_chroot.distgit_branch_name) 42 43 task_dict = { 44 "task_id": task.import_task_id, 45 "owner": copr.owner_name, 46 "project": copr.name, 47 "branches": list(branches), 48 "srpm_url": task.srpm_url, 49 } 50 if task_dict not in builds_list: 51 builds_list.append(task_dict) 52 53 response_dict = {"builds": builds_list} 54 55 return flask.jsonify(response_dict)
56 57 58 @backend_ns.route("/import-completed/", methods=["POST", "PUT"])
59 @misc.backend_authenticated 60 -def dist_git_upload_completed():
61 """ 62 Mark BuildChroot in a Build as uploaded, which means: 63 - set it to pending state 64 - set BuildChroot.git_hash 65 - if it's the last BuildChroot in a Build: 66 - delete local source 67 BuildChroot is identified with task_id which is build id + git branch name 68 - For example: 56-f22 -> build 55, chroots fedora-22-* 69 """ 70 result = {"updated": False} 71 72 if "task_id" in flask.request.json and 'branch' in flask.request.json: 73 app.logger.debug(flask.request.data) 74 task_id = flask.request.json["task_id"] 75 branch = flask.request.json["branch"] 76 build_chroots = BuildsLogic.get_buildchroots_by_build_id_and_branch(task_id, branch) 77 build = build_chroots[0].build 78 79 # Is it OK? 80 if "git_hash" in flask.request.json and "repo_name" in flask.request.json: 81 git_hash = flask.request.json["git_hash"] 82 pkg_name = flask.request.json["pkg_name"] 83 pkg_version = flask.request.json["pkg_version"] 84 85 # Now I need to assign a package to this build 86 if not PackagesLogic.get(build.copr.id, pkg_name).first(): 87 try: 88 package = PackagesLogic.add(build.copr.user, build.copr, pkg_name, build.source_type, build.source_json) 89 db.session.add(package) 90 db.session.commit() 91 except (sqlalchemy.exc.IntegrityError, exceptions.DuplicateException) as e: 92 db.session.rollback() 93 94 package = PackagesLogic.get(build.copr.id, pkg_name).first() 95 build.package_id = package.id 96 build.pkg_version = pkg_version 97 98 for ch in build_chroots: 99 if ch.status == helpers.StatusEnum("importing"): 100 ch.status = helpers.StatusEnum("pending") 101 ch.git_hash = git_hash 102 103 # Failed? 104 elif "error" in flask.request.json: 105 error_type = flask.request.json["error"] 106 107 try: 108 build.fail_type = helpers.FailTypeEnum(error_type) 109 except KeyError: 110 build.fail_type = helpers.FailTypeEnum("unknown_error") 111 112 for ch in build_chroots: 113 ch.status = helpers.StatusEnum("failed") 114 115 # is it the last chroot? 116 if not build.has_importing_chroot: 117 BuildsLogic.delete_local_source(build) 118 119 db.session.commit() 120 121 result.update({"updated": True}) 122 123 return flask.jsonify(result)
124
125 126 -def get_build_record(task):
127 if not task: 128 return None 129 130 build_config = helpers.generate_build_config(task.build.copr, task.mock_chroot.name) 131 build_record = None 132 try: 133 build_record = { 134 "task_id": task.task_id, 135 "build_id": task.build.id, 136 "project_owner": task.build.copr.owner_name, 137 "project_name": task.build.copr.name, 138 "submitter": task.build.user.name if task.build.user else None, # there is no user for webhook builds 139 "chroot": task.mock_chroot.name, 140 141 "repos": task.build.repos, 142 "memory_reqs": task.build.memory_reqs, 143 "timeout": task.build.timeout, 144 "enable_net": task.build.enable_net, 145 "git_repo": task.build.package.dist_git_repo, 146 "git_hash": task.git_hash, 147 "git_branch": task.mock_chroot.distgit_branch_name, 148 "source_type": helpers.BuildSourceEnum("distgit"), 149 "source_json": json.dumps( 150 {'clone_url': task.build.package.dist_git_clone_url, 'branch': task.git_hash}), # TODO: replace branch with ref 151 152 "package_name": task.build.package.name, 153 "package_version": task.build.pkg_version, 154 "repos": build_config.get("repos"), 155 "buildroot_pkgs": build_config.get("additional_packages"), 156 "use_bootstrap_container": build_config.get("use_bootstrap_container") 157 } 158 159 except Exception as err: 160 app.logger.exception(err) 161 162 return build_record
163
164 165 -def get_srpm_build_record(task):
166 if not task: 167 return None 168 169 try: 170 build_record = { 171 "build_id": task.id, 172 "project_owner": task.copr.owner_name, 173 "project_name": task.copr.name, 174 "source_type": task.source_type, 175 "source_json": task.source_json, 176 } 177 178 except Exception as err: 179 app.logger.exception(err) 180 181 return build_record
182
183 184 @backend_ns.route("/waiting/") 185 #@misc.backend_authenticated 186 -def waiting():
187 """ 188 Return a single action and a single build. 189 """ 190 action_record = None 191 build_record = None 192 193 action = actions_logic.ActionsLogic.get_waiting().first() 194 if action: 195 action_record = action.to_dict(options={ 196 "__columns_except__": ["result", "message", "ended_on"] 197 }) 198 199 task = BuildsLogic.select_build_task() 200 build_record = get_build_record(task) 201 202 if not build_record: 203 task = BuildsLogic.select_srpm_build_task() 204 build_record = get_srpm_build_record(task) 205 206 response_dict = {"action": action_record, "build": build_record} 207 return flask.jsonify(response_dict)
208
209 210 @backend_ns.route("/get-build-task/<task_id>") 211 -def get_build_task(task_id):
212 try: 213 task = BuildsLogic.get_build_task(task_id) 214 except exceptions.MalformedArgumentException: 215 jsonout = flask.jsonify({'msg': 'Invalid task ID'}) 216 jsonout.status_code = 500 217 return jsonout 218 except sqlalchemy.orm.exc.NoResultFound: 219 jsonout = flask.jsonify({'msg': 'Specified task ID not found'}) 220 jsonout.status_code = 404 221 return jsonout 222 build_record = get_build_record(task) 223 return flask.jsonify(build_record)
224
225 226 @backend_ns.route("/get-srpm-build-task/<build_id>") 227 -def get_srpm_build_task(build_id):
228 try: 229 task = BuildsLogic.get_srpm_build_task(build_id) 230 except sqlalchemy.orm.exc.NoResultFound: 231 jsonout = flask.jsonify({'msg': 'Specified task ID not found'}) 232 jsonout.status_code = 404 233 return jsonout 234 build_record = get_srpm_build_record(task) 235 return flask.jsonify(build_record)
236 237 238 @backend_ns.route("/update/", methods=["POST", "PUT"])
239 @misc.backend_authenticated 240 -def update():
241 result = {} 242 243 request_data = flask.request.json 244 for typ, logic_cls in [("actions", actions_logic.ActionsLogic), 245 ("builds", BuildsLogic)]: 246 247 if typ not in request_data: 248 continue 249 250 to_update = {} 251 for obj in request_data[typ]: 252 to_update[obj["id"]] = obj 253 254 existing = {} 255 for obj in logic_cls.get_by_ids(to_update.keys()).all(): 256 existing[obj.id] = obj 257 258 non_existing_ids = list(set(to_update.keys()) - set(existing.keys())) 259 260 for i, obj in existing.items(): 261 logic_cls.update_state_from_dict(obj, to_update[i]) 262 263 db.session.commit() 264 result.update({"updated_{0}_ids".format(typ): list(existing.keys()), 265 "non_existing_{0}_ids".format(typ): non_existing_ids}) 266 267 return flask.jsonify(result)
268 269 270 @backend_ns.route("/starting_build/", methods=["POST", "PUT"])
271 @misc.backend_authenticated 272 -def starting_build():
273 """ 274 Check if the build is not cancelled and set it to running state 275 """ 276 277 result = {"can_start": False} 278 279 if "build_id" in flask.request.json and "chroot" in flask.request.json: 280 build = ComplexLogic.get_build_safe(flask.request.json["build_id"]) 281 chroot = flask.request.json.get("chroot") 282 283 if build and chroot and not build.canceled: 284 log.info("mark build {} chroot {} as starting".format(build.id, chroot)) 285 BuildsLogic.update_state_from_dict(build, { 286 "chroot": chroot, 287 "status": StatusEnum("starting") 288 }) 289 db.session.commit() 290 result["can_start"] = True 291 292 return flask.jsonify(result)
293 294 295 @backend_ns.route("/defer_build/", methods=["POST", "PUT"])
296 @misc.backend_authenticated 297 -def defer_build():
298 """ 299 Defer build (keep it out of waiting jobs for some time). 300 """ 301 302 result = {"was_deferred": False} 303 304 if "build_id" in flask.request.json and "chroot" in flask.request.json: 305 build = ComplexLogic.get_build_safe(flask.request.json["build_id"]) 306 chroot = flask.request.json.get("chroot") 307 308 if build and chroot: 309 log.info("Defer build {}, chroot {}".format(build.id, chroot)) 310 BuildsLogic.update_state_from_dict(build, { 311 "chroot": chroot, 312 "last_deferred": int(time.time()), 313 }) 314 db.session.commit() 315 result["was_deferred"] = True 316 317 return flask.jsonify(result)
318 319 320 @backend_ns.route("/reschedule_all_running/", methods=["POST"])
321 @misc.backend_authenticated 322 -def reschedule_all_running():
323 """ 324 Add-hoc handle. Remove after implementation of persistent task handling in copr-backend 325 """ 326 to_reschedule = \ 327 BuildsLogic.get_build_tasks(StatusEnum("starting")).all() + \ 328 BuildsLogic.get_build_tasks(StatusEnum("running")).all() 329 330 if to_reschedule: 331 for build_chroot in to_reschedule: 332 build_chroot.status = StatusEnum("pending") 333 db.session.add(build_chroot) 334 335 db.session.commit() 336 337 return "OK", 200
338 339 340 @backend_ns.route("/reschedule_build_chroot/", methods=["POST", "PUT"])
341 @misc.backend_authenticated 342 -def reschedule_build_chroot():
343 response = {} 344 if "build_id" in flask.request.json and "chroot" in flask.request.json: 345 build = ComplexLogic.get_build_safe(flask.request.json["build_id"]) 346 else: 347 response["result"] = "bad request" 348 response["msg"] = "Request missing `build_id` and/or `chroot`" 349 return flask.jsonify(response) 350 351 if build: 352 if build.canceled: 353 response["result"] = "noop" 354 response["msg"] = "build was cancelled, ignoring" 355 else: 356 chroot = flask.request.json["chroot"] 357 build_chroot = build.chroots_dict_by_name.get(chroot) 358 run_statuses = set([StatusEnum("starting"), StatusEnum("running")]) 359 if build_chroot and build_chroot.status in run_statuses: 360 log.info("rescheduling build {} chroot: {}".format(build.id, build_chroot.name)) 361 BuildsLogic.update_state_from_dict(build, { 362 "chroot": chroot, 363 "status": StatusEnum("pending") 364 }) 365 db.session.commit() 366 response["result"] = "done" 367 else: 368 response["result"] = "noop" 369 response["msg"] = "build is not in running states, ignoring" 370 371 else: 372 response["result"] = "noop" 373 response["msg"] = "Build {} wasn't found".format(flask.request.json["build_id"]) 374 375 return flask.jsonify(response)
376