1 import base64
2 import datetime
3 from functools import wraps
4 import json
5 import os
6 import flask
7 import sqlalchemy
8 import json
9 import requests
10 from wtforms import ValidationError
11
12 from werkzeug import secure_filename
13
14 from coprs import db
15 from coprs import exceptions
16 from coprs import forms
17 from coprs import helpers
18 from coprs import models
19 from coprs.helpers import fix_protocol_for_backend, generate_build_config
20 from coprs.logic.api_logic import MonitorWrapper
21 from coprs.logic.builds_logic import BuildsLogic
22 from coprs.logic.complex_logic import ComplexLogic
23 from coprs.logic.users_logic import UsersLogic
24 from coprs.logic.packages_logic import PackagesLogic
25 from coprs.logic.modules_logic import ModulesLogic
26
27 from coprs.views.misc import login_required, api_login_required
28
29 from coprs.views.api_ns import api_ns
30
31 from coprs.logic import builds_logic
32 from coprs.logic import coprs_logic
33 from coprs.logic.coprs_logic import CoprsLogic
34 from coprs.logic.actions_logic import ActionsLogic
35
36 from coprs.exceptions import (ActionInProgressException,
37 InsufficientRightsException,
38 DuplicateException,
39 LegacyApiError,
40 UnknownSourceTypeException)
53 return wrapper
54
58 """
59 Render the home page of the api.
60 This page provides information on how to call/use the API.
61 """
62
63 return flask.render_template("api.html")
64
65
66 @api_ns.route("/new/", methods=["GET", "POST"])
87
90 infos = []
91
92
93 proxyuser_keys = ["username"]
94 allowed = form.__dict__.keys() + proxyuser_keys
95 for post_key in flask.request.form.keys():
96 if post_key not in allowed:
97 infos.append("Unknown key '{key}' received.".format(key=post_key))
98 return infos
99
100
101 @api_ns.route("/status")
102 -def api_status():
112
113
114 @api_ns.route("/coprs/<username>/new/", methods=["POST"])
117 """
118 Receive information from the user on how to create its new copr,
119 check their validity and create the corresponding copr.
120
121 :arg name: the name of the copr to add
122 :arg chroots: a comma separated list of chroots to use
123 :kwarg repos: a comma separated list of repository that this copr
124 can use.
125 :kwarg initial_pkgs: a comma separated list of initial packages to
126 build in this new copr
127
128 """
129
130 form = forms.CoprFormFactory.create_form_cls()(csrf_enabled=False)
131 infos = []
132
133
134 infos.extend(validate_post_keys(form))
135
136 if form.validate_on_submit():
137 group = ComplexLogic.get_group_by_name_safe(username[1:]) if username[0] == "@" else None
138
139 auto_prune = True
140 if "auto_prune" in flask.request.form:
141 auto_prune = form.auto_prune.data
142
143 try:
144 copr = CoprsLogic.add(
145 name=form.name.data.strip(),
146 repos=" ".join(form.repos.data.split()),
147 user=flask.g.user,
148 selected_chroots=form.selected_chroots,
149 description=form.description.data,
150 instructions=form.instructions.data,
151 check_for_duplicates=True,
152 disable_createrepo=form.disable_createrepo.data,
153 unlisted_on_hp=form.unlisted_on_hp.data,
154 build_enable_net=form.build_enable_net.data,
155 group=group,
156 persistent=form.persistent.data,
157 auto_prune=auto_prune,
158 )
159 infos.append("New project was successfully created.")
160
161 if form.initial_pkgs.data:
162 pkgs = form.initial_pkgs.data.split()
163 for pkg in pkgs:
164 builds_logic.BuildsLogic.add(
165 user=flask.g.user,
166 pkgs=pkg,
167 copr=copr)
168
169 infos.append("Initial packages were successfully "
170 "submitted for building.")
171
172 output = {"output": "ok", "message": "\n".join(infos)}
173 db.session.commit()
174 except (exceptions.DuplicateException,
175 exceptions.NonAdminCannotCreatePersistentProject,
176 exceptions.NonAdminCannotDisableAutoPrunning) as err:
177 db.session.rollback()
178 raise LegacyApiError(str(err))
179
180 else:
181 errormsg = "Validation error\n"
182 if form.errors:
183 for field, emsgs in form.errors.items():
184 errormsg += "- {0}: {1}\n".format(field, "\n".join(emsgs))
185
186 errormsg = errormsg.replace('"', "'")
187 raise LegacyApiError(errormsg)
188
189 return flask.jsonify(output)
190
191
192 @api_ns.route("/coprs/<username>/<coprname>/delete/", methods=["POST"])
217
218
219 @api_ns.route("/coprs/<username>/<coprname>/fork/", methods=["POST"])
220 @api_login_required
221 @api_req_with_copr
222 -def api_copr_fork(copr):
223 """ Fork the project and builds in it
224 """
225 form = forms.CoprForkFormFactory\
226 .create_form_cls(copr=copr, user=flask.g.user, groups=flask.g.user.user_groups)(csrf_enabled=False)
227
228 if form.validate_on_submit() and copr:
229 try:
230 dstgroup = ([g for g in flask.g.user.user_groups if g.at_name == form.owner.data] or [None])[0]
231 if flask.g.user.name != form.owner.data and not dstgroup:
232 return LegacyApiError("There is no such group: {}".format(form.owner.data))
233
234 fcopr, created = ComplexLogic.fork_copr(copr, flask.g.user, dstname=form.name.data, dstgroup=dstgroup)
235 if created:
236 msg = ("Forking project {} for you into {}.\nPlease be aware that it may take a few minutes "
237 "to duplicate a backend data.".format(copr.full_name, fcopr.full_name))
238 elif not created and form.confirm.data == True:
239 msg = ("Updating packages in {} from {}.\nPlease be aware that it may take a few minutes "
240 "to duplicate a backend data.".format(copr.full_name, fcopr.full_name))
241 else:
242 raise LegacyApiError("You are about to fork into existing project: {}\n"
243 "Please use --confirm if you really want to do this".format(fcopr.full_name))
244
245 output = {"output": "ok", "message": msg}
246 db.session.commit()
247
248 except (exceptions.ActionInProgressException,
249 exceptions.InsufficientRightsException) as err:
250 db.session.rollback()
251 raise LegacyApiError(str(err))
252 else:
253 raise LegacyApiError("Invalid request: {0}".format(form.errors))
254
255 return flask.jsonify(output)
256
257
258 @api_ns.route("/coprs/")
259 @api_ns.route("/coprs/<username>/")
260 -def api_coprs_by_owner(username=None):
261 """ Return the list of coprs owned by the given user.
262 username is taken either from GET params or from the URL itself
263 (in this order).
264
265 :arg username: the username of the person one would like to the
266 coprs of.
267
268 """
269 username = flask.request.args.get("username", None) or username
270 if username is None:
271 raise LegacyApiError("Invalid request: missing `username` ")
272
273 release_tmpl = "{chroot.os_release}-{chroot.os_version}-{chroot.arch}"
274
275 if username.startswith("@"):
276 group_name = username[1:]
277 query = CoprsLogic.get_multiple()
278 query = CoprsLogic.filter_by_group_name(query, group_name)
279 else:
280 query = CoprsLogic.get_multiple_owned_by_username(username)
281
282 query = CoprsLogic.join_builds(query)
283 query = CoprsLogic.set_query_order(query)
284
285 repos = query.all()
286 output = {"output": "ok", "repos": []}
287 for repo in repos:
288 yum_repos = {}
289 for build in repo.builds:
290 if build.results:
291 for chroot in repo.active_chroots:
292 release = release_tmpl.format(chroot=chroot)
293 yum_repos[release] = fix_protocol_for_backend(
294 os.path.join(build.results, release + '/'))
295 break
296
297 output["repos"].append({"name": repo.name,
298 "additional_repos": repo.repos,
299 "yum_repos": yum_repos,
300 "description": repo.description,
301 "instructions": repo.instructions,
302 "persistent": repo.persistent,
303 "unlisted_on_hp": repo.unlisted_on_hp,
304 "auto_prune": repo.auto_prune,
305 })
306
307 return flask.jsonify(output)
308
313 """ Return detail of one project.
314
315 :arg username: the username of the person one would like to the
316 coprs of.
317 :arg coprname: the name of project.
318
319 """
320 release_tmpl = "{chroot.os_release}-{chroot.os_version}-{chroot.arch}"
321 output = {"output": "ok", "detail": {}}
322 yum_repos = {}
323
324 build = models.Build.query.filter(
325 models.Build.copr_id == copr.id, models.Build.results != None).first()
326
327 if build:
328 for chroot in copr.active_chroots:
329 release = release_tmpl.format(chroot=chroot)
330 yum_repos[release] = fix_protocol_for_backend(
331 os.path.join(build.results, release + '/'))
332
333 output["detail"] = {
334 "name": copr.name,
335 "additional_repos": copr.repos,
336 "yum_repos": yum_repos,
337 "description": copr.description,
338 "instructions": copr.instructions,
339 "last_modified": builds_logic.BuildsLogic.last_modified(copr),
340 "auto_createrepo": copr.auto_createrepo,
341 "persistent": copr.persistent,
342 "unlisted_on_hp": copr.unlisted_on_hp,
343 "auto_prune": copr.auto_prune,
344 }
345 return flask.jsonify(output)
346
347
348 @api_ns.route("/auth_check/", methods=["POST"])
351 output = {"output": "ok"}
352 return flask.jsonify(output)
353
354
355 @api_ns.route("/coprs/<username>/<coprname>/new_build/", methods=["POST"])
356 @api_login_required
357 @api_req_with_copr
358 -def copr_new_build(copr):
370 return process_creating_new_build(copr, form, create_new_build)
371
372
373 @api_ns.route("/coprs/<username>/<coprname>/new_build_upload/", methods=["POST"])
387 return process_creating_new_build(copr, form, create_new_build)
388
389
390 @api_ns.route("/coprs/<username>/<coprname>/new_build_pypi/", methods=["POST"])
410 return process_creating_new_build(copr, form, create_new_build)
411
412
413 @api_ns.route("/coprs/<username>/<coprname>/new_build_tito/", methods=["POST"])
430 return process_creating_new_build(copr, form, create_new_build)
431
432
433 @api_ns.route("/coprs/<username>/<coprname>/new_build_mock/", methods=["POST"])
451 return process_creating_new_build(copr, form, create_new_build)
452
453
454 @api_ns.route("/coprs/<username>/<coprname>/new_build_rubygems/", methods=["POST"])
468 return process_creating_new_build(copr, form, create_new_build)
469
470
471 @api_ns.route("/coprs/<username>/<coprname>/new_build_distgit/", methods=["POST"])
486 return process_creating_new_build(copr, form, create_new_build)
487
490 infos = []
491
492
493 infos.extend(validate_post_keys(form))
494
495 if not form.validate_on_submit():
496 raise LegacyApiError("Invalid request: bad request parameters: {0}".format(form.errors))
497
498 if not flask.g.user.can_build_in(copr):
499 raise LegacyApiError("Invalid request: user {} is not allowed to build in the copr: {}"
500 .format(flask.g.user.username, copr.full_name))
501
502
503 try:
504
505
506 build = create_new_build()
507 db.session.commit()
508 ids = [build.id] if type(build) != list else [b.id for b in build]
509 infos.append("Build was added to {0}:".format(copr.name))
510 for build_id in ids:
511 infos.append(" " + flask.url_for("coprs_ns.copr_build_redirect",
512 build_id=build_id,
513 _external=True))
514
515 except (ActionInProgressException, InsufficientRightsException) as e:
516 raise LegacyApiError("Invalid request: {}".format(e))
517
518 output = {"output": "ok",
519 "ids": ids,
520 "message": "\n".join(infos)}
521
522 return flask.jsonify(output)
523
524
525 @api_ns.route("/coprs/build_status/<int:build_id>/", methods=["GET"])
532
533
534 @api_ns.route("/coprs/build_detail/<int:build_id>/", methods=["GET"])
535 @api_ns.route("/coprs/build/<int:build_id>/", methods=["GET"])
537 build = ComplexLogic.get_build_safe(build_id)
538
539 chroots = {}
540 results_by_chroot = {}
541 for chroot in build.build_chroots:
542 chroots[chroot.name] = chroot.state
543 results_by_chroot[chroot.name] = chroot.result_dir_url
544
545 built_packages = None
546 if build.built_packages:
547 built_packages = build.built_packages.split("\n")
548
549 output = {
550 "output": "ok",
551 "status": build.state,
552 "project": build.copr.name,
553 "owner": build.copr.owner_name,
554 "results": build.results,
555 "built_pkgs": built_packages,
556 "src_version": build.pkg_version,
557 "chroots": chroots,
558 "submitted_on": build.submitted_on,
559 "started_on": build.min_started_on,
560 "ended_on": build.max_ended_on,
561 "src_pkg": build.pkgs,
562 "submitted_by": build.user.name if build.user else None,
563 "results_by_chroot": results_by_chroot
564 }
565 return flask.jsonify(output)
566
567
568 @api_ns.route("/coprs/cancel_build/<int:build_id>/", methods=["POST"])
581
582
583 @api_ns.route("/coprs/delete_build/<int:build_id>/", methods=["POST"])
596
597
598 @api_ns.route('/coprs/<username>/<coprname>/modify/', methods=["POST"])
599 @api_login_required
600 @api_req_with_copr
601 -def copr_modify(copr):
602 form = forms.CoprModifyForm(csrf_enabled=False)
603
604 if not form.validate_on_submit():
605 raise LegacyApiError("Invalid request: {0}".format(form.errors))
606
607
608
609 if form.description.raw_data and len(form.description.raw_data):
610 copr.description = form.description.data
611 if form.instructions.raw_data and len(form.instructions.raw_data):
612 copr.instructions = form.instructions.data
613 if form.repos.raw_data and len(form.repos.raw_data):
614 copr.repos = form.repos.data
615 if form.disable_createrepo.raw_data and len(form.disable_createrepo.raw_data):
616 copr.disable_createrepo = form.disable_createrepo.data
617
618 if "unlisted_on_hp" in flask.request.form:
619 copr.unlisted_on_hp = form.unlisted_on_hp.data
620 if "build_enable_net" in flask.request.form:
621 copr.build_enable_net = form.build_enable_net.data
622 if "auto_prune" in flask.request.form:
623 copr.auto_prune = form.auto_prune.data
624 if "chroots" in flask.request.form:
625 coprs_logic.CoprChrootsLogic.update_from_names(
626 flask.g.user, copr, form.chroots.data)
627
628 try:
629 CoprsLogic.update(flask.g.user, copr)
630 if copr.group:
631 _ = copr.group.id
632 db.session.commit()
633 except (exceptions.ActionInProgressException,
634 exceptions.InsufficientRightsException,
635 exceptions.NonAdminCannotDisableAutoPrunning) as e:
636 db.session.rollback()
637 raise LegacyApiError("Invalid request: {}".format(e))
638
639 output = {
640 'output': 'ok',
641 'description': copr.description,
642 'instructions': copr.instructions,
643 'repos': copr.repos,
644 'chroots': [c.name for c in copr.mock_chroots],
645 }
646
647 return flask.jsonify(output)
648
649
650 @api_ns.route('/coprs/<username>/<coprname>/modify/<chrootname>/', methods=["POST"])
667
668
669 @api_ns.route('/coprs/<username>/<coprname>/chroot/edit/<chrootname>/', methods=["POST"])
670 @api_login_required
671 @api_req_with_copr
672 -def copr_edit_chroot(copr, chrootname):
673 form = forms.ModifyChrootForm(csrf_enabled=False)
674 chroot = ComplexLogic.get_copr_chroot_safe(copr, chrootname)
675
676 if not form.validate_on_submit():
677 raise LegacyApiError("Invalid request: {0}".format(form.errors))
678 else:
679 buildroot_pkgs = repos = comps_xml = comps_name = None
680 if "buildroot_pkgs" in flask.request.form:
681 buildroot_pkgs = form.buildroot_pkgs.data
682 if "repos" in flask.request.form:
683 repos = form.repos.data
684 if form.upload_comps.has_file():
685 comps_xml = form.upload_comps.data.stream.read()
686 comps_name = form.upload_comps.data.filename
687 if form.delete_comps.data:
688 coprs_logic.CoprChrootsLogic.remove_comps(flask.g.user, chroot)
689 coprs_logic.CoprChrootsLogic.update_chroot(
690 flask.g.user, chroot, buildroot_pkgs, repos, comps=comps_xml, comps_name=comps_name)
691 db.session.commit()
692
693 output = {
694 "output": "ok",
695 "message": "Edit chroot operation was successful.",
696 "chroot": chroot.to_dict(),
697 }
698 return flask.jsonify(output)
699
700
701 @api_ns.route('/coprs/<username>/<coprname>/detail/<chrootname>/', methods=["GET"])
708
709 @api_ns.route('/coprs/<username>/<coprname>/chroot/get/<chrootname>/', methods=["GET"])
715
719 """ Return the list of coprs found in search by the given text.
720 project is taken either from GET params or from the URL itself
721 (in this order).
722
723 :arg project: the text one would like find for coprs.
724
725 """
726 project = flask.request.args.get("project", None) or project
727 if not project:
728 raise LegacyApiError("No project found.")
729
730 try:
731 query = CoprsLogic.get_multiple_fulltext(project)
732
733 repos = query.all()
734 output = {"output": "ok", "repos": []}
735 for repo in repos:
736 output["repos"].append({"username": repo.user.name,
737 "coprname": repo.name,
738 "description": repo.description})
739 except ValueError as e:
740 raise LegacyApiError("Server error: {}".format(e))
741
742 return flask.jsonify(output)
743
747 """ Return list of coprs which are part of playground """
748 query = CoprsLogic.get_playground()
749 repos = query.all()
750 output = {"output": "ok", "repos": []}
751 for repo in repos:
752 output["repos"].append({"username": repo.owner_name,
753 "coprname": repo.name,
754 "chroots": [chroot.name for chroot in repo.active_chroots]})
755
756 jsonout = flask.jsonify(output)
757 jsonout.status_code = 200
758 return jsonout
759
760
761 @api_ns.route("/coprs/<username>/<coprname>/monitor/", methods=["GET"])
762 @api_req_with_copr
763 -def monitor(copr):
767
768
769
770 @api_ns.route("/coprs/<username>/<coprname>/package/add/<source_type_text>/", methods=["POST"])
771 @api_login_required
772 @api_req_with_copr
773 -def copr_add_package(copr, source_type_text):
775
776
777 @api_ns.route("/coprs/<username>/<coprname>/package/<package_name>/edit/<source_type_text>/", methods=["POST"])
778 @api_login_required
779 @api_req_with_copr
780 -def copr_edit_package(copr, package_name, source_type_text):
786
818
821 params = {}
822 if flask.request.args.get('with_latest_build'):
823 params['with_latest_build'] = True
824 if flask.request.args.get('with_latest_succeeded_build'):
825 params['with_latest_succeeded_build'] = True
826 if flask.request.args.get('with_all_builds'):
827 params['with_all_builds'] = True
828 return params
829
832 """
833 A lagging generator to stream JSON so we don't have to hold everything in memory
834 This is a little tricky, as we need to omit the last comma to make valid JSON,
835 thus we use a lagging generator, similar to http://stackoverflow.com/questions/1630320/
836 """
837 packages = query.__iter__()
838 try:
839 prev_package = next(packages)
840 except StopIteration:
841
842 yield '{"packages": []}'
843 raise StopIteration
844
845 yield '{"packages": ['
846
847 for package in packages:
848 yield json.dumps(prev_package.to_dict(**params)) + ', '
849 prev_package = package
850
851 yield json.dumps(prev_package.to_dict(**params)) + ']}'
852
853
854 @api_ns.route("/coprs/<username>/<coprname>/package/list/", methods=["GET"])
860
861
862
863 @api_ns.route("/coprs/<username>/<coprname>/package/get/<package_name>/", methods=["GET"])
873
874
875 @api_ns.route("/coprs/<username>/<coprname>/package/delete/<package_name>/", methods=["POST"])
895
896
897 @api_ns.route("/coprs/<username>/<coprname>/package/reset/<package_name>/", methods=["POST"])
898 @api_login_required
899 @api_req_with_copr
900 -def copr_reset_package(copr, package_name):
917
918
919 @api_ns.route("/coprs/<username>/<coprname>/package/build/<package_name>/", methods=["POST"])
920 @api_login_required
921 @api_req_with_copr
922 -def copr_build_package(copr, package_name):
923 form = forms.BuildFormRebuildFactory.create_form_cls(copr.active_chroots)(csrf_enabled=False)
924
925 try:
926 package = PackagesLogic.get(copr.id, package_name)[0]
927 except IndexError:
928 raise LegacyApiError("No package with name {name} in copr {copr}".format(name=package_name, copr=copr.name))
929
930 if form.validate_on_submit():
931 try:
932 build = PackagesLogic.build_package(flask.g.user, copr, package, form.selected_chroots, **form.data)
933 db.session.commit()
934 except (InsufficientRightsException, ActionInProgressException, NoPackageSourceException) as e:
935 raise LegacyApiError(str(e))
936 else:
937 raise LegacyApiError(form.errors)
938
939 return flask.jsonify({
940 "output": "ok",
941 "ids": [build.id],
942 "message": "Build was added to {0}.".format(copr.name)
943 })
944
945
946 @api_ns.route("/module/build/", methods=["POST"])
949 form = forms.ModuleBuildForm(csrf_enabled=False)
950 if not form.validate_on_submit():
951 raise LegacyApiError(form.errors)
952
953 try:
954 common = {"owner": flask.g.user.name,
955 "copr_owner": form.copr_owner.data,
956 "copr_project": form.copr_project.data}
957 if form.scmurl.data:
958 kwargs = {"json": dict({"scmurl": form.scmurl.data, "branch": form.branch.data}, **common)}
959 else:
960 kwargs = {"data": common, "files": {"yaml": (form.modulemd.data.filename, form.modulemd.data)}}
961
962 response = requests.post(flask.current_app.config["MBS_URL"], verify=False, **kwargs)
963 if response.status_code == 500:
964 raise LegacyApiError("Error from MBS: {} - {}".format(response.status_code, response.reason))
965
966 resp = json.loads(response.content)
967 if response.status_code != 201:
968 raise LegacyApiError("Error from MBS: {}".format(resp["message"]))
969
970 return flask.jsonify({
971 "output": "ok",
972 "message": "Created module {}-{}-{}".format(resp["name"], resp["stream"], resp["version"]),
973 })
974
975 except requests.ConnectionError:
976 raise LegacyApiError("Can't connect to MBS instance")
977
978
979 @api_ns.route("/coprs/<username>/<coprname>/module/make/", methods=["POST"])
983 form = forms.ModuleFormUploadFactory(csrf_enabled=False)
984 if not form.validate_on_submit():
985
986 raise LegacyApiError(form.errors)
987
988 modulemd = form.modulemd.data.read()
989 module = ModulesLogic.from_modulemd(modulemd)
990 try:
991 ModulesLogic.validate(modulemd)
992 msg = "Nothing happened"
993 if form.create.data:
994 module = ModulesLogic.add(flask.g.user, copr, module)
995 db.session.flush()
996 msg = "Module was created"
997
998 if form.build.data:
999 if not module.id:
1000 module = ModulesLogic.get_by_nsv(copr, module.name, module.stream, module.version).one()
1001 ActionsLogic.send_build_module(flask.g.user, copr, module)
1002 msg = "Module build was submitted"
1003 db.session.commit()
1004
1005 return flask.jsonify({
1006 "output": "ok",
1007 "message": msg,
1008 "modulemd": modulemd,
1009 })
1010
1011 except sqlalchemy.exc.IntegrityError:
1012 raise LegacyApiError({"nsv": ["Module {} already exists".format(module.nsv)]})
1013
1014 except sqlalchemy.orm.exc.NoResultFound:
1015 raise LegacyApiError({"nsv": ["Module {} doesn't exist. You need to create it first".format(module.nsv)]})
1016
1017 except ValidationError as ex:
1018 raise LegacyApiError({"nsv": [ex.message]})
1019
1020
1021 @api_ns.route("/coprs/<username>/<coprname>/build-config/<chroot>/", methods=["GET"])
1022 @api_ns.route("/g/<group_name>/<coprname>/build-config/<chroot>/", methods=["GET"])
1025 """
1026 Generate build configuration.
1027 """
1028 output = {
1029 "output": "ok",
1030 "build_config": generate_build_config(copr, chroot),
1031 }
1032
1033 if not output['build_config']:
1034 raise LegacyApiError('Chroot not found.')
1035
1036 return flask.jsonify(output)
1037
1038
1039 @api_ns.route("/module/repo/", methods=["POST"])
1055