Skip to content

Commit

Permalink
backend, frontend: implement project and build deletion in Pulp
Browse files Browse the repository at this point in the history 
Fix #3318
Fix #3319
  • Loading branch information
@FrostyX @praiskup
FrostyX authored and praiskup committed Sep 11, 2024
1 parent 19eff0c commit c5166a1
Show file tree
Hide file tree
Showing 6 changed files with 211 additions and 94 deletions.
138 changes: 50 additions & 88 deletions backend/copr_backend/actions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 20,13 @@
from copr_common.worker_manager import WorkerManager

from copr_backend.worker_manager import BackendQueueTask
from copr_backend.storage import storage_for_enum
from copr_backend.storage import storage_for_enum, BackendStorage

from .sign import create_user_keys, CoprKeygenRequestError
from .exceptions import CreateRepoError, CoprSignError, FrontendClientException
from .helpers import (get_redis_logger, silent_remove, ensure_dir_exists,
get_chroot_arch, format_filename,
uses_devel_repo, call_copr_repo, build_chroot_log_name,
copy2_but_hardlink_rpms)
call_copr_repo, copy2_but_hardlink_rpms)
from .sign import sign_rpms_in_dir, unsign_rpms_in_dir, get_pubkey


Expand Down Expand Up @@ -64,7 63,6 @@ def get_action_class(cls, action):
ActionTypeEnum("rawhide_to_release"): RawhideToRelease,
ActionTypeEnum("fork"): Fork,
ActionTypeEnum("build_module"): BuildModule,
ActionTypeEnum("delete"): Delete,
ActionTypeEnum("remove_dirs"): RemoveDirs,
}.get(action_type, None)

Expand Down Expand Up @@ -102,8 100,15 @@ def __init__(self, opts, action, log=None):
project = self.ext_data.get("projectname")
devel = self.ext_data.get("devel")
appstream = self.ext_data.get("appstream")
self.storage = storage_for_enum(enum, owner, project, appstream,
devel, self.opts, self.log)
args = [owner, project, appstream, devel, self.opts, self.log]
self.storage = storage_for_enum(enum, *args)

# Even though we already have `self.storage` which uses an
# appropriate storage for the project (e.g. Pulp), the project
# still has some data on backend (logs, srpm-builds chroot, etc).
# Many actions need to be performed on `self.storage` and
# `self.backend_storage` at the same time
self.backend_storage = storage_for_enum(StorageEnum.backend, *args)

def __str__(self):
return "<{}(Action): {}>".format(self.__class__.__name__, self.data)
Expand Down Expand Up @@ -221,72 226,24 @@ def run(self):
return result


class Delete(Action):
"""
Abstract class for all other Delete* classes.
"""
# pylint: disable=abstract-method
def _handle_delete_builds(self, ownername, projectname, project_dirname,
chroot_builddirs, build_ids, appstream):
""" call /bin/copr-repo --delete """
devel = uses_devel_repo(self.front_url, ownername, projectname)
result = BackendResultEnum("success")
for chroot, subdirs in chroot_builddirs.items():
chroot_path = os.path.join(self.destdir, ownername, project_dirname,
chroot)
if not os.path.exists(chroot_path):
self.log.error("%s chroot path doesn't exist", chroot_path)
result = BackendResultEnum("failure")
continue

self.log.info("Deleting subdirs [%s] in %s",
", ".join(subdirs), chroot_path)

# Run createrepo first and then remove the files (to avoid old
# repodata temporarily pointing at non-existing files)!
if chroot != "srpm-builds":
# In srpm-builds we don't create repodata at all
if not call_copr_repo(chroot_path, delete=subdirs, devel=devel, appstream=appstream,
logger=self.log):
result = BackendResultEnum("failure")

for build_id in build_ids or []:
log_paths = [
os.path.join(chroot_path, build_chroot_log_name(build_id)),
# we used to create those before
os.path.join(chroot_path, 'build-{}.rsync.log'.format(build_id)),
os.path.join(chroot_path, 'build-{}.log'.format(build_id))]
for log_path in log_paths:
try:
os.unlink(log_path)
except OSError:
self.log.debug("can't remove %s", log_path)
return result


class DeleteProject(Delete):
class DeleteProject(Action):
def run(self):
self.log.debug("Action delete copr")
result = BackendResultEnum("success")

ext_data = json.loads(self.data["data"])
ownername = ext_data["ownername"]
project_dirnames = ext_data["project_dirnames"]
project_dirnames = self.ext_data["project_dirnames"]

if not ownername:
if not self.storage.owner:
self.log.error("Received empty ownername!")
result = BackendResultEnum("failure")
return result
return BackendResultEnum("failure")

for dirname in project_dirnames:
if not dirname:
self.log.warning("Received empty dirname!")
continue
path = os.path.join(self.destdir, ownername, dirname)
if os.path.exists(path):
self.log.info("Removing copr dir %s", path)
shutil.rmtree(path)
return result
self.storage.delete_project(dirname)
if not isinstance(self.storage, BackendStorage):
self.backend_storage.delete_project(dirname)

return BackendResultEnum("success")


class CompsUpdate(Action):
Expand Down Expand Up @@ -322,7 279,7 @@ def run(self):
return result


class DeleteMultipleBuilds(Delete):
class DeleteMultipleBuilds(Action):
def run(self):
self.log.debug("Action delete multiple builds.")

Expand All @@ -334,25 291,24 @@ def run(self):
# srpm-builds: [00849545, 00849546]
# fedora-30-x86_64: [00849545-example, 00849545-foo]
# [...]
ext_data = json.loads(self.data["data"])

ownername = ext_data["ownername"]
projectname = ext_data["projectname"]
project_dirnames = ext_data["project_dirnames"]
build_ids = ext_data["build_ids"]
appstream = ext_data["appstream"]
project_dirnames = self.ext_data["project_dirnames"]
build_ids = self.ext_data["build_ids"]

result = BackendResultEnum("success")
for project_dirname, chroot_builddirs in project_dirnames.items():
if BackendResultEnum("failure") == \
self._handle_delete_builds(ownername, projectname,
project_dirname, chroot_builddirs,
build_ids, appstream):
args = [project_dirname, chroot_builddirs, build_ids]
success = self.storage.delete_builds(*args)

if not isinstance(self.storage, BackendStorage):
success = self.backend_storage.delete_builds(*args) and success

if not success:
result = BackendResultEnum("failure")
return result


class DeleteBuild(Delete):
class DeleteBuild(Action):
def run(self):
self.log.info("Action delete build.")

Expand All @@ -363,25 319,31 @@ def run(self):
# chroot_builddirs:
# srpm-builds: [00849545]
# fedora-30-x86_64: [00849545-example]
ext_data = json.loads(self.data["data"])

try:
ownername = ext_data["ownername"]
build_ids = [self.data['object_id']]
projectname = ext_data["projectname"]
project_dirname = ext_data["project_dirname"]
chroot_builddirs = ext_data["chroot_builddirs"]
appstream = ext_data["appstream"]
except KeyError:
valid = "object_id" in self.data
keys = {"ownername", "projectname", "project_dirname",
"chroot_builddirs", "appstream"}
for key in keys:
if key not in self.ext_data:
valid = False
break

if not valid:
self.log.exception("Invalid action data")
return BackendResultEnum("failure")

return self._handle_delete_builds(ownername, projectname,
project_dirname, chroot_builddirs,
build_ids, appstream)
args = [
self.ext_data["project_dirname"],
self.ext_data["chroot_builddirs"],
[self.data['object_id']],
]
success = self.storage.delete_builds(*args)
if not isinstance(self.storage, BackendStorage):
success = self.backend_storage.delete_builds(*args) and success
return BackendResultEnum("success" if success else "failure")


class DeleteChroot(Delete):
class DeleteChroot(Action):
def run(self):
self.log.info("Action delete project chroot.")
chroot = self.ext_data["chrootname"]
Expand Down
21 changes: 20 additions & 1 deletion backend/copr_backend/pulp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 164,16 @@ def create_content(self, repository, path):
return requests.post(
url, data=data, files=files, **self.request_params)

def delete_content(self, repository, artifacts):
"""
Delete a list of artifacts from a repository
https://pulpproject.org/pulp_rpm/restapi/#tag/Repositories:-Rpm/operation/repositories_rpm_rpm_modify
"""
path = os.path.join(repository, "modify/")
url = self.config["base_url"] path
data = {"remove_content_units": artifacts}
return requests.post(url, json=data, **self.request_params)

def delete_repository(self, repository):
"""
Delete an RPM repository
Expand All @@ -190,7 200,16 @@ def wait_for_finished_task(self, task):
response = self.get_task(task)
if not response.ok:
break
if response.json()["state"] != "waiting":
if response.json()["state"] not in ["waiting", "running"]:
break
time.sleep(5)
return response

def list_distributions(self, prefix):
"""
Get a list of distributions whose names match a given prefix
https://pulpproject.org/pulp_rpm/restapi/#tag/Distributions:-Rpm/operation/distributions_rpm_rpm_list
"""
url = self.url("api/v3/distributions/rpm/rpm/?")
url = urlencode({"name__startswith": prefix})
return requests.get(url, **self.request_params)
Loading

0 comments on commit c5166a1

Please sign in to comment.