Merge pull request #1621 from ScilifelabDataCentre/dev

Prep for release: v2.12.0

Author: valyo

.adr-dir

@@ -1 +1 @@
-doc/architecture/decisions
+docs/architecture/decisions

.github/pull_request_template.md

@@ -40,7 +40,7 @@ jobs:
             updmap-sys &&
             pandoc
             --output=dds_web/static/dds-technical-overview.pdf
-            doc/technical-overview.md
+            docs/technical-overview.md
             "
       - name: Upload technical overview PDF
         uses: actions/upload-artifact@v4
@@ -64,7 +64,7 @@ jobs:
             updmap-sys &&
             pandoc
             --output=dds_web/static/dds-troubleshooting.pdf
-            doc/troubleshooting.md
+            docs/troubleshooting.md
             "
       - name: Upload troubleshooting PDF
         uses: actions/upload-artifact@v4

.github/workflows/publish_and_trivyscan.yml

@@ -4,6 +4,7 @@
 name: Release Drafter
 
 on:
+  workflow_dispatch: # Allows manual triggering of the workflow
   # Run on push to dev and master
   push:
     branches:

.github/workflows/release-drafter.yml

@@ -3,4 +3,4 @@ dds_web/static/css
 dds_web/static/node_modules
 # Would like to enable this but prettier isn't correctly formatting grid tables for pandoc
 # We need grid table markdown due to having tables with multi-column/row spans
-doc/technical-overview.md
+docs/technical-overview.md

.prettierignore

@@ -149,6 +149,7 @@ def post(self):
             curr_date = dds_web.utils.current_time()
             delete_message = ""
             is_aborted = False
+            is_queue_operation = False
 
             # Moving to Available
             if new_status == "Available":
@@ -169,33 +170,23 @@ def post(self):
                 )
             elif new_status == "Archived":
                 is_aborted = json_input.get("is_aborted", False)
+                is_queue_operation = True
                 new_status_row, delete_message = self.archive_project(
-                    project=project, current_time=curr_date, aborted=is_aborted
+                    project=project,
+                    current_time=curr_date,
+                    aborted=is_aborted,
                 )
             else:
                 raise DDSArgumentError(message="Invalid status")
 
-            try:
-                project.project_statuses.append(new_status_row)
-                project.busy = False  # TODO: Use set_busy instead?
-                db.session.commit()
-                flask.current_app.logger.info(
-                    f"Busy status set. Project: '{project.public_id}', Busy: False"
-                )
-            except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err:
-                flask.current_app.logger.exception(err)
-                db.session.rollback()
-                raise DatabaseError(
-                    message=str(err),
-                    alt_message=(
-                        "Status was not updated"
-                        + (
-                            ": Database malfunction."
-                            if isinstance(err, sqlalchemy.exc.OperationalError)
-                            else ": Server Error."
-                        )
-                    ),
-                ) from err
+            return_message = ""
+
+            if not is_queue_operation:
+                # If operations was not handled by a queue, commit the changes
+                self.update_project_status(project=project, new_status_row=new_status_row)
+                return_message += f"Project {project.public_id} updated to status {new_status}"
+            else:
+                return_message += f"The status of project {project.public_id} is being updated to {new_status}. The DDS is handling this in the background.  It may take some time to complete."
 
             # Mail users once project is made available
             if new_status == "Available" and send_email:
@@ -204,10 +195,6 @@ def post(self):
                         userobj=user.researchuser, mail_type="project_release", project=project
                     )
 
-            return_message = f"{project.public_id} updated to status {new_status}" + (
-                " (aborted)" if new_status == "Archived" and is_aborted else ""
-            )
-
             if new_status != "Available":
                 return_message += delete_message + "."
             else:
@@ -235,9 +222,11 @@ def patch(self):
         # Get json input from request
         json_input = flask.request.get_json(silent=True)  # Already checked by json_required
 
-        # the status has changed at least two times,
-        # next time the project expires it wont change again -> error
-        if project.times_expired >= 2:
+        # A project can have the expired status 3 times
+        # It can be released / extended 3 times
+        # If more attempts --> cannot be extended again
+        # will be automatically archived by the system
+        if project.times_expired > 3:
             raise DDSArgumentError(
                 "Project availability limit: The maximum number of changes in data availability has been reached."
             )
@@ -411,9 +400,10 @@ def release_project(
             days=deadline_in
         )
 
-        # Project can only move from Expired 2 times
+        # Project can only MOVE FROM Expired 2 times
+        # but can BE IN Exired 3 times
         if project.current_status == "Expired":
-            if project.times_expired > 2:
+            if project.times_expired > 3:
                 raise DDSArgumentError(
                     "Project availability limit: Project cannot be made Available any more times"
                 )
@@ -506,7 +496,10 @@ def delete_project(self, project: models.Project, current_time: datetime.datetim
         return models.ProjectStatuses(status="Deleted", date_created=current_time), delete_message
 
     def archive_project(
-        self, project: models.Project, current_time: datetime.datetime, aborted: bool = False
+        self,
+        project: models.Project,
+        current_time: datetime.datetime,
+        aborted: bool = False,
     ):
         """Archive project: Make status Archived.
 
@@ -522,20 +515,47 @@ def archive_project(
                     "Please abort the project if you wish to proceed."
                 )
 
+        # Get redis connection to add a job if needed
+        redis_url = flask.current_app.config.get("REDIS_URL")
+        r = Redis.from_url(redis_url)
+        q = Queue(connection=r)
+
+        job_delete_contents = q.enqueue(
+            self.queue_helper_function_delete_contents,
+            project_id=project.public_id,  # It is not possible to pass the project object directly to the queue
+            clear_proj_info=aborted,  # If aborted, clear project info
+        )
+
+        job_update_db = q.enqueue(
+            self.queue_helper_function_update_proj_status,
+            project_id=project.public_id,
+            current_time=current_time,
+            new_status="Archived",
+            aborted=aborted,
+            depends_on=job_delete_contents,  # This job is only executed after success of delete contents
+        )
+
+        return None, ""  # Dummy returns to not break the main function
+
+    @dbsession
+    def queue_helper_function_delete_contents(self, project_id: int, clear_proj_info: bool = False):
+        """Delete the project contents for the archiving operation.
+        Function to be called by the queue."""
+
+        project = models.Project.query.filter_by(public_id=project_id).one_or_none()
+
         try:
             # Deletes files (also commits session in the function - possibly refactor later)
             RemoveContents().delete_project_contents(
                 project_id=project.public_id, delete_bucket=True
             )
-            delete_message = f"\nAll files in {project.public_id} deleted"
             self.rm_project_user_keys(project=project)
 
             # Only mark as inactive after all deletion operations succeed
             project.is_active = False
             # Delete metadata from project row
-            if aborted:
+            if clear_proj_info:
                 project = self.delete_project_info(project)
-                delete_message += " and project info cleared"
 
         except (TypeError, DatabaseError, DeletionError, BucketNotFoundError) as err:
             flask.current_app.logger.exception(err)
@@ -546,11 +566,35 @@ def archive_project(
                 pass_message=True,
             ) from err
 
-        return (
-            models.ProjectStatuses(
-                status="Archived", date_created=current_time, is_aborted=aborted
-            ),
-            delete_message,
+    @dbsession
+    def queue_helper_function_update_proj_status(
+        self,
+        project_id: int,
+        current_time: datetime.datetime,
+        new_status: str,
+        aborted: bool = False,
+    ):
+        """When the delete contents operation has being sucesfully executed. Perform the update in the DB.
+        Function ONLY to be called by the queue.
+        """
+
+        project = models.Project.query.filter_by(public_id=project_id).one_or_none()
+
+        new_status_row = models.ProjectStatuses(
+            status=new_status, date_created=current_time, is_aborted=aborted
+        )
+        self.update_project_status(project=project, new_status_row=new_status_row)
+
+    @dbsession
+    def update_project_status(
+        self, project: models.Project, new_status_row: models.ProjectStatuses
+    ):
+        """Update the project status in the database."""
+        project.project_statuses.append(new_status_row)
+        project.busy = False  # TODO: Use set_busy instead?
+        db.session.commit()
+        flask.current_app.logger.info(
+            f"Busy status set. Project: '{project.public_id}', Busy: False"
         )
 
     def rm_project_user_keys(self, project):