diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e6671dded..e40018673 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,9 +45,6 @@ jobs: - core - filestorage - __flaky__ - storage: - - default - - legacy_storage continue-on-error: true steps: - name: Checkout repo @@ -68,17 +65,6 @@ jobs: cat <> .env STORAGES='{ - "legacy_storage": { - "BACKEND": "qfieldcloud.filestorage.backend.QfcS3Boto3Storage", - "OPTIONS": { - "access_key": "minioadmin", - "secret_key": "minioadmin", - "bucket_name": "qfieldcloud-local-legacy", - "region_name": "", - "endpoint_url": "http://172.17.0.1:8009" - }, - "QFC_IS_LEGACY": true - }, "webdav": { "BACKEND": "qfieldcloud.filestorage.backend.QfcWebDavStorage", "OPTIONS": { @@ -102,7 +88,7 @@ jobs: }' EOF - echo "STORAGES_PROJECT_DEFAULT_STORAGE=${{ matrix.storage }}" >> .env + echo "STORAGES_PROJECT_DEFAULT_STORAGE=default" >> .env - name: Pull docker containers run: docker compose pull diff --git a/docker-app/qfieldcloud/core/migrations/0081_file_storage_project_and_more.py b/docker-app/qfieldcloud/core/migrations/0081_file_storage_project_and_more.py index 9e27829d9..0c34b1369 100644 --- a/docker-app/qfieldcloud/core/migrations/0081_file_storage_project_and_more.py +++ b/docker-app/qfieldcloud/core/migrations/0081_file_storage_project_and_more.py @@ -1,7 +1,6 @@ # Generated by Django 3.2.25 on 2024-06-18 01:02 import django.core.validators -from django.conf import settings from django.db import migrations, models import qfieldcloud.core.models @@ -10,7 +9,7 @@ def get_file_storage_name(): - return settings.LEGACY_STORAGE_NAME or "default" + return "default" class Migration(migrations.Migration): diff --git a/docker-app/qfieldcloud/core/migrations/0095_remove_project_legacy_thumbnail_uri_and_more.py b/docker-app/qfieldcloud/core/migrations/0095_remove_project_legacy_thumbnail_uri_and_more.py new file mode 100644 index 000000000..f060fa4a9 --- /dev/null +++ b/docker-app/qfieldcloud/core/migrations/0095_remove_project_legacy_thumbnail_uri_and_more.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.28 on 2026-02-11 07:31 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0094_alter_project_are_attachments_versioned_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="project", + name="legacy_thumbnail_uri", + ), + migrations.RemoveField( + model_name="useraccount", + name="legacy_avatar_uri", + ), + ] diff --git a/docker-app/qfieldcloud/core/models.py b/docker-app/qfieldcloud/core/models.py index 2f23761ff..a918fbe55 100644 --- a/docker-app/qfieldcloud/core/models.py +++ b/docker-app/qfieldcloud/core/models.py @@ -10,7 +10,6 @@ from typing import TYPE_CHECKING, Any, cast from uuid import uuid4 -from deprecated import deprecated from django.conf import settings from django.contrib.auth.models import AbstractUser from django.contrib.auth.models import UserManager as DjangoUserManager @@ -39,9 +38,8 @@ ) from timezone_field import TimeZoneField -from qfieldcloud.core import utils, validators +from qfieldcloud.core import validators from qfieldcloud.core.fields import DynamicStorageFileField, QfcImageField, QfcImageFile -from qfieldcloud.core.utils2 import storage from qfieldcloud.subscription.exceptions import ReachedMaxOrganizationMembersError if TYPE_CHECKING: @@ -467,13 +465,6 @@ class UserAccount(models.Model): twitter = models.CharField(max_length=255, default="", blank=True) is_email_public = models.BooleanField(default=False) - # TODO Delete with QF-4963 Drop support for legacy storage - legacy_avatar_uri = models.CharField( - _("Legacy Profile Picture URI"), - max_length=255, - blank=True, - ) - avatar = QfcImageField( _("Avatar Picture"), upload_to=get_user_account_avatar_upload_to, @@ -518,23 +509,12 @@ def storage_used_bytes(self) -> float: project_files_used_quota = ( FileVersion.objects.filter( file__file_type=File.FileType.PROJECT_FILE, - file__project__in=self.user.projects.exclude( - file_storage=settings.LEGACY_STORAGE_NAME - ), + file__project__in=self.user.projects.all(), ).aggregate(sum_bytes=Sum("size"))["sum_bytes"] or 0 ) - # TODO: Delete with QF-4963 Drop support for legacy storage - legacy_used_quota = ( - self.user.projects.filter( - file_storage=settings.LEGACY_STORAGE_NAME - ).aggregate(sum_bytes=Sum("file_storage_bytes"))["sum_bytes"] - # if there are no projects, the value will be `None` - or 0 - ) - - used_quota = project_files_used_quota + legacy_used_quota + used_quota = project_files_used_quota return used_quota @@ -1211,11 +1191,6 @@ class Meta: ), ) - # TODO: Delete with QF-4963 Drop support for legacy storage - legacy_thumbnail_uri = models.CharField( - _("Legacy Thumbnail Picture URI"), max_length=255, blank=True - ) - thumbnail = DynamicStorageFileField( _("Thumbnail Picture"), upload_to=get_project_thumbnail_upload_to, @@ -1491,17 +1466,9 @@ def owner_aware_storage_keep_versions(self) -> int: @property def thumbnail_url(self) -> StrOrPromise: - """Returns the url to the project's thumbnail or empty string if no URL provided. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - if self.uses_legacy_storage: - if not self.legacy_thumbnail_uri: - return "" - else: - if not self.thumbnail: - return "" + """Returns the url to the project's thumbnail or empty string if no URL provided.""" + if not self.thumbnail: + return "" return reverse_lazy( "filestorage_project_thumbnails", @@ -1575,30 +1542,6 @@ def private(self) -> bool: # still used in the project serializer return not self.is_public - @property - def uses_legacy_storage(self) -> bool: - """Whether the storage of the project is legacy. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - return self.file_storage == settings.LEGACY_STORAGE_NAME - - @cached_property - @deprecated - def legacy_files(self) -> list[utils.S3ObjectWithVersions]: - """Gets all the files from S3 storage. This is potentially slow. Results are cached on the instance. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - if self.uses_legacy_storage: - return list(utils.get_project_files_with_versions(str(self.id))) - else: - raise NotImplementedError( - "The `Project.legacy_files` method is not implemented for projects stored in non-legacy storage" - ) - @property def project_files(self) -> "FileQueryset": """Returns the files of type PROJECT related to the project.""" @@ -1606,14 +1549,7 @@ def project_files(self) -> "FileQueryset": @property def project_files_count(self) -> int: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - if self.uses_legacy_storage: - return len(self.legacy_files) - else: - return self.project_files.count() + return self.project_files.count() @property def users(self): @@ -1852,14 +1788,7 @@ def direct_collaborators(self) -> ProjectCollaboratorQueryset: ) def delete(self, *args, **kwargs): - """Deletes the project and the thumbnail for the legacy storage. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - if self.uses_legacy_storage: - storage.delete_project_thumbnail(self) - + """Deletes the project and the thumbnail for the legacy storage.""" return super().delete(*args, **kwargs) @property @@ -1878,15 +1807,9 @@ def save(self, recompute_storage=False, *args, **kwargs): additional_update_fields = set() if recompute_storage: - # TODO Delete with QF-4963 Drop support for legacy storage - if self.uses_legacy_storage: - self.file_storage_bytes = storage.get_project_file_storage_in_bytes( - self - ) - else: - self.file_storage_bytes = self.project_files.aggregate( - file_storage_bytes=Sum("versions__size", default=0) - )["file_storage_bytes"] + self.file_storage_bytes = self.project_files.aggregate( + file_storage_bytes=Sum("versions__size", default=0) + )["file_storage_bytes"] additional_update_fields.add("file_storage_bytes") @@ -1912,11 +1835,6 @@ def save(self, recompute_storage=False, *args, **kwargs): def get_file(self, filename: str) -> File: return self.project_files.get_by_name(filename) # type: ignore - def legacy_get_file(self, filename: str) -> utils.S3ObjectWithVersions: - files = filter(lambda f: f.latest.name == filename, self.legacy_files) - - return next(files) - class ProjectCollaboratorQueryset(models.QuerySet): def validated(self, skip_invalid=False): @@ -2710,10 +2628,5 @@ class Meta: ) def _get_file_storage_name(self) -> str: - # TODO Delete with QF-4963 Drop support for legacy storage - # Legacy storage - use default storage - if self.project.uses_legacy_storage: - return "default" - - # Non-legacy storage - use same storage as project + # Use same storage as project return self.project.file_storage diff --git a/docker-app/qfieldcloud/core/tests/test_commands.py b/docker-app/qfieldcloud/core/tests/test_commands.py index d40885ea9..46524a94d 100644 --- a/docker-app/qfieldcloud/core/tests/test_commands.py +++ b/docker-app/qfieldcloud/core/tests/test_commands.py @@ -1,5 +1,4 @@ import csv -import io import os from django.core.files.base import ContentFile @@ -8,7 +7,6 @@ from qfieldcloud.core.models import Person, Project from qfieldcloud.core.tests.utils import set_subscription, setup_subscription_plans -from qfieldcloud.core.utils2 import storage from qfieldcloud.filestorage.models import File, FileVersion @@ -23,20 +21,14 @@ def setUpTestData(cls): # Project cls.p1 = Project.objects.create(name="test_project", owner=user) - # TODO Delete with QF-4963 Drop support for legacy storage - if cls.p1.uses_legacy_storage: - storage.upload_project_file( - cls.p1, io.BytesIO(b"Hello world!"), "project.qgs" - ) - else: - FileVersion.objects.add_version( - project=cls.p1, - filename="file.name", - # NOTE the dummy name is required when running tests on GitHub CI, but not locally. Spent few hours before I isolated this... - content=ContentFile(b"Hello world!", "dummy.name"), - file_type=File.FileType.PROJECT_FILE, - uploaded_by=user, - ) + FileVersion.objects.add_version( + project=cls.p1, + filename="file.name", + # NOTE the dummy name is required when running tests on GitHub CI, but not locally. Spent few hours before I isolated this... + content=ContentFile(b"Hello world!", "dummy.name"), + file_type=File.FileType.PROJECT_FILE, + uploaded_by=user, + ) def test_extracts3data_output_to_file(self): output_file = "extracted.csv" diff --git a/docker-app/qfieldcloud/core/tests/test_deleteorphanedfiles.py b/docker-app/qfieldcloud/core/tests/test_deleteorphanedfiles.py deleted file mode 100644 index d79bcec32..000000000 --- a/docker-app/qfieldcloud/core/tests/test_deleteorphanedfiles.py +++ /dev/null @@ -1,222 +0,0 @@ -""" -Todo: - * Delete with QF-4963 Drop support for legacy storage -""" - -import io - -from django.conf import settings -from django.core.management import call_command -from django.test import TestCase - -from qfieldcloud.core.models import Person, Project -from qfieldcloud.core.utils import get_project_files_count, get_s3_bucket -from qfieldcloud.core.utils2 import storage - -from .utils import set_subscription, setup_subscription_plans - - -class QfcTestCase(TestCase): - def setUp(self): - setup_subscription_plans() - - self.u1 = Person.objects.create(username="u1") - set_subscription(self.u1, "default_user") - self.projects: list[Project] = [] - - get_s3_bucket().objects.filter(Prefix="projects/").delete() - - self.generate_projects(2) - - def generate_projects(self, count: int): - offset = len(self.projects) - for i in range(1, count + 1): - p = Project.objects.create( - name=f"p{offset + i}", - owner=self.u1, - file_storage=settings.LEGACY_STORAGE_NAME, - ) - self.projects.append(p) - file = io.BytesIO(b"Hello world!") - storage.upload_project_file(p, file, "project.qgs") - - def call_command(self, *args, **kwargs): - out = io.StringIO() - call_command( - "deleteorphanedfiles", - *args, - stdout=out, - stderr=out, - **kwargs, - ) - return out.getvalue() - - def test_nothing_to_delete(self): - project_ids = sorted([str(p.id) for p in self.projects]) - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(self.projects[1].project_files_count, 1) - - out = self.call_command() - - self.assertEqual( - out.strip(), - "\n".join( - [ - "Checking the last 2 project id(s) from the storage...", - "No project files to delete.", - ] - ), - ) - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(self.projects[1].project_files_count, 1) - - def test_dry_run(self): - project_ids = sorted([str(p.id) for p in self.projects]) - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(self.projects[1].project_files_count, 1) - - out = self.call_command(dry_run=True) - - self.assertEqual( - out.strip(), - "\n".join( - [ - "Dry run, no files will be deleted.", - "Checking the last 2 project id(s) from the storage...", - "No project files to delete.", - ] - ), - ) - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(self.projects[1].project_files_count, 1) - - def test_delete_files(self): - project_ids = sorted([str(p.id) for p in self.projects]) - Project.objects.filter(id__in=project_ids).delete() - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(get_project_files_count(project_ids[1]), 1) - - out = self.call_command() - - self.assertEqual( - out.strip(), - "\n".join( - [ - "Checking the last 2 project id(s) from the storage...", - f'Deleting project files for "{project_ids[0]}"...', - f'Deleting project files for "{project_ids[1]}"...', - ] - ), - ) - - self.assertEqual(get_project_files_count(project_ids[0]), 0) - self.assertEqual(get_project_files_count(project_ids[1]), 0) - - def test_delete_files_dry_run(self): - project_ids = sorted([str(p.id) for p in self.projects]) - Project.objects.filter(id__in=project_ids).delete() - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(get_project_files_count(project_ids[1]), 1) - - out = self.call_command(dry_run=True) - - self.assertEqual( - out.strip(), - "\n".join( - [ - "Dry run, no files will be deleted.", - "Checking the last 2 project id(s) from the storage...", - f'Deleting project files for "{project_ids[0]}"...', - f'Deleting project files for "{project_ids[1]}"...', - ] - ), - ) - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(get_project_files_count(project_ids[1]), 1) - - def test_invalid_uuid(self): - project_ids = sorted([str(p.id) for p in self.projects]) - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(self.projects[1].project_files_count, 1) - - file = io.BytesIO(b"Hello world!") - storage.upload_file(file, "projects/strangename/project.qgs") - - out = self.call_command() - - self.assertEqual( - out.strip(), - "\n".join( - [ - "Invalid uuid: strangename/project.qgs", - "Checking the last 2 project id(s) from the storage...", - "No project files to delete.", - ] - ), - ) - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(self.projects[1].project_files_count, 1) - - def test_batches(self): - self.generate_projects(2) - project_ids = sorted([str(p.id) for p in self.projects]) - Project.objects.filter(id__in=project_ids[:2]).delete() - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(get_project_files_count(project_ids[1]), 1) - self.assertEqual(get_project_files_count(project_ids[2]), 1) - self.assertEqual(get_project_files_count(project_ids[3]), 1) - - out = self.call_command(limit=2) - - self.assertEqual( - out.strip(), - "\n".join( - [ - "Checking a batch of 2 project ids from the storage...", - "Checking a batch of 2 project ids from the storage...", - f'Deleting project files for "{project_ids[0]}"...', - f'Deleting project files for "{project_ids[1]}"...', - ] - ), - ) - - self.assertEqual(get_project_files_count(project_ids[0]), 0) - self.assertEqual(get_project_files_count(project_ids[1]), 0) - self.assertEqual(get_project_files_count(project_ids[2]), 1) - self.assertEqual(get_project_files_count(project_ids[3]), 1) - - def test_deletes_extra_files_on_second_level(self): - file = io.BytesIO(b"Hello world!") - storage.upload_project_file(self.projects[0], file, "inner/path/data.txt") - - project_ids = sorted([str(p.id) for p in self.projects]) - Project.objects.filter(id__in=project_ids).delete() - - self.assertEqual(get_project_files_count(str(self.projects[0].id)), 2) - self.assertEqual(get_project_files_count(str(self.projects[1].id)), 1) - - out = self.call_command() - - self.assertEqual( - out.strip(), - "\n".join( - [ - "Checking the last 2 project id(s) from the storage...", - f'Deleting project files for "{project_ids[0]}"...', - f'Deleting project files for "{project_ids[1]}"...', - ] - ), - ) - - self.assertEqual(get_project_files_count(project_ids[0]), 0) - self.assertEqual(get_project_files_count(project_ids[1]), 0) diff --git a/docker-app/qfieldcloud/core/tests/test_packages.py b/docker-app/qfieldcloud/core/tests/test_packages.py index 6223b021a..028d0cd94 100644 --- a/docker-app/qfieldcloud/core/tests/test_packages.py +++ b/docker-app/qfieldcloud/core/tests/test_packages.py @@ -32,7 +32,6 @@ wait_for_project_ok_status, ) from qfieldcloud.core.utils2.jobs import repackage -from qfieldcloud.core.utils2.storage import get_stored_package_ids from qfieldcloud.filestorage.models import File logging.disable(logging.CRITICAL) @@ -639,11 +638,6 @@ def test_outdated_packaged_files_are_deleted(self): ) self.conn.commit() - if not self.project1.uses_legacy_storage: - self.assertEqual( - File.objects.filter(file_type=File.FileType.PACKAGE_FILE).count(), 0 - ) - self.upload_files_and_check_package( token=self.token1.key, project=self.project1, @@ -662,18 +656,10 @@ def test_outdated_packaged_files_are_deleted(self): "created_at" ) - # TODO Delete with QF-4963 Drop support for legacy storage - if self.project1.uses_legacy_storage: - stored_package_ids = get_stored_package_ids(self.project1) - self.assertIn(str(old_package.id), stored_package_ids) - self.assertEqual(len(stored_package_ids), 1) - else: - self.assertGreaterEqual( - File.objects.filter(package_job=old_package).count(), 1 - ) - self.assertGreaterEqual( - File.objects.filter(file_type=File.FileType.PACKAGE_FILE).count(), 1 - ) + self.assertGreaterEqual(File.objects.filter(package_job=old_package).count(), 1) + self.assertGreaterEqual( + File.objects.filter(file_type=File.FileType.PACKAGE_FILE).count(), 1 + ) self.check_package( self.token1.key, @@ -689,22 +675,11 @@ def test_outdated_packaged_files_are_deleted(self): "created_at" ) - # TODO Delete with QF-4963 Drop support for legacy storage - if self.project1.uses_legacy_storage: - stored_package_ids = get_stored_package_ids(self.project1) - - self.assertNotEqual(old_package.id, new_package.id) - self.assertNotIn(str(old_package.id), stored_package_ids) - self.assertIn(str(new_package.id), stored_package_ids) - self.assertEqual(len(stored_package_ids), 1) - else: - self.assertEqual(File.objects.filter(package_job=old_package).count(), 0) - self.assertGreaterEqual( - File.objects.filter(package_job=new_package).count(), 1 - ) - self.assertGreaterEqual( - File.objects.filter(file_type=File.FileType.PACKAGE_FILE).count(), 1 - ) + self.assertEqual(File.objects.filter(package_job=old_package).count(), 0) + self.assertGreaterEqual(File.objects.filter(package_job=new_package).count(), 1) + self.assertGreaterEqual( + File.objects.filter(file_type=File.FileType.PACKAGE_FILE).count(), 1 + ) def test_package_and_project_file_attachments(self): # upload attachments to the project @@ -764,64 +739,45 @@ def test_purge_obsolete_package_files_works_fine(self): wait_for_project_ok_status(self.project1) self.project1.refresh_from_db() - if self.project1.uses_legacy_storage: - # TODO Delete with QF-4963 Drop support for legacy storage - stored_package_ids = get_stored_package_ids(self.project1) - - self.assertIn(str(package_job_1.id), stored_package_ids) - self.assertIn(str(other_user_package_job.id), stored_package_ids) - self.assertEquals(len(stored_package_ids), 2) - - else: - package_files_p1_qs = File.objects.filter( - project=self.project1, - file_type=File.FileType.PACKAGE_FILE, - package_job=package_job_1, - ) + package_files_p1_qs = File.objects.filter( + project=self.project1, + file_type=File.FileType.PACKAGE_FILE, + package_job=package_job_1, + ) - self.assertEquals(package_files_p1_qs.count(), 3) + self.assertEquals(package_files_p1_qs.count(), 3) # repackage the project for the same user. package_job_2 = repackage(self.project1, self.user1) wait_for_project_ok_status(self.project1) self.project1.refresh_from_db() - if self.project1.uses_legacy_storage: - # TODO Delete with QF-4963 Drop support for legacy storage - stored_package_ids = get_stored_package_ids(self.project1) - - self.assertNotIn(str(package_job_1.id), stored_package_ids) - self.assertIn(str(package_job_2.id), stored_package_ids) - self.assertIn(str(other_user_package_job.id), stored_package_ids) - self.assertEquals(len(stored_package_ids), 2) - - else: - # make sure old package files are deleted. - package_files_p1_qs = File.objects.filter( - project=self.project1, - file_type=File.FileType.PACKAGE_FILE, - package_job=package_job_1, - ) + # make sure old package files are deleted. + package_files_p1_qs = File.objects.filter( + project=self.project1, + file_type=File.FileType.PACKAGE_FILE, + package_job=package_job_1, + ) - self.assertEquals(package_files_p1_qs.count(), 0) + self.assertEquals(package_files_p1_qs.count(), 0) - # make sure new package files are there. - package_files_p2_qs = File.objects.filter( - project=self.project1, - file_type=File.FileType.PACKAGE_FILE, - package_job=package_job_2, - ) + # make sure new package files are there. + package_files_p2_qs = File.objects.filter( + project=self.project1, + file_type=File.FileType.PACKAGE_FILE, + package_job=package_job_2, + ) - self.assertEquals(package_files_p2_qs.count(), 3) + self.assertEquals(package_files_p2_qs.count(), 3) - # make sure the other user's package files are there. - other_user_package_files_qs = File.objects.filter( - project=self.project1, - file_type=File.FileType.PACKAGE_FILE, - package_job=other_user_package_job, - ) + # make sure the other user's package files are there. + other_user_package_files_qs = File.objects.filter( + project=self.project1, + file_type=File.FileType.PACKAGE_FILE, + package_job=other_user_package_job, + ) - self.assertEquals(other_user_package_files_qs.count(), 3) + self.assertEquals(other_user_package_files_qs.count(), 3) def test_needs_repackaging(self): # 0. Create two users, where one owns the project and the other is a project collaborator. diff --git a/docker-app/qfieldcloud/core/tests/test_qgis_file.py b/docker-app/qfieldcloud/core/tests/test_qgis_file.py index b03567adf..3ae2a9fde 100644 --- a/docker-app/qfieldcloud/core/tests/test_qgis_file.py +++ b/docker-app/qfieldcloud/core/tests/test_qgis_file.py @@ -304,12 +304,6 @@ def test_upload_and_list_file_checksum(self): self.assertEqual(json[0]["name"], "file.txt") self.assertEqual(json[0]["size"], 13) - # The `sha256` key is optional only for the legacy storage, there is no performance penalty for the non-legacy storage if we send it back, - # therefore `skip_metadata` is ignored in non-legacy storage - # TODO Delete with QF-4963 Drop support for legacy storage - if self.project1.uses_legacy_storage: - self.assertNotIn("sha256", json[0]) - self.assertIn("md5sum", json[0]) self.assertEqual( json[0]["md5sum"], @@ -703,28 +697,18 @@ def count_versions(): """counts the versions in first file of project1""" project = Project.objects.get(pk=self.project1.pk) - # TODO Delete with QF-4963 Drop support for legacy storage - if project.uses_legacy_storage: - return len(project.legacy_get_file("file.txt").versions) - else: - return project.get_file("file.txt").versions.count() + return project.get_file("file.txt").versions.count() def read_version(n): """returns the content of version in first file of project1""" project = Project.objects.get(pk=self.project1.pk) - # TODO Delete with QF-4963 Drop support for legacy storage - if project.uses_legacy_storage: - file = ( - project.legacy_get_file("file.txt").versions[n]._data.get()["Body"] - ) - else: - file = ( - project.get_file("file.txt") - .versions.all() - .order_by("uploaded_at")[n] - .content - ) + file = ( + project.get_file("file.txt") + .versions.all() + .order_by("uploaded_at")[n] + .content + ) return file.read().decode() diff --git a/docker-app/qfieldcloud/core/urls.py b/docker-app/qfieldcloud/core/urls.py index fd5bc4cf8..b3860113f 100644 --- a/docker-app/qfieldcloud/core/urls.py +++ b/docker-app/qfieldcloud/core/urls.py @@ -61,15 +61,15 @@ ), path( "packages//latest/", - package_views.compatibility_latest_package_view, + package_views.LatestPackageView.as_view(), ), path( "packages//latest/files//", - package_views.compatibility_package_download_files_view, + package_views.LatestPackageDownloadFilesView.as_view(), ), path( "packages///files//", - package_views.compatibility_package_upload_files_view, + package_views.PackageUploadFilesView.as_view(), ), path("members//", members_views.ListCreateMembersView.as_view()), path( diff --git a/docker-app/qfieldcloud/core/utils.py b/docker-app/qfieldcloud/core/utils.py index 8098401d8..ae00523de 100644 --- a/docker-app/qfieldcloud/core/utils.py +++ b/docker-app/qfieldcloud/core/utils.py @@ -169,18 +169,6 @@ def get_s3_client() -> mypy_boto3_s3.Client: return s3_client -def get_sha256(file: IO) -> str: - """Return the sha256 hash of the file - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - if type(file) is InMemoryUploadedFile or type(file) is TemporaryUploadedFile: - return _get_sha256_memory_file(file) - else: - return _get_sha256_file(file) - - def _get_sha256_memory_file(file: InMemoryUploadedFile | TemporaryUploadedFile) -> str: BLOCKSIZE = 65536 hasher = hashlib.sha256() @@ -206,18 +194,6 @@ def _get_sha256_file(file: IO) -> str: return hasher.hexdigest() -def get_md5sum(file: IO) -> str: - """Return the md5sum hash of the file - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - if type(file) is InMemoryUploadedFile or type(file) is TemporaryUploadedFile: - return _get_md5sum_memory_file(file) - else: - return _get_md5sum_file(file) - - def _get_md5sum_memory_file(file: InMemoryUploadedFile | TemporaryUploadedFile) -> str: BLOCKSIZE = 65536 hasher = hashlib.md5() @@ -329,29 +305,6 @@ def get_qgis_project_file(project_id: str) -> str | None: return None -def check_legacy_s3_file_exists(key: str, should_raise: bool = True) -> bool: - """Check to see if an object exists on S3. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - client = get_s3_client() - try: - client.head_object( - Bucket=get_legacy_s3_credentials()["OPTIONS"]["bucket_name"], - Key=key, - ) - return True - except ClientError as e: - if e.response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404: - return False - else: - if should_raise: - raise e - else: - return False - - def check_s3_key(key: str) -> str | None: """Check to see if an object exists on S3. It it exists, the function returns the sha256 of the file from the metadata @@ -418,55 +371,6 @@ def get_project_files(project_id: str, path: str = "") -> list[S3Object]: return list_files(bucket, prefix, root_prefix) -def get_project_files_with_versions( - project_id: str, -) -> Generator[S3ObjectWithVersions, None, None]: - """Returns a generator of files and their versions. - - Args: - project_id: the project id - - Returns: - the list of files - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = get_s3_bucket() - prefix = f"projects/{project_id}/files/" - - return list_files_with_versions(bucket, prefix, prefix) - - -def get_project_file_with_versions( - project_id: str, filename: str -) -> S3ObjectWithVersions | None: - """Returns a list of files and their versions. - - Args: - project_id: the project id - - Returns: - the list of files - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = get_s3_bucket() - root_prefix = f"projects/{project_id}/files/" - prefix = f"projects/{project_id}/files/{filename}" - files = [ - f - for f in list_files_with_versions(bucket, prefix, root_prefix) - if f.latest.key == prefix - ] - - if files: - return files[0] - else: - return None - - def get_project_package_files(project_id: str, package_id: str) -> list[S3Object]: """Returns a list of package files. @@ -485,32 +389,6 @@ def get_project_package_files(project_id: str, package_id: str) -> list[S3Object return list_files(bucket, prefix, prefix) -def get_project_files_count(project_id: str) -> int: - """Returns the number of files within a project. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = get_s3_bucket() - prefix = f"projects/{project_id}/files/" - files = list(bucket.objects.filter(Prefix=prefix)) - - return len(files) - - -def get_project_package_files_count(project_id: str) -> int: - """Returns the number of package files within a project. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = get_s3_bucket() - prefix = f"projects/{project_id}/export/" - files = list(bucket.objects.filter(Prefix=prefix)) - - return len(files) - - def list_files( bucket: mypy_boto3_s3.service_resource.Bucket, prefix: str, diff --git a/docker-app/qfieldcloud/core/utils2/packages.py b/docker-app/qfieldcloud/core/utils2/packages.py index 0d942ec0f..19947392f 100644 --- a/docker-app/qfieldcloud/core/utils2/packages.py +++ b/docker-app/qfieldcloud/core/utils2/packages.py @@ -1,9 +1,7 @@ import logging -import uuid from typing import Iterable from qfieldcloud.core import models -from qfieldcloud.core.utils2 import storage from qfieldcloud.filestorage.models import File logger = logging.getLogger(__name__) @@ -29,59 +27,31 @@ def delete_obsolete_packages(projects: Iterable[models.Project]) -> None: ) for project in projects: - # TODO Delete with QF-4963 Drop support for legacy storage - if project.uses_legacy_storage: - stored_package_ids = list( - map(uuid.UUID, storage.get_stored_package_ids(project)) - ) + latest_project_package_jobs_ids = project.latest_package_jobs().values_list( + "id", flat=True + ) - latest_package_job_ids = project.latest_package_jobs().values_list( - "id", flat=True + files_to_delete_qs = ( + File.objects.filter( + project=project, + file_type=File.FileType.PACKAGE_FILE, ) - - for stored_package_id in stored_package_ids: - # the job is still active, so it might be one of the new packages - if stored_package_id in active_package_job_ids: - continue - - # keep packages that are used by other users due to user-assigned secrets. - shall_skip_package = False - for latest_package_job_id in latest_package_job_ids: - if stored_package_id == latest_package_job_id: - shall_skip_package = True - break - - if shall_skip_package: - continue - - storage.delete_stored_package(project, str(stored_package_id)) - - else: - latest_project_package_jobs_ids = project.latest_package_jobs().values_list( - "id", flat=True + .exclude( + package_job_id__in=active_package_job_ids, ) - - files_to_delete_qs = ( - File.objects.filter( - project=project, - file_type=File.FileType.PACKAGE_FILE, - ) - .exclude( - package_job_id__in=active_package_job_ids, - ) - .exclude( - package_job_id__in=latest_project_package_jobs_ids, - ) + .exclude( + package_job_id__in=latest_project_package_jobs_ids, ) + ) - logger.info( - "Deleting {} package files from previous and obsolete packages for project id {}:\n{}".format( - files_to_delete_qs.count(), - project.id, - "\n".join(files_to_delete_qs.values_list("name", flat=True)), - ) + logger.info( + "Deleting {} package files from previous and obsolete packages for project id {}:\n{}".format( + files_to_delete_qs.count(), + project.id, + "\n".join(files_to_delete_qs.values_list("name", flat=True)), ) + ) - delete_count = files_to_delete_qs.delete() + delete_count = files_to_delete_qs.delete() - logger.info(f"Deleted {delete_count} package files.") + logger.info(f"Deleted {delete_count} package files.") diff --git a/docker-app/qfieldcloud/core/utils2/storage.py b/docker-app/qfieldcloud/core/utils2/storage.py index afb3fdbaa..c5d3ec0cb 100644 --- a/docker-app/qfieldcloud/core/utils2/storage.py +++ b/docker-app/qfieldcloud/core/utils2/storage.py @@ -1,216 +1,17 @@ -"""A module with legacy file storage management. - -Todo: - * Delete with QF-4963 Drop support for legacy storage -""" - -from __future__ import annotations - import hashlib import logging import re -from enum import Enum -from pathlib import PurePath -from typing import IO -from django.conf import settings from django.core.files.base import ContentFile from django.core.files.storage import storages -from django.db import transaction -from django.http import FileResponse, HttpRequest -from django.http.response import HttpResponse, HttpResponseBase -from django.utils import timezone -from mypy_boto3_s3.type_defs import ObjectIdentifierTypeDef import qfieldcloud.core.models import qfieldcloud.core.utils -from qfieldcloud.core.utils2.audit import LogEntry, audit from qfieldcloud.filestorage.backend import QfcS3Boto3Storage -from qfieldcloud.filestorage.utils import is_admin_restricted_file logger = logging.getLogger(__name__) -def legacy_only(func): - """ - Decorator to verify that given project is stored on the legacy storage. - Otherwise, it calls the decorated function. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - * Delete all decorated functions with QF-4963 Drop support for legacy storage - """ - - def wrapper(project, *args, **kwargs): - if not project.uses_legacy_storage: - raise NotImplementedError( - f"This function is not implemented for '{project.file_storage}' file storage!" - ) - - return func(project, *args, **kwargs) - - return wrapper - - -def _delete_by_prefix_versioned(prefix: str): - """ - Delete all objects and their versions starting with a given prefix. - - Similar concept to delete a directory. - Do not use when deleting objects with precise key, as it will delete all objects that start with the same name. - Deleting with this method will leave a deleted version and the deletion is not permanent. - In other words, it is a soft delete. Read more here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/DeletingObjectVersions.html - - Args: - prefix: Object's prefix to search and delete. Check the given prefix if it matches the expected format before using this function! - - Raises: - RuntimeError: When the given prefix is not a string, empty string or leading slash. Check is very basic, do a throrogh checks before calling! - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - logging.info(f"S3 object deletion (versioned) with {prefix=}") - - # Illegal prefix is either empty string ("") or slash ("/"), it will delete random 1000 objects. - if not isinstance(prefix, str) or prefix == "" or prefix == "/": - raise RuntimeError(f"Attempt to delete S3 object with illegal {prefix=}") - - bucket = qfieldcloud.core.utils.get_s3_bucket() - return bucket.objects.filter(Prefix=prefix).delete() - - -def _delete_by_prefix_permanently(prefix: str): - """ - Delete all objects and their versions starting with a given prefix. - - Similar concept to delete a directory. - Do not use when deleting objects with precise key, as it will delete all objects that start with the same name. - Deleting with this method will permanently delete objects and all their versions and the deletion is impossible to recover. - In other words, it is a hard delete. Read more here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/DeletingObjectVersions.html - - Args: - prefix: Object's prefix to search and delete. Check the given prefix if it matches the expected format before using this function! - - Raises: - RuntimeError: When the given prefix is not a string, empty string or leading slash. Check is very basic, do a throrogh checks before calling! - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - logging.info(f"S3 object deletion (permanent) with {prefix=}") - - # Illegal prefix is either empty string ("") or slash ("/"), it will delete random 1000 object versions. - if not isinstance(prefix, str) or prefix == "" or prefix == "/": - raise RuntimeError(f"Attempt to delete S3 object with illegal {prefix=}") - - bucket = qfieldcloud.core.utils.get_s3_bucket() - return bucket.object_versions.filter(Prefix=prefix).delete() - - -def _delete_by_key_versioned(key: str): - """ - Delete an object with a given key. - - Deleting with this method will leave a deleted version and the deletion is not permanent. - In other words, it is a soft delete. - - Args: - key: Object's key to search and delete. Check the given key if it matches the expected format before using this function! - - Raises: - RuntimeError: When the given key is not a string, empty string or leading slash. Check is very basic, do a throrogh checks before calling! - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - logging.info(f"Delete (versioned) S3 object with {key=}") - - # prevent disastrous results when prefix is either empty string ("") or slash ("/"). - if not isinstance(key, str) or key == "" or key == "/": - raise RuntimeError( - f"Attempt to delete (versioned) S3 object with illegal {key=}" - ) - - bucket = qfieldcloud.core.utils.get_s3_bucket() - - return bucket.delete_objects( - Delete={ - "Objects": [ - { - "Key": key, - } - ], - }, - ) - - -def _delete_by_key_permanently(key: str): - """ - Delete an object with a given key. - - Deleting with this method will permanently delete objects and all their versions and the deletion is impossible to recover. - In other words, it is a hard delete. - - Args: - key: Object's key to search and delete. Check the given key if it matches the expected format before using this function! - - Raises: - RuntimeError: When the given key is not a string, empty string or leading slash. Check is very basic, do a throrogh checks before calling! - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - logging.info(f"Delete (permanently) S3 object with {key=}") - - # prevent disastrous results when prefix is either empty string ("") or slash ("/"). - if not isinstance(key, str) or key == "" or key == "/": - raise RuntimeError( - f"Attempt to delete (permanently) S3 object with illegal {key=}" - ) - - bucket = qfieldcloud.core.utils.get_s3_bucket() - - # NOTE filer by prefix will return all objects with that prefix. E.g. for given key="orho.tif", it will return "ortho.tif", "ortho.tif.aux.xml" and "ortho.tif.backup" - temp_objects = bucket.object_versions.filter( - Prefix=key, - ) - object_to_delete: list[ObjectIdentifierTypeDef] = [] - for temp_object in temp_objects: - # filter out objects that do not have the same key as the requested deletion key. - if temp_object.key != key: - continue - - object_to_delete.append( - { - "Key": key, - "VersionId": temp_object.id, - } - ) - - if len(object_to_delete) == 0: - logging.warning( - f"Attempt to delete (permanently) S3 objects did not match any existing objects for {key=}", - extra={ - "all_objects": [ - (o.key, o.version_id, o.e_tag, o.last_modified, o.is_latest) - for o in temp_objects - ] - }, - ) - return None - - logging.info( - f"Delete (permanently) S3 object with {key=} will delete delete {len(object_to_delete)} version(s)" - ) - - return bucket.delete_objects( - Delete={ - "Objects": object_to_delete, - }, - ) - - def delete_version_permanently(version_obj: qfieldcloud.core.utils.S3ObjectVersion): """ Todo: @@ -242,209 +43,6 @@ def get_attachment_dir_prefix( return "" -def file_response( - request: HttpRequest, - key: str, - expires: int = 60, - version: str | None = None, - as_attachment: bool = False, -) -> HttpResponseBase: - """Return a Django HTTP response with nginx speedup if reverse proxy detected. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - url = "" - filename = PurePath(key).name - extra_params = {} - - if version is not None: - extra_params["VersionId"] = version - - # Assume that if the request is secure, we are behind a nginx proxy and we can use `X-Accel-Redirect` - if request.is_secure() and not settings.IN_TEST_SUITE: - if as_attachment: - extra_params["ResponseContentType"] = "application/force-download" - extra_params["ResponseContentDisposition"] = ( - f'attachment;filename="{filename}"' - ) - - url = qfieldcloud.core.utils.get_s3_client().generate_presigned_url( - "get_object", - Params={ - **extra_params, - "Key": key, - "Bucket": qfieldcloud.core.utils.get_s3_bucket().name, - }, - ExpiresIn=expires, - HttpMethod="GET", - ) - - # Let's NGINX handle the redirect to the storage and streaming the file contents back to the client - response = HttpResponse() - response["X-Accel-Redirect"] = "/storage-download/" - response["redirect_uri"] = url - - return response - elif settings.DEBUG or settings.IN_TEST_SUITE: - return_file = ContentFile(b"") - qfieldcloud.core.utils.get_s3_bucket().download_fileobj( - key, - return_file, - extra_params, - ) - - return FileResponse( - return_file.open(), - as_attachment=as_attachment, - filename=filename, - ) - - raise Exception( - "Expected to either run behind nginx proxy, debug mode or within a test suite." - ) - - -class ImageMimeTypes(str, Enum): - svg = "image/svg+xml" - png = "image/png" - jpg = "image/jpeg" - - @classmethod - def or_none(cls, string: str) -> ImageMimeTypes | None: - try: - return cls(string) - except ValueError: - return None - - -def upload_user_avatar( - user: qfieldcloud.core.models.User, file: IO, mimetype: ImageMimeTypes -) -> str: # noqa: F821 - """Uploads a picture as a user avatar. - - NOTE this function does NOT modify the `UserAccount.legacy_avatar_uri` field - - Args: - user: - file: file used as avatar - mimetype: file mimetype - - Returns: - URI to the avatar - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = qfieldcloud.core.utils.get_s3_bucket() - key = f"users/{user.username}/avatar.{mimetype.name}" - bucket.upload_fileobj( - file, - key, - { - "ContentType": mimetype.value, - }, - ) - return key - - -def delete_user_avatar(user: qfieldcloud.core.models.User) -> None: # noqa: F821 - """Deletes the user's avatar file. - - NOTE this function does NOT modify the `UserAccount.legacy_avatar_uri` field - - Args: - user: - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - key = user.useraccount.legacy_avatar_uri - - # it well could be the user has no avatar yet - if not key: - return - - # e.g. "users/suricactus/avatar.svg" - if not key or not re.match(r"^users/[\w-]+/avatar\.(png|jpg|svg)$", key): - raise RuntimeError(f"Suspicious S3 deletion of user avatar {key=}") - - _delete_by_key_permanently(key) - - -@legacy_only -def upload_project_thumbail( - project: qfieldcloud.core.models.Project, - file: IO, - mimetype: str, - filename: str, # noqa: F821 -) -> str: - """Uploads a picture as a project thumbnail. - - NOTE this function does NOT modify the `Project.thumbnail_uri` field - - Args: - project: - file: file used as thumbail - mimetype: file mimetype - filename: filename - - Returns: - URI to the thumbnail - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = qfieldcloud.core.utils.get_s3_bucket() - - # for now we always expect PNGs - if mimetype == "image/svg+xml": - extension = "svg" - elif mimetype == "image/png": - extension = "png" - elif mimetype == "image/jpeg": - extension = "jpg" - else: - raise Exception(f"Unknown mimetype: {mimetype}") - - key = f"projects/{project.id}/meta/{filename}.{extension}" - bucket.upload_fileobj( - file, - key, - { - "ContentType": mimetype, - }, - ) - return key - - -@legacy_only -def delete_project_thumbnail( - project: qfieldcloud.core.models.Project, -) -> None: # noqa: F821 - """Delete a picture as a project thumbnail. - - NOTE this function does NOT modify the `Project.thumbnail_uri` field - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - key = project.legacy_thumbnail_uri - - # it well could be the project has no thumbnail yet - if not key: - return - - if not key or not re.match( - # e.g. "projects/9bf34e75-0a5d-47c3-a2f0-ebb7126eeccc/meta/thumbnail.png" - r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/meta/thumbnail\.(png|jpg|svg)$", - key, - ): - raise RuntimeError(f"Suspicious S3 deletion of project thumbnail image {key=}") - - _delete_by_key_permanently(key) - - def purge_previous_thumbnails_versions( project: qfieldcloud.core.models.Project, ) -> None: @@ -502,335 +100,6 @@ def purge_previous_thumbnails_versions( # TODO: audit ? take implementation from files_views.py:211 -@legacy_only -def purge_old_file_versions_legacy( - project: qfieldcloud.core.models.Project, -) -> None: # noqa: F821 - """ - Deletes old versions of all files in the given project. Will keep __3__ - versions for COMMUNITY user accounts, and __10__ versions for PRO user - accounts - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - - keep_count = project.owner_aware_storage_keep_versions - - logger.info(f"Cleaning up old files for {project} to {keep_count} versions") - - # Process file by file - for file in qfieldcloud.core.utils.get_project_files_with_versions(project.pk): - # Skip the newest N - old_versions_to_purge = sorted( - file.versions, key=lambda v: v.last_modified, reverse=True - )[keep_count:] - - # Debug print - logger.info( - f'Purging {len(old_versions_to_purge)} out of {len(file.versions)} old versions for "{file.latest.name}"...' - ) - - # Remove the N oldest - for old_version in old_versions_to_purge: - logger.info( - f'Purging {old_version.key=} {old_version.id=} as old version for "{file.latest.name}"...' - ) - - if old_version.is_latest: - # This is not supposed to happen, as versions were sorted above, - # but leaving it here as a security measure in case version - # ordering changes for some reason. - raise Exception("Trying to delete latest version") - - if not old_version.key or not re.match( - r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/.+$", old_version.key - ): - raise RuntimeError( - f"Suspicious S3 file version deletion {old_version.key=} {old_version.id=}" - ) - # TODO: any way to batch those ? will probaby get slow on production - delete_version_permanently(old_version) - # TODO: audit ? take implementation from files_views.py:211 - - # Update the project size - project.save(recompute_storage=True) - - -def upload_file(file: IO, key: str): - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = qfieldcloud.core.utils.get_s3_bucket() - bucket.upload_fileobj( - file, - key, - ) - return key - - -@legacy_only -def upload_project_file( - project: qfieldcloud.core.models.Project, file: IO, filename: str -) -> str: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - key = f"projects/{project.id}/files/{filename}" - bucket = qfieldcloud.core.utils.get_s3_bucket() - bucket.upload_fileobj( - file, - key, - ) - return key - - -def delete_all_project_files_permanently(project_id: str) -> None: - """Deletes all project files permanently. - - Args: - project_id: the project which files shall be deleted. Note that the `project_id` might be a of a already deleted project which files are still dangling around. - - Raises: - RuntimeError: if the produced Object Storage key to delete is not in the right format - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - prefix = f"projects/{project_id}/" - - if not re.match(r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/$", prefix): - raise RuntimeError( - f"Suspicious S3 deletion of all project files with {prefix=}" - ) - - _delete_by_prefix_permanently(prefix) - - -@legacy_only -def delete_project_file_permanently( - project: qfieldcloud.core.models.Project, filename: str -): # noqa: F821 - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - logger.info(f"Requested delete (permanent) of project file {filename=}") - - file = qfieldcloud.core.utils.get_project_file_with_versions( - str(project.id), filename - ) - - if not file: - raise Exception( - f"No file with such name in the given project found {filename=}" - ) - - if not re.match(r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/.+$", file.latest.key): - raise RuntimeError(f"Suspicious S3 file deletion {file.latest.key=}") - - # NOTE the file operations depend on HTTP calls to the S3 storage and they might fail, - # we need to choose source of truth between DB and S3. - # For now the source of truth is what is on the S3 storage, - # because we do most of our file operations directly by calling the S3 API. - # 1. S3 storage modification. If it fails, it will cancel the transaction - # and do not update anything in the database. - # We assume S3 storage is transactional, while it might not be true. - # 2. DB modification. If it fails, the states betewen DB and S3 mismatch, - # but can be easyly synced from the S3 to DB with a manual script. - with transaction.atomic(): - _delete_by_key_permanently(file.latest.key) - - update_fields = ["data_last_updated_at"] - - now = timezone.now() - project.data_last_updated_at = now - - if is_admin_restricted_file(filename, project.the_qgis_file_name): - update_fields.append("restricted_data_last_updated_at") - project.restricted_data_last_updated_at = now - - if qfieldcloud.core.utils.is_the_qgis_file(filename): - update_fields.append("the_qgis_file_name") - project.the_qgis_file_name = None - - project.save(update_fields=update_fields, recompute_storage=True) - - # NOTE force audits to be required when deleting files - audit( - project, - LogEntry.Action.DELETE, - changes={f"{filename} ALL": [file.latest.e_tag, None]}, - ) - - -@legacy_only -def delete_project_file_version_permanently( - project: qfieldcloud.core.models.Project, - filename: str, - version_id: str, - include_older: bool = False, -) -> list[qfieldcloud.core.utils.S3ObjectVersion]: - """Deletes a specific version of given file. - - Args: - project: project the file belongs to - filename: filename the version belongs to - version_id: version id to delete - include_older: when True, versions older than the passed `version` will also be deleted. If the version_id is the latest version of a file, this parameter will treated as False. Defaults to False. - - Returns: - the number of versions deleted - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - project_id = str(project.id) - file = qfieldcloud.core.utils.get_project_file_with_versions(project_id, filename) - - if not file: - raise Exception( - f"No file with such name in the given project found {filename=} {version_id=}" - ) - - if file.latest.id == version_id: - include_older = False - - if len(file.versions) == 1: - raise RuntimeError( - "Forbidded attempt to delete a specific file version which is the only file version available." - ) - - versions_latest_first = list(reversed(file.versions)) - versions_to_delete: list[qfieldcloud.core.utils.S3ObjectVersion] = [] - - for file_version in versions_latest_first: - if file_version.id == version_id: - versions_to_delete.append(file_version) - - if include_older: - continue - else: - break - - if versions_to_delete: - assert include_older, ( - "We should continue to loop only if `include_older` is True" - ) - assert versions_to_delete[-1].last_modified > file_version.last_modified, ( - "Assert the other versions are really older than the requested one" - ) - - versions_to_delete.append(file_version) - - with transaction.atomic(): - update_fields = ["data_last_updated_at"] - for file_version in versions_to_delete: - if ( - not re.match( - r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/.+$", - file_version._data.key, - ) - or not file_version.id - ): - raise RuntimeError( - f"Suspicious S3 file version deletion {filename=} {version_id=} {include_older=}" - ) - - audit_suffix = file_version.display - - audit( - project, - LogEntry.Action.DELETE, - changes={f"{filename} {audit_suffix}": [file_version.e_tag, None]}, - ) - - delete_version_permanently(file_version) - - now = timezone.now() - project.data_last_updated_at = now - - if is_admin_restricted_file(filename, project.the_qgis_file_name): - update_fields.append("restricted_data_last_updated_at") - project.restricted_data_last_updated_at = now - - project.save(update_fields=update_fields, recompute_storage=True) - - return versions_to_delete - - -@legacy_only -def get_stored_package_ids(project: qfieldcloud.core.models.Project) -> set[str]: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - project_id = project.id - bucket = qfieldcloud.core.utils.get_s3_bucket() - prefix = f"projects/{project_id}/packages/" - root_path = PurePath(prefix) - package_ids = set() - - for file in bucket.objects.filter(Prefix=prefix): - file_path = PurePath(file.key) - parts = file_path.relative_to(root_path).parts - package_ids.add(parts[0]) - - return package_ids - - -@legacy_only -def delete_stored_package( - project: qfieldcloud.core.models.Project, package_id: str -) -> None: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - project_id = str(project.id) - prefix = f"projects/{project_id}/packages/{package_id}/" - - if not re.match( - # e.g. "projects/878039c4-b945-4356-a44e-a908fd3f2263/packages/633cd4f7-db14-4e6e-9b2b-c0ce98f9d338/" - r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/packages/[\w]{8}(-[\w]{4}){3}-[\w]{12}/$", - prefix, - ): - raise RuntimeError( - f"Suspicious S3 deletion on stored project package {project_id=} {package_id=}" - ) - - _delete_by_prefix_permanently(prefix) - - -@legacy_only -def get_project_file_storage_in_bytes(project: qfieldcloud.core.models.Project) -> int: - """Calculates the project files storage in bytes, including their versions. - - WARNING This function can be quite slow on projects with thousands of files. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - project_id = str(project.id) - bucket = qfieldcloud.core.utils.get_s3_bucket() - total_bytes = 0 - prefix = f"projects/{project_id}/files/" - - logger.info(f"Project file storage size requested for {project_id=}") - - if not re.match(r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/files/$", prefix): - raise RuntimeError( - f"Suspicious S3 calculation of all project files with {prefix=}" - ) - - for version in bucket.object_versions.filter(Prefix=prefix): - total_bytes += version.size or 0 - - return total_bytes - - def calculate_checksums( content: ContentFile, alrgorithms: tuple[str, ...], blocksize: int = 65536 ) -> tuple[bytes, ...]: diff --git a/docker-app/qfieldcloud/core/views/files_views.py b/docker-app/qfieldcloud/core/views/files_views.py deleted file mode 100644 index 9d7e64197..000000000 --- a/docker-app/qfieldcloud/core/views/files_views.py +++ /dev/null @@ -1,493 +0,0 @@ -""" -Todo: - * Delete with QF-4963 Drop support for legacy storage -""" - -import copy -import io -import logging -from pathlib import PurePath -from traceback import print_stack - -import qfieldcloud.core.utils2 as utils2 -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist -from django.db import transaction -from django.utils import timezone -from drf_spectacular.utils import ( - OpenApiParameter, - OpenApiTypes, - extend_schema, - extend_schema_view, -) -from qfieldcloud.core import exceptions, permissions_utils, utils -from qfieldcloud.core.models import Job, ProcessProjectfileJob, Project -from qfieldcloud.core.serializers import FileWithVersionsSerializer -from qfieldcloud.core.utils import S3ObjectVersion, get_project_file_with_versions -from qfieldcloud.core.utils2.audit import LogEntry, audit -from qfieldcloud.core.utils2.sentry import report_serialization_diff_to_sentry -from qfieldcloud.core.utils2.storage import ( - get_attachment_dir_prefix, - purge_old_file_versions_legacy, -) -from qfieldcloud.filestorage.utils import is_admin_restricted_file -from rest_framework import permissions, serializers, status, views -from rest_framework.exceptions import NotFound -from rest_framework.parsers import DataAndFiles, MultiPartParser -from rest_framework.request import Request -from rest_framework.response import Response - -logger = logging.getLogger(__name__) - - -class ListFilesViewPermissions(permissions.BasePermission): - def has_permission(self, request, view): - if "projectid" not in request.parser_context["kwargs"]: - return False - - projectid = request.parser_context["kwargs"]["projectid"] - project = Project.objects.get(id=projectid) - - return permissions_utils.can_read_files(request.user, project) - - -@extend_schema_view( - get=extend_schema( - description="Get all the project's file versions", - responses={200: serializers.ListSerializer(child=FileWithVersionsSerializer())}, - parameters=[ - OpenApiParameter( - name="skip_metadata", - type=OpenApiTypes.INT, - required=False, - default=0, - enum=[1, 0], - description="Skip obtaining file metadata (e.g. `sha256`). Makes responses much faster. In the future `skip_metadata=1` might be default behaviour.", - ), - ], - ), -) -class ListFilesView(views.APIView): - # TODO: docstring - - permission_classes = [permissions.IsAuthenticated, ListFilesViewPermissions] - - def get(self, request: Request, projectid: str) -> Response: - try: - project = Project.objects.get(id=projectid) - except ObjectDoesNotExist: - raise NotFound(detail=projectid) - - bucket = utils.get_s3_bucket() - prefix = f"projects/{projectid}/files/" - - files = {} - for version in bucket.object_versions.filter(Prefix=prefix): - # Created the dict entry if doesn't exist - if version.key not in files: - files[version.key] = {"versions": []} - - path = PurePath(version.key) - filename = str(path.relative_to(*path.parts[:3])) - last_modified = version.last_modified.strftime( - settings.QFIELDCLOUD_STORAGE_DT_LAST_MODIFIED_FORMAT - ) - # NOTE ETag is a MD5. But for the multipart uploaded files, the MD5 is computed from the concatenation of the MD5s of each uploaded part. - # TODO make sure when file metadata is in the DB (QF-2760), this is a real md5sum of the current file. - md5sum = version.e_tag.replace('"', "") - - version_data = { - "size": version.size, - "md5sum": md5sum, - "version_id": version.version_id, - "last_modified": last_modified, - "is_latest": version.is_latest, - "display": S3ObjectVersion(version.key, version).display, - } - - # NOTE Some clients (e.g. QFieldSync) are still requiring the `sha256` key to check whether the files needs to be reuploaded. - # Since we do not have control on these old client versions, we need to keep the API backward compatible for some time and assume `skip_metadata=0` by default. - skip_metadata_param = request.GET.get("skip_metadata", "0") - if skip_metadata_param == "0": - skip_metadata = False - else: - skip_metadata = bool(skip_metadata_param) - - if not skip_metadata: - head = version.head() - # We cannot be sure of the metadata's first letter case - # https://github.com/boto/boto3/issues/1709 - metadata = head["Metadata"] - if "sha256sum" in metadata: - sha256sum = metadata["sha256sum"] - else: - sha256sum = metadata["Sha256sum"] - - version_data["sha256"] = sha256sum - - if version.is_latest: - is_attachment = get_attachment_dir_prefix(project, filename) != "" - - files[version.key]["name"] = filename - files[version.key]["size"] = version.size - files[version.key]["md5sum"] = md5sum - files[version.key]["last_modified"] = last_modified - files[version.key]["is_attachment"] = is_attachment - - if not skip_metadata: - files[version.key]["sha256"] = sha256sum - - files[version.key]["versions"].append(version_data) - - result_list = [files[key] for key in files] - return Response(result_list) - - -@extend_schema_view( - get=extend_schema( - description="Get the project's file metadata", - responses={200: FileWithVersionsSerializer()}, - ), -) -class LegacyFileMetadataView(views.APIView): - permission_classes = [permissions.IsAuthenticated, ListFilesViewPermissions] - - def get(self, request: Request, projectid: str, filename: str) -> Response: - try: - project = Project.objects.get(id=projectid) - except ObjectDoesNotExist: - raise NotFound(detail=projectid) - - file_obj = get_project_file_with_versions(projectid, filename) - - if not file_obj: - raise NotFound(detail=filename) - - versions_data = [] - file_data: dict = {} - - for version in file_obj.versions: - last_modified = version.last_modified.strftime( - settings.QFIELDCLOUD_STORAGE_DT_LAST_MODIFIED_FORMAT - ) - md5sum = version.md5sum - - head = version._data.head() - # We cannot be sure of the metadata's first letter case - # https://github.com/boto/boto3/issues/1709 - metadata = head["Metadata"] - sha256sum = metadata.get("sha256sum") or metadata.get("Sha256sum") - - version_data = { - "size": version.size, - "md5sum": md5sum, - "version_id": version.id, - "last_modified": last_modified, - "is_latest": version.is_latest, - "display": version.display, - "sha256": sha256sum, - } - - versions_data.append(version_data) - - if version.is_latest: - is_attachment = get_attachment_dir_prefix(project, filename) != "" - - file_data["name"] = filename - file_data["size"] = version.size - file_data["md5sum"] = md5sum - file_data["last_modified"] = last_modified - file_data["is_attachment"] = is_attachment - file_data["sha256"] = sha256sum - - file_data["versions"] = versions_data - return Response(file_data) - - -class DownloadPushDeleteFileViewPermissions(permissions.BasePermission): - def has_permission(self, request, view): - if "projectid" not in request.parser_context["kwargs"]: - return False - - projectid = request.parser_context["kwargs"]["projectid"] - project = Project.objects.get(id=projectid) - user = request.user - - if request.method == "GET": - return permissions_utils.can_read_files(user, project) - elif request.method == "DELETE": - return permissions_utils.can_delete_files(user, project) - elif request.method == "POST": - return permissions_utils.can_create_files(user, project) - - return False - - -class QfcMultiPartSerializer(MultiPartParser): - errors: list[str] = [] - - # QF-2540 - def parse(self, stream, media_type=None, parser_context=None) -> DataAndFiles: - """Substitute to MultiPartParser for debugging `EmptyContentError`""" - parsed: DataAndFiles = super().parse(stream, media_type, parser_context) - - if "file" not in parsed.files or not parsed.files["file"]: - self.errors.append( - f"QfcMultiPartParser was able to obtain `DataAndFiles` from the request's input stream, but `MultiValueDict` either lacks a `file` key or a value at `file`! parser_context: {parser_context}. `EmptyContentError` expected." - ) - - return parsed - - -@extend_schema_view( - get=extend_schema( - description="Download a file from a project", - responses={ - (200, "*/*"): OpenApiTypes.BINARY, - }, - ), - post=extend_schema( - description="Upload a file to the project", - parameters=[ - OpenApiParameter( - name="file", - type=OpenApiTypes.BINARY, - location=OpenApiParameter.QUERY, - required=True, - description="File to be uploaded", - ) - ], - ), - delete=extend_schema(description="Delete a file from a project"), -) -class DownloadPushDeleteFileView(views.APIView): - # TODO: swagger doc - # TODO: docstring - parser_classes = [QfcMultiPartSerializer] - permission_classes = [ - permissions.IsAuthenticated, - DownloadPushDeleteFileViewPermissions, - ] - - def get(self, request, projectid, filename): - Project.objects.get(id=projectid) - - version = None - if "version" in self.request.query_params: - version = self.request.query_params["version"] - - key = utils.safe_join(f"projects/{projectid}/files/", filename) - return utils2.storage.file_response( - request, - key, - expires=600, - version=version, - as_attachment=True, - ) - - # TODO refactor this function by moving the actual upload and Project model updates to library function outside the view - def post(self, request, projectid, filename, format=None): - if len(request.FILES.getlist("file")) > 1: - raise exceptions.MultipleContentsError() - - # QF-2540 - # Getting traceback in case the traceback provided by Sentry is too short - # Add post-serialization keys for diff-ing with pre-serialization keys - if "file" not in request.data and not request.FILES.getlist("file"): - if "file" not in request.data: - logger.warning( - 'The key "file" was not found in `request.data`. Sending report to Sentry.' - ) - - if not request.FILES.getlist("file"): - logger.warning( - 'The key "file" occurs in `request.data` but maps to an empty list. Sending report to Sentry.' - ) - - callstack_buffer = io.StringIO() - print_stack(limit=50, file=callstack_buffer) - - request_attributes = { - "data": str(copy.copy(self.request.data).keys()), - "files": str(self.request.FILES.keys()), - "meta": str(self.request.META), - } - - # QF-2540 - report_serialization_diff_to_sentry( - # using the 'X-Request-Id' added to the request by RequestIDMiddleware - name=f"{request.META.get('X-Request-Id')}_{projectid}", - pre_serialization=request.attached_keys, - post_serialization=",".join(QfcMultiPartSerializer.errors) - + str(request_attributes), - buffer=callstack_buffer, - body_stream=getattr(request, "body_stream", None), - ) - raise exceptions.EmptyContentError() - - # get project from request or db - project = Project.objects.get(id=projectid) - - is_the_qgis_file = utils.is_the_qgis_file(filename) - - if is_the_qgis_file and project.is_shared_datasets_project: - raise exceptions.QGISProjectFileNotAllowedError( - "QGIS project files are not allowed in shared datasets projects." - ) - - # check if the project restricts qgs/qgz file modification to admins - if is_the_qgis_file and not permissions_utils.can_modify_qgis_projectfile( - request.user, project - ): - raise exceptions.RestrictedProjectModificationError( - "The project restricts modification of the QGIS project file to managers and administrators." - ) - - # check only one qgs/qgz file per project - if ( - is_the_qgis_file - and project.has_the_qgis_file - and PurePath(filename) != PurePath(project.the_qgis_file_name) - ): - raise exceptions.MultipleProjectsError( - "Only one QGIS project per project allowed" - ) - - request_file = request.FILES.get("file") - - if hasattr(request, "auth") and hasattr(request.auth, "client_type"): - client_type = request.auth.client_type - else: - client_type = request.session.get("client_type") - - permissions_utils.check_can_upload_file(project, client_type, request_file.size) - - old_object = get_project_file_with_versions(project.id, filename) - sha256sum = utils.get_sha256(request_file) - bucket = utils.get_s3_bucket() - - key = utils.safe_join(f"projects/{projectid}/files/", filename) - metadata = {"Sha256sum": sha256sum} - - bucket.upload_fileobj(request_file, key, ExtraArgs={"Metadata": metadata}) - - new_object = get_project_file_with_versions(project.id, filename) - - assert new_object - - with transaction.atomic(): - # we only enter a transaction after the file is uploaded above because we do not - # want to lock the project row for way too long. If we reselect for update the - # project and update it now, it guarantees there will be no other file upload editing - # the same project row. - project = Project.objects.select_for_update().get(id=projectid) - update_fields = ["data_last_updated_at"] - - if get_attachment_dir_prefix(project, filename) == "" and ( - is_the_qgis_file or project.has_the_qgis_file - ): - if is_the_qgis_file: - project.the_qgis_file_name = filename - update_fields.append("the_qgis_file_name") - - running_jobs = ProcessProjectfileJob.objects.filter( - project=project, - created_by=self.request.user, - status__in=[ - Job.Status.PENDING, - Job.Status.QUEUED, - Job.Status.STARTED, - ], - ) - - if not running_jobs.exists(): - ProcessProjectfileJob.objects.create( - project=project, created_by=self.request.user - ) - - now = timezone.now() - project.data_last_updated_at = now - - if is_admin_restricted_file(filename, project.the_qgis_file_name): - update_fields.append("restricted_data_last_updated_at") - project.restricted_data_last_updated_at = now - - project.save(update_fields=update_fields, recompute_storage=True) - - if old_object: - audit( - project, - LogEntry.Action.UPDATE, - changes={filename: [old_object.latest.e_tag, new_object.latest.e_tag]}, - ) - else: - audit( - project, - LogEntry.Action.CREATE, - changes={filename: [None, new_object.latest.e_tag]}, - ) - - # Delete the old file versions - purge_old_file_versions_legacy(project) - - return Response(status=status.HTTP_201_CREATED) - - @transaction.atomic() - def delete(self, request, projectid, filename): - project = Project.objects.select_for_update().get(id=projectid) - version_id = request.GET.get("version", request.headers.get("x-file-version")) - - if version_id: - utils2.storage.delete_project_file_version_permanently( - project, filename, version_id, False - ) - else: - utils2.storage.delete_project_file_permanently(project, filename) - - return Response(status=status.HTTP_200_OK) - - -@extend_schema_view( - get=extend_schema( - description="Download the metadata of a project's file", - responses={ - (200, "*/*"): OpenApiTypes.BINARY, - }, - ) -) -class ProjectMetafilesView(views.APIView): - parser_classes = [MultiPartParser] - permission_classes = [ - permissions.IsAuthenticated, - DownloadPushDeleteFileViewPermissions, - ] - - def get(self, request, projectid, filename): - key = utils.safe_join(f"projects/{projectid}/meta/", filename) - return utils2.storage.file_response(request, key) - - -@extend_schema_view( - get=extend_schema( - description="Download a public file, e.g. user avatar.", - responses={ - (200, "*/*"): OpenApiTypes.BINARY, - }, - ) -) -class PublicFilesView(views.APIView): - parser_classes = [MultiPartParser] - permission_classes = [] - - def get(self, request, filename): - return utils2.storage.file_response(request, filename) - - -@extend_schema(exclude=True) -class AdminDownloadPushDeleteFileView(DownloadPushDeleteFileView): - """Allowing `DownloadPushDeleteFileView` to be excluded from the OpenAPI schema documentation""" - - -@extend_schema(exclude=True) -class AdminListFilesViews(ListFilesView): - """Allowing `ListFilesView` to be excluded from the OpenAPI schema documentation""" diff --git a/docker-app/qfieldcloud/core/views/package_views.py b/docker-app/qfieldcloud/core/views/package_views.py index 237698667..00b02c835 100644 --- a/docker-app/qfieldcloud/core/views/package_views.py +++ b/docker-app/qfieldcloud/core/views/package_views.py @@ -4,7 +4,6 @@ from django.core.exceptions import ObjectDoesNotExist from django.shortcuts import get_object_or_404 from django.urls import reverse -from django.views.decorators.csrf import csrf_exempt from drf_spectacular.utils import ( OpenApiParameter, OpenApiTypes, @@ -12,15 +11,10 @@ extend_schema_view, ) from qfieldcloud.authentication.models import AuthToken -from qfieldcloud.core import exceptions, utils +from qfieldcloud.core import exceptions from qfieldcloud.core import permissions_utils as perms from qfieldcloud.core.models import PackageJob, Project from qfieldcloud.core.serializers import LatestPackageSerializer -from qfieldcloud.core.utils import ( - check_s3_key, - get_project_files, - get_project_package_files, -) from qfieldcloud.core.utils2 import storage from qfieldcloud.filestorage.models import ( File, @@ -76,193 +70,6 @@ def has_permission(self, request, view): return False -@extend_schema_view( - get=extend_schema( - description="Get all the files in a project package with files.", - responses={200: LatestPackageSerializer()}, - ), -) -class LegacyLatestPackageView(views.APIView): - permission_classes = [permissions.IsAuthenticated, PackageViewPermissions] - - def get(self, request, project_id): - """Get last project package status and file list.""" - project = Project.objects.get(id=project_id) - latest_finished_package_job = project.latest_finished_package_job_for_user( - request.user - ) - - # Check if the project was packaged at least once - if not latest_finished_package_job: - raise exceptions.InvalidJobError( - "Packaging has never been triggered or successful for this project." - ) - - filenames = set() - files = [] - - # NOTE Some clients (e.g. QFieldSync) are still requiring the `sha256` key to check whether the files needs to be reuploaded. - # Since we do not have control on these old client versions, we need to keep the API backward compatible for some time and assume `skip_metadata=0` by default. - skip_metadata_param = request.GET.get("skip_metadata", "0") - if skip_metadata_param == "0": - skip_metadata = False - else: - skip_metadata = bool(skip_metadata_param) - - for f in get_project_package_files( - project_id, str(latest_finished_package_job.id) - ): - file_data = { - "name": f.name, - "size": f.size, - "last_modified": f.last_modified, - "md5sum": f.md5sum, - "is_attachment": False, - } - - if not skip_metadata: - file_data["sha256"] = check_s3_key(f.key) - - filenames.add(f.name) - files.append(file_data) - - # get attachment files directly from the original project files, not from the package - for attachment_dir in project.attachment_dirs: - for f in get_project_files(project_id, attachment_dir): - # skip files that are part of the package - if f.name in filenames: - continue - - file_data = { - "name": f.name, - "size": f.size, - "last_modified": f.last_modified, - "md5sum": f.md5sum, - "is_attachment": True, - } - - if not skip_metadata: - file_data["sha256"] = check_s3_key(f.key) - - filenames.add(f.name) - files.append(file_data) - - if not files: - raise exceptions.InvalidJobError("Empty project package.") - - assert latest_finished_package_job.feedback - - feedback_version = latest_finished_package_job.feedback.get("feedback_version") - - # version 2 and 3 have the same format - if feedback_version in ["2.0", "3.0"]: - layers = latest_finished_package_job.feedback["outputs"][ - "qgis_layers_data" - ]["layers_by_id"] - # support some ancient QFieldCloud job data - elif feedback_version is None: - steps = latest_finished_package_job.feedback.get("steps", []) - - layers = None - if len(steps) > 2 and steps[1].get("stage", 1) == 2: - layers = steps[1]["outputs"]["layer_checks"] - - # be paranoid and raise for newer versions - else: - raise NotImplementedError() - - return Response( - { - "files": files, - "layers": layers, - "status": latest_finished_package_job.status, - "package_id": latest_finished_package_job.pk, - "packaged_at": latest_finished_package_job.project.data_last_packaged_at, - "data_last_updated_at": latest_finished_package_job.project.data_last_updated_at, - } - ) - - -@extend_schema_view( - get=extend_schema( - description="Download a file from a project package.", - responses={ - (200, "*/*"): OpenApiTypes.BINARY, - }, - ), -) -class LegacyLatestPackageDownloadFilesView(views.APIView): - permission_classes = [permissions.IsAuthenticated, PackageViewPermissions] - - def get(self, request, project_id, filename): - """Download package file. - - Raises: - exceptions.InvalidJobError: [description] - """ - project = Project.objects.get(id=project_id) - latest_finished_package_job = project.latest_finished_package_job_for_user( - request.user - ) - - # Check if the project was packaged at least once - if not latest_finished_package_job: - raise exceptions.InvalidJobError( - "Packaging has never been triggered or successful for this project." - ) - - key = f"projects/{project_id}/packages/{latest_finished_package_job.id}/{filename}" - - # files within attachment dirs that do not exist is the packaged files should be served - # directly from the original data storage - if storage.get_attachment_dir_prefix(project, filename) and not check_s3_key( - key - ): - key = f"projects/{project_id}/files/{filename}" - - # NOTE the `expires` kwarg is sending the `Expires` header to the client, keep it a low value (in seconds). - return storage.file_response(request, key, expires=10, as_attachment=True) - - -@extend_schema_view( - post=extend_schema( - description="Upload a file to the package", - parameters=[ - OpenApiParameter( - name="file", - type=OpenApiTypes.BINARY, - location=OpenApiParameter.QUERY, - required=True, - description="File to be uploaded", - ) - ], - ) -) -class LegacyPackageUploadFilesView(views.APIView): - permission_classes = [permissions.IsAuthenticated, PackageUploadViewPermissions] - - def post(self, request, project_id, job_id, filename): - """Upload the package files.""" - key = utils.safe_join(f"projects/{project_id}/packages/{job_id}/", filename) - - request_file = request.FILES.get("file") - sha256sum = utils.get_sha256(request_file) - md5sum = utils.get_md5sum(request_file) - metadata = {"Sha256sum": sha256sum} - - bucket = utils.get_s3_bucket() - bucket.upload_fileobj(request_file, key, ExtraArgs={"Metadata": metadata}) - - return Response( - { - "name": filename, - "size": request_file.size, - "sha256": sha256sum, - "md5sum": md5sum, - } - ) - - @extend_schema_view( get=extend_schema( description="Get all the files in a project package with files.", @@ -432,76 +239,3 @@ def post( status=status.HTTP_201_CREATED, headers=headers, ) - - -@csrf_exempt -def compatibility_latest_package_view(request: Request, *args, **kwargs) -> Response: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - project_id: UUID = kwargs["project_id"] - project = get_object_or_404(Project, id=project_id) - - if project.uses_legacy_storage: - logger.debug( - f"Project {project_id=} will be using the legacy package file management." - ) - - return LegacyLatestPackageView.as_view()(request, *args, **kwargs) - else: - logger.debug( - f"Project {project_id=} will be using the regular package file management." - ) - - return LatestPackageView.as_view()(request, *args, **kwargs) - - -@csrf_exempt -def compatibility_package_download_files_view( - request: Request, *args, **kwargs -) -> Response: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - project_id: UUID = kwargs["project_id"] - project = get_object_or_404(Project, id=project_id) - - if project.uses_legacy_storage: - logger.debug( - f"Project {project_id=} will be using the legacy package file management." - ) - - return LegacyLatestPackageDownloadFilesView.as_view()(request, *args, **kwargs) - else: - logger.debug( - f"Project {project_id=} will be using the regular package file management." - ) - - return LatestPackageDownloadFilesView.as_view()(request, *args, **kwargs) - - -@csrf_exempt -def compatibility_package_upload_files_view( - request: Request, *args, **kwargs -) -> Response: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - project_id: UUID = kwargs["project_id"] - project = get_object_or_404(Project, id=project_id) - - if project.uses_legacy_storage: - logger.debug( - f"Project {project_id=} will be using the legacy package file management." - ) - - return LegacyPackageUploadFilesView.as_view()(request, *args, **kwargs) - else: - logger.debug( - f"Project {project_id=} will be using the regular package file management." - ) - - return PackageUploadFilesView.as_view()(request, *args, **kwargs) diff --git a/docker-app/qfieldcloud/core/views/projects_views.py b/docker-app/qfieldcloud/core/views/projects_views.py index d43f4bd15..dd9c45b22 100644 --- a/docker-app/qfieldcloud/core/views/projects_views.py +++ b/docker-app/qfieldcloud/core/views/projects_views.py @@ -11,7 +11,6 @@ from qfieldcloud.core.filters import ProjectFilterSet from qfieldcloud.core.models import Project, ProjectQueryset from qfieldcloud.core.serializers import ProjectSerializer -from qfieldcloud.core.utils2 import storage from qfieldcloud.subscription.exceptions import QuotaError from rest_framework import filters as drf_filters from rest_framework import generics, permissions, viewsets @@ -141,10 +140,7 @@ def perform_update(self, serializer: ProjectSerializer) -> None: def destroy(self, request, projectid): # Delete files from storage - project = Project.objects.get(id=projectid) - - if project.uses_legacy_storage: - storage.delete_all_project_files_permanently(projectid) + _project = Project.objects.get(id=projectid) return super().destroy(request, projectid) diff --git a/docker-app/qfieldcloud/filestorage/management/commands/migrateavatars.py b/docker-app/qfieldcloud/filestorage/management/commands/migrateavatars.py index 5b230797c..a48bfa889 100644 --- a/docker-app/qfieldcloud/filestorage/management/commands/migrateavatars.py +++ b/docker-app/qfieldcloud/filestorage/management/commands/migrateavatars.py @@ -1,3 +1,8 @@ +""" +Migrate avatars from the legacy storage to the `default` storage. +NOTE: this command is legacy and deprecated, it should not be used anymore. It is only kept for historical reference and should be removed in the future. +""" + import logging from datetime import datetime diff --git a/docker-app/qfieldcloud/filestorage/serializers.py b/docker-app/qfieldcloud/filestorage/serializers.py index 74c86f259..88a3c7870 100644 --- a/docker-app/qfieldcloud/filestorage/serializers.py +++ b/docker-app/qfieldcloud/filestorage/serializers.py @@ -46,7 +46,6 @@ class Meta: "uploaded_at", "display", "is_latest", - # TODO delete the fields below, they are deprecated and exists only as a compatibility layer for the legacy storage "last_modified", "sha256", ) @@ -58,7 +57,6 @@ class Meta: "uploaded_at", "display", "is_latest", - # TODO delete the fields below, they are deprecated and exists only as a compatibility layer for the legacy storage "last_modified", "sha256", ) @@ -97,7 +95,6 @@ class Meta: "uploaded_at", "is_attachment", "md5sum", - # TODO delete the fields below, they are deprecated and exists only as a compatibility layer for the legacy storage "last_modified", "sha256", ] @@ -107,7 +104,6 @@ class Meta: "uploaded_at", "is_attachment", "md5sum", - # TODO delete the fields below, they are deprecated and exists only as a compatibility layer for the legacy storage "last_modified", "sha256", ] diff --git a/docker-app/qfieldcloud/filestorage/tests/test_files_api.py b/docker-app/qfieldcloud/filestorage/tests/test_files_api.py index 6370e616c..fc533db40 100644 --- a/docker-app/qfieldcloud/filestorage/tests/test_files_api.py +++ b/docker-app/qfieldcloud/filestorage/tests/test_files_api.py @@ -50,10 +50,7 @@ def setUp(self): def assertFileUploaded( self, user: User, project: Project, filename: str, content: IO ) -> HttpResponse | Response: - if project.uses_legacy_storage: - return self._assertFileUploadedLegacy(user, project, filename, content) - else: - return self._assertFileUploaded(user, project, filename, content) + return self._assertFileUploaded(user, project, filename, content) def _assertFileUploadedLegacy( self, user: User, project: Project, filename: str, content: IO diff --git a/docker-app/qfieldcloud/filestorage/tests/test_storage_usage.py b/docker-app/qfieldcloud/filestorage/tests/test_storage_usage.py index 8814173a0..b8cf74353 100644 --- a/docker-app/qfieldcloud/filestorage/tests/test_storage_usage.py +++ b/docker-app/qfieldcloud/filestorage/tests/test_storage_usage.py @@ -56,31 +56,3 @@ def test_upload_file_succeeds(self): self.assertEqual(self.p1.get_file("file.name").versions.count(), 2) self.assertEqual(self.p1.file_storage_bytes, 13) self.assertEqual(self.u1.useraccount.storage_used_bytes, 13) - - # 3) creating a second project with legacy storage backend - # TODO: Delete with QF-4963 Drop support for legacy storage - p2 = Project.objects.create( - owner=self.u1, - name="p2", - file_storage="legacy_storage", - ) - - # 4) adding a file in the legacy storage backend - response = self._upload_file(self.u1, p2, "file.name", StringIO("Hello3!")) - - self.p1.refresh_from_db() - p2.refresh_from_db() - self.u1.useraccount.refresh_from_db() - - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - # p1 checks - self.assertEqual(self.p1.project_files.count(), 1) - self.assertEqual(self.p1.get_file("file.name").versions.count(), 2) - self.assertEqual(self.p1.file_storage_bytes, 13) - # TODO: Change the number of files with QF-4963 Drop support for legacy storage - # p2 checks - # since the project is in the legacy storage, no `File`` object is created. - self.assertEqual(p2.project_files.count(), 0) - self.assertEqual(p2.file_storage_bytes, 7) - - self.assertEqual(self.u1.useraccount.storage_used_bytes, 20) diff --git a/docker-app/qfieldcloud/filestorage/urls.py b/docker-app/qfieldcloud/filestorage/urls.py index 18763a751..240807d89 100644 --- a/docker-app/qfieldcloud/filestorage/urls.py +++ b/docker-app/qfieldcloud/filestorage/urls.py @@ -2,31 +2,31 @@ from .views import ( AvatarFileReadView, - compatibility_file_crud_view, - compatibility_file_list_view, - compatibility_file_metadata_view, - compatibility_project_meta_file_read_view, + FileCrudView, + FileListView, + FileMetadataView, + ProjectMetaFileReadView, ) urlpatterns = [ path( "files//", - compatibility_file_list_view, + FileListView.as_view(), name="filestorage_list_files", ), path( "files///", - compatibility_file_crud_view, + FileCrudView.as_view(), name="filestorage_crud_file", ), path( "files/metadata///", - compatibility_file_metadata_view, + FileMetadataView.as_view(), name="filestorage_file_metadata", ), path( "files/thumbnails//", - compatibility_project_meta_file_read_view, + ProjectMetaFileReadView.as_view(), name="filestorage_project_thumbnails", ), path( diff --git a/docker-app/qfieldcloud/filestorage/views.py b/docker-app/qfieldcloud/filestorage/views.py index be738ff8b..0ff532787 100644 --- a/docker-app/qfieldcloud/filestorage/views.py +++ b/docker-app/qfieldcloud/filestorage/views.py @@ -5,10 +5,9 @@ from django.core import signing from django.db.models import Q, QuerySet from django.http import Http404 -from django.http.response import HttpResponse, HttpResponseBase +from django.http.response import HttpResponseBase from django.shortcuts import get_object_or_404, redirect from django.urls import reverse -from django.views.decorators.csrf import csrf_exempt from drf_spectacular.utils import ( OpenApiParameter, OpenApiTypes, @@ -22,24 +21,11 @@ from qfieldcloud.core import ( pagination, permissions_utils, - utils2, ) from qfieldcloud.core.models import ( Project, UserAccount, ) -from qfieldcloud.core.views.files_views import ( - DownloadPushDeleteFileView as LegacyFileCrudView, -) -from qfieldcloud.core.views.files_views import ( - LegacyFileMetadataView, -) -from qfieldcloud.core.views.files_views import ( - ListFilesView as LegacyFileListView, -) -from qfieldcloud.core.views.files_views import ( - ProjectMetafilesView as LegacyProjectMetaFileReadView, -) from qfieldcloud.filestorage.models import ( File, ) @@ -262,148 +248,4 @@ def get( str(useraccount.avatar), ) else: - if useraccount.legacy_avatar_uri: - return utils2.storage.file_response( - request._request, - useraccount.legacy_avatar_uri, - ) - else: - return redirect(staticfiles_storage.url("logo.svg")) - - -@csrf_exempt -def compatibility_file_list_view( - request: Request, *args, **kwargs -) -> Response | HttpResponse: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - # let's assume that `kwargs["project_id"]` will no throw a `KeyError` - project_id: UUID = kwargs["project_id"] - view_kwargs = kwargs.pop("view_kwargs", {}) - - try: - project = Project.objects.get(id=project_id) - except Project.DoesNotExist: - # if the project does not exist, we just fallback to the new view, which will return JSON formatted 404 later - return FileListView.as_view(**view_kwargs)(request, *args, **kwargs) - - if project.uses_legacy_storage: - # rename the `project_id` to previously used `projectid`, so we don't change anything in the legacy code - kwargs["projectid"] = kwargs.pop("project_id") - - logger.debug(f"Project {project_id=} will be using the legacy file management.") - - return LegacyFileListView.as_view(**view_kwargs)(request, *args, **kwargs) - else: - logger.debug( - f"Project {project_id=} will be using the regular file management." - ) - - return FileListView.as_view(**view_kwargs)(request, *args, **kwargs) - - -@csrf_exempt -def compatibility_file_metadata_view( - request: Request, *args, **kwargs -) -> Response | HttpResponse: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - # let's assume that `kwargs["project_id"]` will no throw a `KeyError` - project_id: UUID = kwargs["project_id"] - view_kwargs = kwargs.pop("view_kwargs", {}) - - try: - project = Project.objects.get(id=project_id) - except Project.DoesNotExist: - # if the project does not exist, we just fallback to the new view, which will return JSON formatted 404 later - return FileMetadataView.as_view(**view_kwargs)(request, *args, **kwargs) - - if project.uses_legacy_storage: - # rename the `project_id` to previously used `projectid`, so we don't change anything in the legacy code - kwargs["projectid"] = kwargs.pop("project_id") - - logger.debug(f"Project {project_id=} will be using the legacy file management.") - - return LegacyFileMetadataView.as_view(**view_kwargs)(request, *args, **kwargs) - else: - logger.debug( - f"Project {project_id=} will be using the regular file management." - ) - - return FileMetadataView.as_view(**view_kwargs)(request, *args, **kwargs) - - -@csrf_exempt -def compatibility_file_crud_view( - request: Request, *args, **kwargs -) -> Response | HttpResponse: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - # let's assume that `kwargs["project_id"]` will no throw a `KeyError` - project_id: UUID = kwargs["project_id"] - view_kwargs = kwargs.pop("view_kwargs", {}) - - try: - project = Project.objects.get(id=project_id) - except Project.DoesNotExist: - # if the project does not exist, we just fallback to the new view, which will return JSON formatted 404 later - return FileCrudView.as_view(**view_kwargs)(request, *args, **kwargs) - - if project.uses_legacy_storage: - # rename the `project_id` to previously used `projectid`, so we don't change anything in the legacy code - kwargs["projectid"] = kwargs.pop("project_id") - - logger.debug(f"Project {project_id=} will be using the legacy file management.") - - return LegacyFileCrudView.as_view(**view_kwargs)(request, *args, **kwargs) - else: - logger.debug( - f"Project {project_id=} will be using the regular file management." - ) - - return FileCrudView.as_view(**view_kwargs)(request, *args, **kwargs) - - -@csrf_exempt -def compatibility_project_meta_file_read_view( - request: Request, *args, **kwargs -) -> Response | HttpResponse: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - # let's assume that `kwargs["project_id"]` will no throw a `KeyError` - project_id: UUID = kwargs["project_id"] - view_kwargs = kwargs.pop("view_kwargs", {}) - - try: - project = Project.objects.get(id=project_id) - except Project.DoesNotExist: - # if the project does not exist, we just fallback to the new view, which will return JSON formatted 404 later - return ProjectMetaFileReadView.as_view(**view_kwargs)(request, *args, **kwargs) - - if project.uses_legacy_storage: - # rename the `project_id` to previously used `projectid`, so we don't change anything in the legacy code - kwargs["projectid"] = kwargs.pop("project_id") - # hardcode the thumbnail file name - kwargs["filename"] = "thumbnail.png" - - logger.debug( - f"Project {project_id=} will be using the legacy file management for meta files." - ) - - return LegacyProjectMetaFileReadView.as_view(**view_kwargs)( - request, *args, **kwargs - ) - else: - logger.debug( - f"Project {project_id=} will be using the regular file management for meta files." - ) - - return ProjectMetaFileReadView.as_view(**view_kwargs)(request, *args, **kwargs) + return redirect(staticfiles_storage.url("logo.svg")) diff --git a/docker-app/qfieldcloud/settings.py b/docker-app/qfieldcloud/settings.py index 2aec4b460..aadb4b957 100644 --- a/docker-app/qfieldcloud/settings.py +++ b/docker-app/qfieldcloud/settings.py @@ -299,8 +299,6 @@ }, } -LEGACY_STORAGE_NAME = _storage_config["LEGACY_STORAGE_NAME"] - # Maximum filename length in characters # NOTE the keys on S3 cannot be longer than 1024 _bytes_, see https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html # NOTE the files on Windows cannot be longer than 260 _chars_ by default, see https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file?redirectedfrom=MSDN#maximum-path-length-limitation diff --git a/docker-app/qfieldcloud/settings_utils.py b/docker-app/qfieldcloud/settings_utils.py index 6b6a510da..3efb8baa7 100644 --- a/docker-app/qfieldcloud/settings_utils.py +++ b/docker-app/qfieldcloud/settings_utils.py @@ -28,7 +28,6 @@ class DjangoStorages(TypedDict): class StoragesConfig(TypedDict): STORAGES: dict[str, DjangoStorages] - LEGACY_STORAGE_NAME: str def get_storages_config() -> StoragesConfig: @@ -43,17 +42,7 @@ def get_storages_config() -> StoragesConfig: "Envvar STORAGES should be a parsable JSON string!" ) else: - raw_storages["default"] = { - "BACKEND": "qfieldcloud.filestorage.backend.QfcS3Boto3Storage", - "OPTIONS": { - "access_key": os.environ["STORAGE_ACCESS_KEY_ID"], - "secret_key": os.environ["STORAGE_SECRET_ACCESS_KEY"], - "bucket_name": os.environ["STORAGE_BUCKET_NAME"], - "region_name": os.environ["STORAGE_REGION_NAME"], - "endpoint_url": os.environ["STORAGE_ENDPOINT_URL"], - }, - "QFC_IS_LEGACY": True, - } + raise ConfigValidationError("No STORAGES envvar configured!") if not isinstance(raw_storages, dict): raise ConfigValidationError( @@ -61,11 +50,7 @@ def get_storages_config() -> StoragesConfig: ) if "default" not in raw_storages: - raise ConfigValidationError( - "Envvar STORAGES_SERVICE_BY_DEFAULT should be non-empty string!" - ) - - legacy_storage_name = "" + raise ConfigValidationError("Envvar STORAGES should contain a default storage!") for service_name, service_config in raw_storages.items(): if not service_name: @@ -83,17 +68,8 @@ def get_storages_config() -> StoragesConfig: ) ) - if service_config["QFC_IS_LEGACY"]: - if legacy_storage_name: - raise ConfigValidationError( - 'There must be only one legacy storage, but both "{service_name}" and "{legacy_storage}" have `QFC_IS_LEGACY` set to `True`!' - ) - - legacy_storage_name = service_name - return { "STORAGES": raw_storages, - "LEGACY_STORAGE_NAME": legacy_storage_name, } diff --git a/docker-app/qfieldcloud/subscription/tests/test_job_creation.py b/docker-app/qfieldcloud/subscription/tests/test_job_creation.py index 3ceb1379f..be6ee3f63 100644 --- a/docker-app/qfieldcloud/subscription/tests/test_job_creation.py +++ b/docker-app/qfieldcloud/subscription/tests/test_job_creation.py @@ -95,19 +95,13 @@ def test_create_job_if_project_owner_is_over_quota(self): plan = self.user1.useraccount.current_subscription.plan more_bytes_than_plan = (plan.storage_mb * 1000 * 1000) + 1 - # TODO Delete with QF-4963 Drop support for legacy storage - if self.project1.uses_legacy_storage: - # Create a project that uses all the storage - self.project1.file_storage_bytes = more_bytes_than_plan - self.project1.save() - else: - FileVersion.objects.add_version( - project=self.project1, - filename="bigfile.name", - content=ContentFile(b"x" * more_bytes_than_plan, "dummy.name"), - file_type=File.FileType.PROJECT_FILE, - uploaded_by=self.user1, - ) + FileVersion.objects.add_version( + project=self.project1, + filename="bigfile.name", + content=ContentFile(b"x" * more_bytes_than_plan, "dummy.name"), + file_type=File.FileType.PROJECT_FILE, + uploaded_by=self.user1, + ) self.check_cannot_create_jobs(QuotaError) self.check_can_update_existing_jobs() diff --git a/docker-app/qfieldcloud/subscription/tests/test_package.py b/docker-app/qfieldcloud/subscription/tests/test_package.py index a904db362..c25adacaf 100644 --- a/docker-app/qfieldcloud/subscription/tests/test_package.py +++ b/docker-app/qfieldcloud/subscription/tests/test_package.py @@ -639,21 +639,13 @@ def test_used_storage_changes_when_uploading_and_deleting_files_and_versions(sel storage_free_mb=0.6, ) - # TODO Delete with QF-4963 Drop support for legacy storage - if p1.uses_legacy_storage: - version = p1.legacy_files[0].versions[0] - else: - version = p1.project_files[0].versions.all().reverse()[0] + version = p1.project_files[0].versions.all().reverse()[0] response = self.client.delete( f"/api/v1/files/{p1.id}/file.name/?version={str(version.id)}", ) - # TODO Delete with QF-4963 Drop support for legacy storage - if p1.uses_legacy_storage: - self.assertEqual(response.status_code, status.HTTP_200_OK) - else: - self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) p1.save(recompute_storage=True) @@ -673,11 +665,7 @@ def test_used_storage_changes_when_uploading_and_deleting_files_and_versions(sel response = self.client.delete(f"/api/v1/files/{p1.id}/file.name/") - # TODO Delete with QF-4963 Drop support for legacy storage - if p1.uses_legacy_storage: - self.assertEqual(response.status_code, status.HTTP_200_OK) - else: - self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) p1.save(recompute_storage=True) diff --git a/docker-app/qfieldcloud/urls.py b/docker-app/qfieldcloud/urls.py index ffb8c6646..ee49ed3ab 100644 --- a/docker-app/qfieldcloud/urls.py +++ b/docker-app/qfieldcloud/urls.py @@ -17,8 +17,6 @@ 'blog/', include('blog.urls')) """ -from functools import wraps - from django.conf import settings from django.contrib import admin from django.urls import include, path @@ -35,8 +33,8 @@ from qfieldcloud.core.admin import qfc_admin_site from qfieldcloud.core.views.redirect_views import redirect_to_admin_project_view from qfieldcloud.filestorage.views import ( - compatibility_file_crud_view, - compatibility_file_list_view, + FileCrudView, + FileListView, ) admin.site.site_header = _("QFieldCloud Admin") @@ -44,21 +42,6 @@ admin.site.index_title = _("Welcome to QFieldCloud Admin") -def add_view_kwargs(view, **view_kwargs): - """Adds kwargs to DRF views in the `.as_view()` call. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - - @wraps(view) - def wrapper(request, *args, **kwargs): - kwargs["view_kwargs"] = view_kwargs - return view(request, *args, **kwargs) - - return wrapper - - urlpatterns = [ path( "", @@ -82,15 +65,11 @@ def wrapper(request, *args, **kwargs): ), path( settings.QFIELDCLOUD_ADMIN_URI + "api/files//", - add_view_kwargs( - compatibility_file_list_view, permission_classes=[permissions.IsAdminUser] - ), + FileListView.as_view(permission_classes=[permissions.IsAdminUser]), ), path( settings.QFIELDCLOUD_ADMIN_URI + "api/files///", - add_view_kwargs( - compatibility_file_crud_view, permission_classes=[permissions.IsAdminUser] - ), + FileCrudView.as_view(permission_classes=[permissions.IsAdminUser]), name="project_file_download", ), path(settings.QFIELDCLOUD_ADMIN_URI, qfc_admin_site.urls), diff --git a/docker-app/worker_wrapper/wrapper.py b/docker-app/worker_wrapper/wrapper.py index 5174ea68c..7ad1ee546 100644 --- a/docker-app/worker_wrapper/wrapper.py +++ b/docker-app/worker_wrapper/wrapper.py @@ -634,27 +634,17 @@ def after_docker_run(self) -> None: thumbnail_filename = self.shared_tempdir.joinpath("thumbnail.png") with open(thumbnail_filename, "rb") as f: - # TODO Delete with QF-4963 Drop support for legacy storage - if project.uses_legacy_storage: - legacy_thumbnail_uri = storage.upload_project_thumbail( - project, f, "image/png", "thumbnail" - ) - project.legacy_thumbnail_uri = ( - project.legacy_thumbnail_uri or legacy_thumbnail_uri - ) - else: - project.thumbnail = ContentFile(f.read(), "dummy_thumbnail_name.png") + project.thumbnail = ContentFile(f.read(), "dummy_thumbnail_name.png") project.save( update_fields=( "project_details", - "legacy_thumbnail_uri", "thumbnail", ) ) - # for non-legacy storage, keep only one thumbnail version if so. - if not project.uses_legacy_storage and project.thumbnail: + # keep only one thumbnail version if so. + if project.thumbnail: storage.purge_previous_thumbnails_versions(project) def after_docker_exception(self) -> None: