From 2cb732a9731670384df96f71fcab7556dc3c16a1 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Wed, 11 Feb 2026 05:16:37 +0100 Subject: [PATCH 01/22] chore: get rid of legacy storage --- docker-app/qfieldcloud/settings.py | 2 -- docker-app/qfieldcloud/settings_utils.py | 28 ++---------------------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/docker-app/qfieldcloud/settings.py b/docker-app/qfieldcloud/settings.py index 2aec4b460..aadb4b957 100644 --- a/docker-app/qfieldcloud/settings.py +++ b/docker-app/qfieldcloud/settings.py @@ -299,8 +299,6 @@ }, } -LEGACY_STORAGE_NAME = _storage_config["LEGACY_STORAGE_NAME"] - # Maximum filename length in characters # NOTE the keys on S3 cannot be longer than 1024 _bytes_, see https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html # NOTE the files on Windows cannot be longer than 260 _chars_ by default, see https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file?redirectedfrom=MSDN#maximum-path-length-limitation diff --git a/docker-app/qfieldcloud/settings_utils.py b/docker-app/qfieldcloud/settings_utils.py index 6b6a510da..3efb8baa7 100644 --- a/docker-app/qfieldcloud/settings_utils.py +++ b/docker-app/qfieldcloud/settings_utils.py @@ -28,7 +28,6 @@ class DjangoStorages(TypedDict): class StoragesConfig(TypedDict): STORAGES: dict[str, DjangoStorages] - LEGACY_STORAGE_NAME: str def get_storages_config() -> StoragesConfig: @@ -43,17 +42,7 @@ def get_storages_config() -> StoragesConfig: "Envvar STORAGES should be a parsable JSON string!" ) else: - raw_storages["default"] = { - "BACKEND": "qfieldcloud.filestorage.backend.QfcS3Boto3Storage", - "OPTIONS": { - "access_key": os.environ["STORAGE_ACCESS_KEY_ID"], - "secret_key": os.environ["STORAGE_SECRET_ACCESS_KEY"], - "bucket_name": os.environ["STORAGE_BUCKET_NAME"], - "region_name": os.environ["STORAGE_REGION_NAME"], - "endpoint_url": os.environ["STORAGE_ENDPOINT_URL"], - }, - "QFC_IS_LEGACY": True, - } + raise ConfigValidationError("No STORAGES envvar configured!") if not isinstance(raw_storages, dict): raise ConfigValidationError( @@ -61,11 +50,7 @@ def get_storages_config() -> StoragesConfig: ) if "default" not in raw_storages: - raise ConfigValidationError( - "Envvar STORAGES_SERVICE_BY_DEFAULT should be non-empty string!" - ) - - legacy_storage_name = "" + raise ConfigValidationError("Envvar STORAGES should contain a default storage!") for service_name, service_config in raw_storages.items(): if not service_name: @@ -83,17 +68,8 @@ def get_storages_config() -> StoragesConfig: ) ) - if service_config["QFC_IS_LEGACY"]: - if legacy_storage_name: - raise ConfigValidationError( - 'There must be only one legacy storage, but both "{service_name}" and "{legacy_storage}" have `QFC_IS_LEGACY` set to `True`!' - ) - - legacy_storage_name = service_name - return { "STORAGES": raw_storages, - "LEGACY_STORAGE_NAME": legacy_storage_name, } From a71022f866f8a5515d82da34f4218aed60b554c1 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Wed, 11 Feb 2026 05:19:00 +0100 Subject: [PATCH 02/22] chore: get rid of legacy storage in the `core` module --- docker-app/qfieldcloud/core/models.py | 111 +-- .../qfieldcloud/core/tests/test_commands.py | 24 +- .../core/tests/test_deleteorphanedfiles.py | 222 ----- .../qfieldcloud/core/tests/test_packages.py | 116 +-- .../qfieldcloud/core/tests/test_qgis_file.py | 30 +- docker-app/qfieldcloud/core/utils.py | 518 +---------- .../qfieldcloud/core/utils2/packages.py | 70 +- docker-app/qfieldcloud/core/utils2/storage.py | 852 ------------------ .../qfieldcloud/core/views/files_views.py | 493 ---------- .../qfieldcloud/core/views/package_views.py | 64 +- .../qfieldcloud/core/views/projects_views.py | 6 +- 11 files changed, 100 insertions(+), 2406 deletions(-) delete mode 100644 docker-app/qfieldcloud/core/tests/test_deleteorphanedfiles.py delete mode 100644 docker-app/qfieldcloud/core/utils2/storage.py delete mode 100644 docker-app/qfieldcloud/core/views/files_views.py diff --git a/docker-app/qfieldcloud/core/models.py b/docker-app/qfieldcloud/core/models.py index 2f23761ff..50dcb9b74 100644 --- a/docker-app/qfieldcloud/core/models.py +++ b/docker-app/qfieldcloud/core/models.py @@ -10,7 +10,6 @@ from typing import TYPE_CHECKING, Any, cast from uuid import uuid4 -from deprecated import deprecated from django.conf import settings from django.contrib.auth.models import AbstractUser from django.contrib.auth.models import UserManager as DjangoUserManager @@ -39,9 +38,8 @@ ) from timezone_field import TimeZoneField -from qfieldcloud.core import utils, validators +from qfieldcloud.core import validators from qfieldcloud.core.fields import DynamicStorageFileField, QfcImageField, QfcImageFile -from qfieldcloud.core.utils2 import storage from qfieldcloud.subscription.exceptions import ReachedMaxOrganizationMembersError if TYPE_CHECKING: @@ -467,13 +465,6 @@ class UserAccount(models.Model): twitter = models.CharField(max_length=255, default="", blank=True) is_email_public = models.BooleanField(default=False) - # TODO Delete with QF-4963 Drop support for legacy storage - legacy_avatar_uri = models.CharField( - _("Legacy Profile Picture URI"), - max_length=255, - blank=True, - ) - avatar = QfcImageField( _("Avatar Picture"), upload_to=get_user_account_avatar_upload_to, @@ -518,23 +509,12 @@ def storage_used_bytes(self) -> float: project_files_used_quota = ( FileVersion.objects.filter( file__file_type=File.FileType.PROJECT_FILE, - file__project__in=self.user.projects.exclude( - file_storage=settings.LEGACY_STORAGE_NAME - ), + file__project__in=self.user.projects, ).aggregate(sum_bytes=Sum("size"))["sum_bytes"] or 0 ) - # TODO: Delete with QF-4963 Drop support for legacy storage - legacy_used_quota = ( - self.user.projects.filter( - file_storage=settings.LEGACY_STORAGE_NAME - ).aggregate(sum_bytes=Sum("file_storage_bytes"))["sum_bytes"] - # if there are no projects, the value will be `None` - or 0 - ) - - used_quota = project_files_used_quota + legacy_used_quota + used_quota = project_files_used_quota return used_quota @@ -1211,11 +1191,6 @@ class Meta: ), ) - # TODO: Delete with QF-4963 Drop support for legacy storage - legacy_thumbnail_uri = models.CharField( - _("Legacy Thumbnail Picture URI"), max_length=255, blank=True - ) - thumbnail = DynamicStorageFileField( _("Thumbnail Picture"), upload_to=get_project_thumbnail_upload_to, @@ -1491,17 +1466,9 @@ def owner_aware_storage_keep_versions(self) -> int: @property def thumbnail_url(self) -> StrOrPromise: - """Returns the url to the project's thumbnail or empty string if no URL provided. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - if self.uses_legacy_storage: - if not self.legacy_thumbnail_uri: - return "" - else: - if not self.thumbnail: - return "" + """Returns the url to the project's thumbnail or empty string if no URL provided.""" + if not self.thumbnail: + return "" return reverse_lazy( "filestorage_project_thumbnails", @@ -1575,30 +1542,6 @@ def private(self) -> bool: # still used in the project serializer return not self.is_public - @property - def uses_legacy_storage(self) -> bool: - """Whether the storage of the project is legacy. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - return self.file_storage == settings.LEGACY_STORAGE_NAME - - @cached_property - @deprecated - def legacy_files(self) -> list[utils.S3ObjectWithVersions]: - """Gets all the files from S3 storage. This is potentially slow. Results are cached on the instance. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - if self.uses_legacy_storage: - return list(utils.get_project_files_with_versions(str(self.id))) - else: - raise NotImplementedError( - "The `Project.legacy_files` method is not implemented for projects stored in non-legacy storage" - ) - @property def project_files(self) -> "FileQueryset": """Returns the files of type PROJECT related to the project.""" @@ -1606,14 +1549,7 @@ def project_files(self) -> "FileQueryset": @property def project_files_count(self) -> int: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - if self.uses_legacy_storage: - return len(self.legacy_files) - else: - return self.project_files.count() + return self.project_files.count() @property def users(self): @@ -1852,14 +1788,7 @@ def direct_collaborators(self) -> ProjectCollaboratorQueryset: ) def delete(self, *args, **kwargs): - """Deletes the project and the thumbnail for the legacy storage. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - if self.uses_legacy_storage: - storage.delete_project_thumbnail(self) - + """Deletes the project and the thumbnail for the legacy storage.""" return super().delete(*args, **kwargs) @property @@ -1878,15 +1807,9 @@ def save(self, recompute_storage=False, *args, **kwargs): additional_update_fields = set() if recompute_storage: - # TODO Delete with QF-4963 Drop support for legacy storage - if self.uses_legacy_storage: - self.file_storage_bytes = storage.get_project_file_storage_in_bytes( - self - ) - else: - self.file_storage_bytes = self.project_files.aggregate( - file_storage_bytes=Sum("versions__size", default=0) - )["file_storage_bytes"] + self.file_storage_bytes = self.project_files.aggregate( + file_storage_bytes=Sum("versions__size", default=0) + )["file_storage_bytes"] additional_update_fields.add("file_storage_bytes") @@ -1912,11 +1835,6 @@ def save(self, recompute_storage=False, *args, **kwargs): def get_file(self, filename: str) -> File: return self.project_files.get_by_name(filename) # type: ignore - def legacy_get_file(self, filename: str) -> utils.S3ObjectWithVersions: - files = filter(lambda f: f.latest.name == filename, self.legacy_files) - - return next(files) - class ProjectCollaboratorQueryset(models.QuerySet): def validated(self, skip_invalid=False): @@ -2710,10 +2628,5 @@ class Meta: ) def _get_file_storage_name(self) -> str: - # TODO Delete with QF-4963 Drop support for legacy storage - # Legacy storage - use default storage - if self.project.uses_legacy_storage: - return "default" - - # Non-legacy storage - use same storage as project + # Use same storage as project return self.project.file_storage diff --git a/docker-app/qfieldcloud/core/tests/test_commands.py b/docker-app/qfieldcloud/core/tests/test_commands.py index d40885ea9..46524a94d 100644 --- a/docker-app/qfieldcloud/core/tests/test_commands.py +++ b/docker-app/qfieldcloud/core/tests/test_commands.py @@ -1,5 +1,4 @@ import csv -import io import os from django.core.files.base import ContentFile @@ -8,7 +7,6 @@ from qfieldcloud.core.models import Person, Project from qfieldcloud.core.tests.utils import set_subscription, setup_subscription_plans -from qfieldcloud.core.utils2 import storage from qfieldcloud.filestorage.models import File, FileVersion @@ -23,20 +21,14 @@ def setUpTestData(cls): # Project cls.p1 = Project.objects.create(name="test_project", owner=user) - # TODO Delete with QF-4963 Drop support for legacy storage - if cls.p1.uses_legacy_storage: - storage.upload_project_file( - cls.p1, io.BytesIO(b"Hello world!"), "project.qgs" - ) - else: - FileVersion.objects.add_version( - project=cls.p1, - filename="file.name", - # NOTE the dummy name is required when running tests on GitHub CI, but not locally. Spent few hours before I isolated this... - content=ContentFile(b"Hello world!", "dummy.name"), - file_type=File.FileType.PROJECT_FILE, - uploaded_by=user, - ) + FileVersion.objects.add_version( + project=cls.p1, + filename="file.name", + # NOTE the dummy name is required when running tests on GitHub CI, but not locally. Spent few hours before I isolated this... + content=ContentFile(b"Hello world!", "dummy.name"), + file_type=File.FileType.PROJECT_FILE, + uploaded_by=user, + ) def test_extracts3data_output_to_file(self): output_file = "extracted.csv" diff --git a/docker-app/qfieldcloud/core/tests/test_deleteorphanedfiles.py b/docker-app/qfieldcloud/core/tests/test_deleteorphanedfiles.py deleted file mode 100644 index d79bcec32..000000000 --- a/docker-app/qfieldcloud/core/tests/test_deleteorphanedfiles.py +++ /dev/null @@ -1,222 +0,0 @@ -""" -Todo: - * Delete with QF-4963 Drop support for legacy storage -""" - -import io - -from django.conf import settings -from django.core.management import call_command -from django.test import TestCase - -from qfieldcloud.core.models import Person, Project -from qfieldcloud.core.utils import get_project_files_count, get_s3_bucket -from qfieldcloud.core.utils2 import storage - -from .utils import set_subscription, setup_subscription_plans - - -class QfcTestCase(TestCase): - def setUp(self): - setup_subscription_plans() - - self.u1 = Person.objects.create(username="u1") - set_subscription(self.u1, "default_user") - self.projects: list[Project] = [] - - get_s3_bucket().objects.filter(Prefix="projects/").delete() - - self.generate_projects(2) - - def generate_projects(self, count: int): - offset = len(self.projects) - for i in range(1, count + 1): - p = Project.objects.create( - name=f"p{offset + i}", - owner=self.u1, - file_storage=settings.LEGACY_STORAGE_NAME, - ) - self.projects.append(p) - file = io.BytesIO(b"Hello world!") - storage.upload_project_file(p, file, "project.qgs") - - def call_command(self, *args, **kwargs): - out = io.StringIO() - call_command( - "deleteorphanedfiles", - *args, - stdout=out, - stderr=out, - **kwargs, - ) - return out.getvalue() - - def test_nothing_to_delete(self): - project_ids = sorted([str(p.id) for p in self.projects]) - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(self.projects[1].project_files_count, 1) - - out = self.call_command() - - self.assertEqual( - out.strip(), - "\n".join( - [ - "Checking the last 2 project id(s) from the storage...", - "No project files to delete.", - ] - ), - ) - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(self.projects[1].project_files_count, 1) - - def test_dry_run(self): - project_ids = sorted([str(p.id) for p in self.projects]) - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(self.projects[1].project_files_count, 1) - - out = self.call_command(dry_run=True) - - self.assertEqual( - out.strip(), - "\n".join( - [ - "Dry run, no files will be deleted.", - "Checking the last 2 project id(s) from the storage...", - "No project files to delete.", - ] - ), - ) - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(self.projects[1].project_files_count, 1) - - def test_delete_files(self): - project_ids = sorted([str(p.id) for p in self.projects]) - Project.objects.filter(id__in=project_ids).delete() - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(get_project_files_count(project_ids[1]), 1) - - out = self.call_command() - - self.assertEqual( - out.strip(), - "\n".join( - [ - "Checking the last 2 project id(s) from the storage...", - f'Deleting project files for "{project_ids[0]}"...', - f'Deleting project files for "{project_ids[1]}"...', - ] - ), - ) - - self.assertEqual(get_project_files_count(project_ids[0]), 0) - self.assertEqual(get_project_files_count(project_ids[1]), 0) - - def test_delete_files_dry_run(self): - project_ids = sorted([str(p.id) for p in self.projects]) - Project.objects.filter(id__in=project_ids).delete() - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(get_project_files_count(project_ids[1]), 1) - - out = self.call_command(dry_run=True) - - self.assertEqual( - out.strip(), - "\n".join( - [ - "Dry run, no files will be deleted.", - "Checking the last 2 project id(s) from the storage...", - f'Deleting project files for "{project_ids[0]}"...', - f'Deleting project files for "{project_ids[1]}"...', - ] - ), - ) - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(get_project_files_count(project_ids[1]), 1) - - def test_invalid_uuid(self): - project_ids = sorted([str(p.id) for p in self.projects]) - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(self.projects[1].project_files_count, 1) - - file = io.BytesIO(b"Hello world!") - storage.upload_file(file, "projects/strangename/project.qgs") - - out = self.call_command() - - self.assertEqual( - out.strip(), - "\n".join( - [ - "Invalid uuid: strangename/project.qgs", - "Checking the last 2 project id(s) from the storage...", - "No project files to delete.", - ] - ), - ) - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(self.projects[1].project_files_count, 1) - - def test_batches(self): - self.generate_projects(2) - project_ids = sorted([str(p.id) for p in self.projects]) - Project.objects.filter(id__in=project_ids[:2]).delete() - - self.assertEqual(get_project_files_count(project_ids[0]), 1) - self.assertEqual(get_project_files_count(project_ids[1]), 1) - self.assertEqual(get_project_files_count(project_ids[2]), 1) - self.assertEqual(get_project_files_count(project_ids[3]), 1) - - out = self.call_command(limit=2) - - self.assertEqual( - out.strip(), - "\n".join( - [ - "Checking a batch of 2 project ids from the storage...", - "Checking a batch of 2 project ids from the storage...", - f'Deleting project files for "{project_ids[0]}"...', - f'Deleting project files for "{project_ids[1]}"...', - ] - ), - ) - - self.assertEqual(get_project_files_count(project_ids[0]), 0) - self.assertEqual(get_project_files_count(project_ids[1]), 0) - self.assertEqual(get_project_files_count(project_ids[2]), 1) - self.assertEqual(get_project_files_count(project_ids[3]), 1) - - def test_deletes_extra_files_on_second_level(self): - file = io.BytesIO(b"Hello world!") - storage.upload_project_file(self.projects[0], file, "inner/path/data.txt") - - project_ids = sorted([str(p.id) for p in self.projects]) - Project.objects.filter(id__in=project_ids).delete() - - self.assertEqual(get_project_files_count(str(self.projects[0].id)), 2) - self.assertEqual(get_project_files_count(str(self.projects[1].id)), 1) - - out = self.call_command() - - self.assertEqual( - out.strip(), - "\n".join( - [ - "Checking the last 2 project id(s) from the storage...", - f'Deleting project files for "{project_ids[0]}"...', - f'Deleting project files for "{project_ids[1]}"...', - ] - ), - ) - - self.assertEqual(get_project_files_count(project_ids[0]), 0) - self.assertEqual(get_project_files_count(project_ids[1]), 0) diff --git a/docker-app/qfieldcloud/core/tests/test_packages.py b/docker-app/qfieldcloud/core/tests/test_packages.py index 6223b021a..028d0cd94 100644 --- a/docker-app/qfieldcloud/core/tests/test_packages.py +++ b/docker-app/qfieldcloud/core/tests/test_packages.py @@ -32,7 +32,6 @@ wait_for_project_ok_status, ) from qfieldcloud.core.utils2.jobs import repackage -from qfieldcloud.core.utils2.storage import get_stored_package_ids from qfieldcloud.filestorage.models import File logging.disable(logging.CRITICAL) @@ -639,11 +638,6 @@ def test_outdated_packaged_files_are_deleted(self): ) self.conn.commit() - if not self.project1.uses_legacy_storage: - self.assertEqual( - File.objects.filter(file_type=File.FileType.PACKAGE_FILE).count(), 0 - ) - self.upload_files_and_check_package( token=self.token1.key, project=self.project1, @@ -662,18 +656,10 @@ def test_outdated_packaged_files_are_deleted(self): "created_at" ) - # TODO Delete with QF-4963 Drop support for legacy storage - if self.project1.uses_legacy_storage: - stored_package_ids = get_stored_package_ids(self.project1) - self.assertIn(str(old_package.id), stored_package_ids) - self.assertEqual(len(stored_package_ids), 1) - else: - self.assertGreaterEqual( - File.objects.filter(package_job=old_package).count(), 1 - ) - self.assertGreaterEqual( - File.objects.filter(file_type=File.FileType.PACKAGE_FILE).count(), 1 - ) + self.assertGreaterEqual(File.objects.filter(package_job=old_package).count(), 1) + self.assertGreaterEqual( + File.objects.filter(file_type=File.FileType.PACKAGE_FILE).count(), 1 + ) self.check_package( self.token1.key, @@ -689,22 +675,11 @@ def test_outdated_packaged_files_are_deleted(self): "created_at" ) - # TODO Delete with QF-4963 Drop support for legacy storage - if self.project1.uses_legacy_storage: - stored_package_ids = get_stored_package_ids(self.project1) - - self.assertNotEqual(old_package.id, new_package.id) - self.assertNotIn(str(old_package.id), stored_package_ids) - self.assertIn(str(new_package.id), stored_package_ids) - self.assertEqual(len(stored_package_ids), 1) - else: - self.assertEqual(File.objects.filter(package_job=old_package).count(), 0) - self.assertGreaterEqual( - File.objects.filter(package_job=new_package).count(), 1 - ) - self.assertGreaterEqual( - File.objects.filter(file_type=File.FileType.PACKAGE_FILE).count(), 1 - ) + self.assertEqual(File.objects.filter(package_job=old_package).count(), 0) + self.assertGreaterEqual(File.objects.filter(package_job=new_package).count(), 1) + self.assertGreaterEqual( + File.objects.filter(file_type=File.FileType.PACKAGE_FILE).count(), 1 + ) def test_package_and_project_file_attachments(self): # upload attachments to the project @@ -764,64 +739,45 @@ def test_purge_obsolete_package_files_works_fine(self): wait_for_project_ok_status(self.project1) self.project1.refresh_from_db() - if self.project1.uses_legacy_storage: - # TODO Delete with QF-4963 Drop support for legacy storage - stored_package_ids = get_stored_package_ids(self.project1) - - self.assertIn(str(package_job_1.id), stored_package_ids) - self.assertIn(str(other_user_package_job.id), stored_package_ids) - self.assertEquals(len(stored_package_ids), 2) - - else: - package_files_p1_qs = File.objects.filter( - project=self.project1, - file_type=File.FileType.PACKAGE_FILE, - package_job=package_job_1, - ) + package_files_p1_qs = File.objects.filter( + project=self.project1, + file_type=File.FileType.PACKAGE_FILE, + package_job=package_job_1, + ) - self.assertEquals(package_files_p1_qs.count(), 3) + self.assertEquals(package_files_p1_qs.count(), 3) # repackage the project for the same user. package_job_2 = repackage(self.project1, self.user1) wait_for_project_ok_status(self.project1) self.project1.refresh_from_db() - if self.project1.uses_legacy_storage: - # TODO Delete with QF-4963 Drop support for legacy storage - stored_package_ids = get_stored_package_ids(self.project1) - - self.assertNotIn(str(package_job_1.id), stored_package_ids) - self.assertIn(str(package_job_2.id), stored_package_ids) - self.assertIn(str(other_user_package_job.id), stored_package_ids) - self.assertEquals(len(stored_package_ids), 2) - - else: - # make sure old package files are deleted. - package_files_p1_qs = File.objects.filter( - project=self.project1, - file_type=File.FileType.PACKAGE_FILE, - package_job=package_job_1, - ) + # make sure old package files are deleted. + package_files_p1_qs = File.objects.filter( + project=self.project1, + file_type=File.FileType.PACKAGE_FILE, + package_job=package_job_1, + ) - self.assertEquals(package_files_p1_qs.count(), 0) + self.assertEquals(package_files_p1_qs.count(), 0) - # make sure new package files are there. - package_files_p2_qs = File.objects.filter( - project=self.project1, - file_type=File.FileType.PACKAGE_FILE, - package_job=package_job_2, - ) + # make sure new package files are there. + package_files_p2_qs = File.objects.filter( + project=self.project1, + file_type=File.FileType.PACKAGE_FILE, + package_job=package_job_2, + ) - self.assertEquals(package_files_p2_qs.count(), 3) + self.assertEquals(package_files_p2_qs.count(), 3) - # make sure the other user's package files are there. - other_user_package_files_qs = File.objects.filter( - project=self.project1, - file_type=File.FileType.PACKAGE_FILE, - package_job=other_user_package_job, - ) + # make sure the other user's package files are there. + other_user_package_files_qs = File.objects.filter( + project=self.project1, + file_type=File.FileType.PACKAGE_FILE, + package_job=other_user_package_job, + ) - self.assertEquals(other_user_package_files_qs.count(), 3) + self.assertEquals(other_user_package_files_qs.count(), 3) def test_needs_repackaging(self): # 0. Create two users, where one owns the project and the other is a project collaborator. diff --git a/docker-app/qfieldcloud/core/tests/test_qgis_file.py b/docker-app/qfieldcloud/core/tests/test_qgis_file.py index b03567adf..3ae2a9fde 100644 --- a/docker-app/qfieldcloud/core/tests/test_qgis_file.py +++ b/docker-app/qfieldcloud/core/tests/test_qgis_file.py @@ -304,12 +304,6 @@ def test_upload_and_list_file_checksum(self): self.assertEqual(json[0]["name"], "file.txt") self.assertEqual(json[0]["size"], 13) - # The `sha256` key is optional only for the legacy storage, there is no performance penalty for the non-legacy storage if we send it back, - # therefore `skip_metadata` is ignored in non-legacy storage - # TODO Delete with QF-4963 Drop support for legacy storage - if self.project1.uses_legacy_storage: - self.assertNotIn("sha256", json[0]) - self.assertIn("md5sum", json[0]) self.assertEqual( json[0]["md5sum"], @@ -703,28 +697,18 @@ def count_versions(): """counts the versions in first file of project1""" project = Project.objects.get(pk=self.project1.pk) - # TODO Delete with QF-4963 Drop support for legacy storage - if project.uses_legacy_storage: - return len(project.legacy_get_file("file.txt").versions) - else: - return project.get_file("file.txt").versions.count() + return project.get_file("file.txt").versions.count() def read_version(n): """returns the content of version in first file of project1""" project = Project.objects.get(pk=self.project1.pk) - # TODO Delete with QF-4963 Drop support for legacy storage - if project.uses_legacy_storage: - file = ( - project.legacy_get_file("file.txt").versions[n]._data.get()["Body"] - ) - else: - file = ( - project.get_file("file.txt") - .versions.all() - .order_by("uploaded_at")[n] - .content - ) + file = ( + project.get_file("file.txt") + .versions.all() + .order_by("uploaded_at")[n] + .content + ) return file.read().decode() diff --git a/docker-app/qfieldcloud/core/utils.py b/docker-app/qfieldcloud/core/utils.py index 8098401d8..474cceb8a 100644 --- a/docker-app/qfieldcloud/core/utils.py +++ b/docker-app/qfieldcloud/core/utils.py @@ -3,184 +3,15 @@ import json import logging import os -import posixpath -from datetime import datetime -from pathlib import PurePath -from typing import IO, Generator, NamedTuple +from typing import IO -import boto3 import jsonschema -import mypy_boto3_s3 -from botocore.errorfactory import ClientError from django.conf import settings from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile -from qfieldcloud.settings_utils import DjangoStorages - logger = logging.getLogger(__name__) -class S3PrefixPath(NamedTuple): - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - - Key: str - - -class S3Object(NamedTuple): - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - - name: str - key: str - last_modified: datetime - size: int - etag: str - md5sum: str - - -class S3ObjectVersion: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - - def __init__( - self, name: str, data: mypy_boto3_s3.service_resource.ObjectVersion - ) -> None: - self.name = name - self._data = data - - @property - def id(self) -> str: - """Returns the version id""" - # NOTE id and version_id are the same thing - return self._data.id - - @property - def key(self) -> str: - return self._data.key - - @property - def last_modified(self) -> datetime: - return self._data.last_modified - - @property - def size(self) -> int: - return self._data.size - - @property - def e_tag(self) -> str: - return self._data.e_tag - - @property - def md5sum(self) -> str: - return self._data.e_tag.replace('"', "") - - @property - def is_latest(self) -> bool: - return self._data.is_latest - - @property - def display(self) -> str: - return self.last_modified.strftime("v%Y%m%d%H%M%S") - - -class S3ObjectWithVersions(NamedTuple): - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - - latest: S3ObjectVersion - versions: list[S3ObjectVersion] - - @property - def total_size(self) -> int: - """Total size of all versions""" - # latest is also in versions - return sum(v.size for v in self.versions if v.size is not None) - - -def get_legacy_s3_credentials() -> DjangoStorages: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - assert settings.LEGACY_STORAGE_NAME - - return settings.STORAGES[settings.LEGACY_STORAGE_NAME] - - -def get_s3_session() -> boto3.Session: - """Get a new S3 Session instance using Django settings - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - session = boto3.Session( - aws_access_key_id=get_legacy_s3_credentials()["OPTIONS"]["access_key"], - aws_secret_access_key=get_legacy_s3_credentials()["OPTIONS"]["secret_key"], - region_name=get_legacy_s3_credentials()["OPTIONS"]["region_name"], - ) - return session - - -def get_s3_bucket() -> mypy_boto3_s3.service_resource.Bucket: - """ - Get a new S3 Bucket instance using Django settings. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket_name = get_legacy_s3_credentials()["OPTIONS"]["bucket_name"] - - assert bucket_name, "Expected `bucket_name` to be non-empty string!" - - session = get_s3_session() - s3 = session.resource( - "s3", - endpoint_url=get_legacy_s3_credentials()["OPTIONS"]["endpoint_url"], - ) - - # Ensure the bucket exists - s3.meta.client.head_bucket(Bucket=bucket_name) - - # Get the bucket resource - return s3.Bucket(bucket_name) - - -def get_s3_client() -> mypy_boto3_s3.Client: - """Get a new S3 client instance using Django settings - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - - s3_session = get_s3_session() - s3_client = s3_session.client( - "s3", - endpoint_url=get_legacy_s3_credentials()["OPTIONS"]["endpoint_url"], - ) - return s3_client - - -def get_sha256(file: IO) -> str: - """Return the sha256 hash of the file - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - if type(file) is InMemoryUploadedFile or type(file) is TemporaryUploadedFile: - return _get_sha256_memory_file(file) - else: - return _get_sha256_file(file) - - def _get_sha256_memory_file(file: InMemoryUploadedFile | TemporaryUploadedFile) -> str: BLOCKSIZE = 65536 hasher = hashlib.sha256() @@ -206,18 +37,6 @@ def _get_sha256_file(file: IO) -> str: return hasher.hexdigest() -def get_md5sum(file: IO) -> str: - """Return the md5sum hash of the file - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - if type(file) is InMemoryUploadedFile or type(file) is TemporaryUploadedFile: - return _get_md5sum_memory_file(file) - else: - return _get_md5sum_file(file) - - def _get_md5sum_memory_file(file: InMemoryUploadedFile | TemporaryUploadedFile) -> str: BLOCKSIZE = 65536 hasher = hashlib.md5() @@ -253,132 +72,6 @@ def strip_json_null_bytes(file: IO) -> IO: return result -def safe_join(base: str, *paths: str) -> str: - """ - A version of django.utils._os.safe_join for S3 paths. - Joins one or more path components to the base path component - intelligently. Returns a normalized version of the final path. - The final path must be located inside of the base path component - (otherwise a ValueError is raised). - Paths outside the base path indicate a possible security - sensitive operation. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - base_path = base - base_path = base_path.rstrip("/") - paths = tuple(paths) - - final_path = base_path + "/" - - for path in paths: - _final_path = posixpath.normpath(posixpath.join(final_path, path)) - - # posixpath.normpath() strips the trailing /. Add it back. - if path.endswith("/") or _final_path + "/" == final_path: - _final_path += "/" - - final_path = _final_path - - if final_path == base_path: - final_path += "/" - - # Ensure final_path starts with base_path and that the next character after - # the base path is /. - base_path_len = len(base_path) - if not final_path.startswith(base_path) or final_path[base_path_len] != "/": - raise ValueError( - "the joined path is located outside of the base path component" - ) - - return final_path.lstrip("/") - - -def is_the_qgis_file(filename: str) -> bool: - """Returns whether the filename seems to be a QGIS project file by checking the file extension. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - path = PurePath(filename) - - if path.suffix.lower() in (".qgs", ".qgz"): - return True - - return False - - -def get_qgis_project_file(project_id: str) -> str | None: - """Return the relative path inside the project of the qgs/qgz file or - None if no qgs/qgz file is present - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - - bucket = get_s3_bucket() - - prefix = f"projects/{project_id}/files/" - - for obj in bucket.objects.filter(Prefix=prefix): - if is_the_qgis_file(obj.key): - path = PurePath(obj.key) - return str(path.relative_to(*path.parts[:3])) - - return None - - -def check_legacy_s3_file_exists(key: str, should_raise: bool = True) -> bool: - """Check to see if an object exists on S3. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - client = get_s3_client() - try: - client.head_object( - Bucket=get_legacy_s3_credentials()["OPTIONS"]["bucket_name"], - Key=key, - ) - return True - except ClientError as e: - if e.response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404: - return False - else: - if should_raise: - raise e - else: - return False - - -def check_s3_key(key: str) -> str | None: - """Check to see if an object exists on S3. It it exists, the function - returns the sha256 of the file from the metadata - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - - client = get_s3_client() - try: - head = client.head_object( - Bucket=get_legacy_s3_credentials()["OPTIONS"]["bucket_name"], - Key=key, - ) - except ClientError as e: - if e.response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404: - return None - else: - raise e - - metadata = head["Metadata"] - if "sha256sum" in metadata: - return metadata["sha256sum"] - else: - return metadata["Sha256sum"] - - def get_deltafile_schema_validator() -> jsonschema.Draft7Validator: """Creates a JSON schema validator to check whether the provided delta file is valid. @@ -398,215 +91,6 @@ def get_deltafile_schema_validator() -> jsonschema.Draft7Validator: return jsonschema.Draft7Validator(schema_dict) -def get_project_files(project_id: str, path: str = "") -> list[S3Object]: - """Returns a list of files and their versions. - - Args: - project_id: the project id - path: additional filter prefix - - Returns: - the list of files - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = get_s3_bucket() - root_prefix = f"projects/{project_id}/files/" - prefix = f"projects/{project_id}/files/{path}" - - return list_files(bucket, prefix, root_prefix) - - -def get_project_files_with_versions( - project_id: str, -) -> Generator[S3ObjectWithVersions, None, None]: - """Returns a generator of files and their versions. - - Args: - project_id: the project id - - Returns: - the list of files - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = get_s3_bucket() - prefix = f"projects/{project_id}/files/" - - return list_files_with_versions(bucket, prefix, prefix) - - -def get_project_file_with_versions( - project_id: str, filename: str -) -> S3ObjectWithVersions | None: - """Returns a list of files and their versions. - - Args: - project_id: the project id - - Returns: - the list of files - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = get_s3_bucket() - root_prefix = f"projects/{project_id}/files/" - prefix = f"projects/{project_id}/files/{filename}" - files = [ - f - for f in list_files_with_versions(bucket, prefix, root_prefix) - if f.latest.key == prefix - ] - - if files: - return files[0] - else: - return None - - -def get_project_package_files(project_id: str, package_id: str) -> list[S3Object]: - """Returns a list of package files. - - Args: - project_id: the project id - - Returns: - the list of package files - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = get_s3_bucket() - prefix = f"projects/{project_id}/packages/{package_id}/" - - return list_files(bucket, prefix, prefix) - - -def get_project_files_count(project_id: str) -> int: - """Returns the number of files within a project. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = get_s3_bucket() - prefix = f"projects/{project_id}/files/" - files = list(bucket.objects.filter(Prefix=prefix)) - - return len(files) - - -def get_project_package_files_count(project_id: str) -> int: - """Returns the number of package files within a project. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = get_s3_bucket() - prefix = f"projects/{project_id}/export/" - files = list(bucket.objects.filter(Prefix=prefix)) - - return len(files) - - -def list_files( - bucket: mypy_boto3_s3.service_resource.Bucket, - prefix: str, - strip_prefix: str = "", -) -> list[S3Object]: - """List a bucket's objects under prefix. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - files = [] - for f in bucket.objects.filter(Prefix=prefix): - if strip_prefix: - start_idx = len(strip_prefix) - name = f.key[start_idx:] - else: - name = f.key - - files.append( - S3Object( - name=name, - key=f.key, - last_modified=f.last_modified, - size=f.size, - etag=f.e_tag, - md5sum=f.e_tag.replace('"', ""), - ) - ) - - files.sort(key=lambda f: f.name) - - return files - - -def list_versions( - bucket: mypy_boto3_s3.service_resource.Bucket, - prefix: str, - strip_prefix: str = "", -) -> list[S3ObjectVersion]: - """Iterator that lists a bucket's objects under prefix. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - versions = [] - for v in bucket.object_versions.filter(Prefix=prefix): - if strip_prefix: - start_idx = len(prefix) - name = v.key[start_idx:] - else: - name = v.key - - versions.append(S3ObjectVersion(name, v)) - - versions.sort(key=lambda v: (v.key, v.last_modified)) - - return versions - - -def list_files_with_versions( - bucket: mypy_boto3_s3.service_resource.Bucket, - prefix: str, - strip_prefix: str = "", -) -> Generator[S3ObjectWithVersions, None, None]: - """Yields an object with all it's versions - Yields: - Generator[S3ObjectWithVersions]: the object with its versions - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - last_key = None - versions: list[S3ObjectVersion] = [] - latest: S3ObjectVersion | None = None - - for v in list_versions(bucket, prefix, strip_prefix): - if last_key != v.key: - if last_key: - assert latest - - yield S3ObjectWithVersions(latest, versions) - - latest = None - versions = [] - last_key = v.key - - versions.append(v) - - if v.is_latest: - latest = v - - if last_key: - assert latest - yield S3ObjectWithVersions(latest, versions) - - def get_file_storage_choices() -> list[tuple[str, str]]: """ Returns configured storages keys. diff --git a/docker-app/qfieldcloud/core/utils2/packages.py b/docker-app/qfieldcloud/core/utils2/packages.py index 0d942ec0f..19947392f 100644 --- a/docker-app/qfieldcloud/core/utils2/packages.py +++ b/docker-app/qfieldcloud/core/utils2/packages.py @@ -1,9 +1,7 @@ import logging -import uuid from typing import Iterable from qfieldcloud.core import models -from qfieldcloud.core.utils2 import storage from qfieldcloud.filestorage.models import File logger = logging.getLogger(__name__) @@ -29,59 +27,31 @@ def delete_obsolete_packages(projects: Iterable[models.Project]) -> None: ) for project in projects: - # TODO Delete with QF-4963 Drop support for legacy storage - if project.uses_legacy_storage: - stored_package_ids = list( - map(uuid.UUID, storage.get_stored_package_ids(project)) - ) + latest_project_package_jobs_ids = project.latest_package_jobs().values_list( + "id", flat=True + ) - latest_package_job_ids = project.latest_package_jobs().values_list( - "id", flat=True + files_to_delete_qs = ( + File.objects.filter( + project=project, + file_type=File.FileType.PACKAGE_FILE, ) - - for stored_package_id in stored_package_ids: - # the job is still active, so it might be one of the new packages - if stored_package_id in active_package_job_ids: - continue - - # keep packages that are used by other users due to user-assigned secrets. - shall_skip_package = False - for latest_package_job_id in latest_package_job_ids: - if stored_package_id == latest_package_job_id: - shall_skip_package = True - break - - if shall_skip_package: - continue - - storage.delete_stored_package(project, str(stored_package_id)) - - else: - latest_project_package_jobs_ids = project.latest_package_jobs().values_list( - "id", flat=True + .exclude( + package_job_id__in=active_package_job_ids, ) - - files_to_delete_qs = ( - File.objects.filter( - project=project, - file_type=File.FileType.PACKAGE_FILE, - ) - .exclude( - package_job_id__in=active_package_job_ids, - ) - .exclude( - package_job_id__in=latest_project_package_jobs_ids, - ) + .exclude( + package_job_id__in=latest_project_package_jobs_ids, ) + ) - logger.info( - "Deleting {} package files from previous and obsolete packages for project id {}:\n{}".format( - files_to_delete_qs.count(), - project.id, - "\n".join(files_to_delete_qs.values_list("name", flat=True)), - ) + logger.info( + "Deleting {} package files from previous and obsolete packages for project id {}:\n{}".format( + files_to_delete_qs.count(), + project.id, + "\n".join(files_to_delete_qs.values_list("name", flat=True)), ) + ) - delete_count = files_to_delete_qs.delete() + delete_count = files_to_delete_qs.delete() - logger.info(f"Deleted {delete_count} package files.") + logger.info(f"Deleted {delete_count} package files.") diff --git a/docker-app/qfieldcloud/core/utils2/storage.py b/docker-app/qfieldcloud/core/utils2/storage.py deleted file mode 100644 index afb3fdbaa..000000000 --- a/docker-app/qfieldcloud/core/utils2/storage.py +++ /dev/null @@ -1,852 +0,0 @@ -"""A module with legacy file storage management. - -Todo: - * Delete with QF-4963 Drop support for legacy storage -""" - -from __future__ import annotations - -import hashlib -import logging -import re -from enum import Enum -from pathlib import PurePath -from typing import IO - -from django.conf import settings -from django.core.files.base import ContentFile -from django.core.files.storage import storages -from django.db import transaction -from django.http import FileResponse, HttpRequest -from django.http.response import HttpResponse, HttpResponseBase -from django.utils import timezone -from mypy_boto3_s3.type_defs import ObjectIdentifierTypeDef - -import qfieldcloud.core.models -import qfieldcloud.core.utils -from qfieldcloud.core.utils2.audit import LogEntry, audit -from qfieldcloud.filestorage.backend import QfcS3Boto3Storage -from qfieldcloud.filestorage.utils import is_admin_restricted_file - -logger = logging.getLogger(__name__) - - -def legacy_only(func): - """ - Decorator to verify that given project is stored on the legacy storage. - Otherwise, it calls the decorated function. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - * Delete all decorated functions with QF-4963 Drop support for legacy storage - """ - - def wrapper(project, *args, **kwargs): - if not project.uses_legacy_storage: - raise NotImplementedError( - f"This function is not implemented for '{project.file_storage}' file storage!" - ) - - return func(project, *args, **kwargs) - - return wrapper - - -def _delete_by_prefix_versioned(prefix: str): - """ - Delete all objects and their versions starting with a given prefix. - - Similar concept to delete a directory. - Do not use when deleting objects with precise key, as it will delete all objects that start with the same name. - Deleting with this method will leave a deleted version and the deletion is not permanent. - In other words, it is a soft delete. Read more here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/DeletingObjectVersions.html - - Args: - prefix: Object's prefix to search and delete. Check the given prefix if it matches the expected format before using this function! - - Raises: - RuntimeError: When the given prefix is not a string, empty string or leading slash. Check is very basic, do a throrogh checks before calling! - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - logging.info(f"S3 object deletion (versioned) with {prefix=}") - - # Illegal prefix is either empty string ("") or slash ("/"), it will delete random 1000 objects. - if not isinstance(prefix, str) or prefix == "" or prefix == "/": - raise RuntimeError(f"Attempt to delete S3 object with illegal {prefix=}") - - bucket = qfieldcloud.core.utils.get_s3_bucket() - return bucket.objects.filter(Prefix=prefix).delete() - - -def _delete_by_prefix_permanently(prefix: str): - """ - Delete all objects and their versions starting with a given prefix. - - Similar concept to delete a directory. - Do not use when deleting objects with precise key, as it will delete all objects that start with the same name. - Deleting with this method will permanently delete objects and all their versions and the deletion is impossible to recover. - In other words, it is a hard delete. Read more here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/DeletingObjectVersions.html - - Args: - prefix: Object's prefix to search and delete. Check the given prefix if it matches the expected format before using this function! - - Raises: - RuntimeError: When the given prefix is not a string, empty string or leading slash. Check is very basic, do a throrogh checks before calling! - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - logging.info(f"S3 object deletion (permanent) with {prefix=}") - - # Illegal prefix is either empty string ("") or slash ("/"), it will delete random 1000 object versions. - if not isinstance(prefix, str) or prefix == "" or prefix == "/": - raise RuntimeError(f"Attempt to delete S3 object with illegal {prefix=}") - - bucket = qfieldcloud.core.utils.get_s3_bucket() - return bucket.object_versions.filter(Prefix=prefix).delete() - - -def _delete_by_key_versioned(key: str): - """ - Delete an object with a given key. - - Deleting with this method will leave a deleted version and the deletion is not permanent. - In other words, it is a soft delete. - - Args: - key: Object's key to search and delete. Check the given key if it matches the expected format before using this function! - - Raises: - RuntimeError: When the given key is not a string, empty string or leading slash. Check is very basic, do a throrogh checks before calling! - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - logging.info(f"Delete (versioned) S3 object with {key=}") - - # prevent disastrous results when prefix is either empty string ("") or slash ("/"). - if not isinstance(key, str) or key == "" or key == "/": - raise RuntimeError( - f"Attempt to delete (versioned) S3 object with illegal {key=}" - ) - - bucket = qfieldcloud.core.utils.get_s3_bucket() - - return bucket.delete_objects( - Delete={ - "Objects": [ - { - "Key": key, - } - ], - }, - ) - - -def _delete_by_key_permanently(key: str): - """ - Delete an object with a given key. - - Deleting with this method will permanently delete objects and all their versions and the deletion is impossible to recover. - In other words, it is a hard delete. - - Args: - key: Object's key to search and delete. Check the given key if it matches the expected format before using this function! - - Raises: - RuntimeError: When the given key is not a string, empty string or leading slash. Check is very basic, do a throrogh checks before calling! - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - logging.info(f"Delete (permanently) S3 object with {key=}") - - # prevent disastrous results when prefix is either empty string ("") or slash ("/"). - if not isinstance(key, str) or key == "" or key == "/": - raise RuntimeError( - f"Attempt to delete (permanently) S3 object with illegal {key=}" - ) - - bucket = qfieldcloud.core.utils.get_s3_bucket() - - # NOTE filer by prefix will return all objects with that prefix. E.g. for given key="orho.tif", it will return "ortho.tif", "ortho.tif.aux.xml" and "ortho.tif.backup" - temp_objects = bucket.object_versions.filter( - Prefix=key, - ) - object_to_delete: list[ObjectIdentifierTypeDef] = [] - for temp_object in temp_objects: - # filter out objects that do not have the same key as the requested deletion key. - if temp_object.key != key: - continue - - object_to_delete.append( - { - "Key": key, - "VersionId": temp_object.id, - } - ) - - if len(object_to_delete) == 0: - logging.warning( - f"Attempt to delete (permanently) S3 objects did not match any existing objects for {key=}", - extra={ - "all_objects": [ - (o.key, o.version_id, o.e_tag, o.last_modified, o.is_latest) - for o in temp_objects - ] - }, - ) - return None - - logging.info( - f"Delete (permanently) S3 object with {key=} will delete delete {len(object_to_delete)} version(s)" - ) - - return bucket.delete_objects( - Delete={ - "Objects": object_to_delete, - }, - ) - - -def delete_version_permanently(version_obj: qfieldcloud.core.utils.S3ObjectVersion): - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - logging.info( - f'S3 object version deletion (permanent) with "{version_obj.key=}" and "{version_obj.id=}"' - ) - - version_obj._data.delete() - - -def get_attachment_dir_prefix( - project: qfieldcloud.core.models.Project, filename: str -) -> str: # noqa: F821 - """Returns the attachment dir where the file belongs to or empty string if it does not. - - Args: - project: project to check - filename: filename to check - - Returns: - the attachment dir or empty string if no match found - """ - for attachment_dir in project.attachment_dirs: - if filename.startswith(attachment_dir): - return attachment_dir - - return "" - - -def file_response( - request: HttpRequest, - key: str, - expires: int = 60, - version: str | None = None, - as_attachment: bool = False, -) -> HttpResponseBase: - """Return a Django HTTP response with nginx speedup if reverse proxy detected. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - url = "" - filename = PurePath(key).name - extra_params = {} - - if version is not None: - extra_params["VersionId"] = version - - # Assume that if the request is secure, we are behind a nginx proxy and we can use `X-Accel-Redirect` - if request.is_secure() and not settings.IN_TEST_SUITE: - if as_attachment: - extra_params["ResponseContentType"] = "application/force-download" - extra_params["ResponseContentDisposition"] = ( - f'attachment;filename="{filename}"' - ) - - url = qfieldcloud.core.utils.get_s3_client().generate_presigned_url( - "get_object", - Params={ - **extra_params, - "Key": key, - "Bucket": qfieldcloud.core.utils.get_s3_bucket().name, - }, - ExpiresIn=expires, - HttpMethod="GET", - ) - - # Let's NGINX handle the redirect to the storage and streaming the file contents back to the client - response = HttpResponse() - response["X-Accel-Redirect"] = "/storage-download/" - response["redirect_uri"] = url - - return response - elif settings.DEBUG or settings.IN_TEST_SUITE: - return_file = ContentFile(b"") - qfieldcloud.core.utils.get_s3_bucket().download_fileobj( - key, - return_file, - extra_params, - ) - - return FileResponse( - return_file.open(), - as_attachment=as_attachment, - filename=filename, - ) - - raise Exception( - "Expected to either run behind nginx proxy, debug mode or within a test suite." - ) - - -class ImageMimeTypes(str, Enum): - svg = "image/svg+xml" - png = "image/png" - jpg = "image/jpeg" - - @classmethod - def or_none(cls, string: str) -> ImageMimeTypes | None: - try: - return cls(string) - except ValueError: - return None - - -def upload_user_avatar( - user: qfieldcloud.core.models.User, file: IO, mimetype: ImageMimeTypes -) -> str: # noqa: F821 - """Uploads a picture as a user avatar. - - NOTE this function does NOT modify the `UserAccount.legacy_avatar_uri` field - - Args: - user: - file: file used as avatar - mimetype: file mimetype - - Returns: - URI to the avatar - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = qfieldcloud.core.utils.get_s3_bucket() - key = f"users/{user.username}/avatar.{mimetype.name}" - bucket.upload_fileobj( - file, - key, - { - "ContentType": mimetype.value, - }, - ) - return key - - -def delete_user_avatar(user: qfieldcloud.core.models.User) -> None: # noqa: F821 - """Deletes the user's avatar file. - - NOTE this function does NOT modify the `UserAccount.legacy_avatar_uri` field - - Args: - user: - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - key = user.useraccount.legacy_avatar_uri - - # it well could be the user has no avatar yet - if not key: - return - - # e.g. "users/suricactus/avatar.svg" - if not key or not re.match(r"^users/[\w-]+/avatar\.(png|jpg|svg)$", key): - raise RuntimeError(f"Suspicious S3 deletion of user avatar {key=}") - - _delete_by_key_permanently(key) - - -@legacy_only -def upload_project_thumbail( - project: qfieldcloud.core.models.Project, - file: IO, - mimetype: str, - filename: str, # noqa: F821 -) -> str: - """Uploads a picture as a project thumbnail. - - NOTE this function does NOT modify the `Project.thumbnail_uri` field - - Args: - project: - file: file used as thumbail - mimetype: file mimetype - filename: filename - - Returns: - URI to the thumbnail - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = qfieldcloud.core.utils.get_s3_bucket() - - # for now we always expect PNGs - if mimetype == "image/svg+xml": - extension = "svg" - elif mimetype == "image/png": - extension = "png" - elif mimetype == "image/jpeg": - extension = "jpg" - else: - raise Exception(f"Unknown mimetype: {mimetype}") - - key = f"projects/{project.id}/meta/{filename}.{extension}" - bucket.upload_fileobj( - file, - key, - { - "ContentType": mimetype, - }, - ) - return key - - -@legacy_only -def delete_project_thumbnail( - project: qfieldcloud.core.models.Project, -) -> None: # noqa: F821 - """Delete a picture as a project thumbnail. - - NOTE this function does NOT modify the `Project.thumbnail_uri` field - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - key = project.legacy_thumbnail_uri - - # it well could be the project has no thumbnail yet - if not key: - return - - if not key or not re.match( - # e.g. "projects/9bf34e75-0a5d-47c3-a2f0-ebb7126eeccc/meta/thumbnail.png" - r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/meta/thumbnail\.(png|jpg|svg)$", - key, - ): - raise RuntimeError(f"Suspicious S3 deletion of project thumbnail image {key=}") - - _delete_by_key_permanently(key) - - -def purge_previous_thumbnails_versions( - project: qfieldcloud.core.models.Project, -) -> None: - # this method applies only to S3 storage - if not isinstance(project.file_storage, QfcS3Boto3Storage): - return - - bucket = storages[project.file_storage].bucket # type: ignore - prefix = project.thumbnail.name - - if not prefix: - return - - thumbnail_files = list( - qfieldcloud.core.utils.list_files_with_versions(bucket, prefix) - ) - - if len(thumbnail_files) == 0: - logger.info(f'No thumbnail found to delete for project "{project.id}"!') - return - - assert len(thumbnail_files) == 1 - - thumbnail_file = thumbnail_files[0] - - # we only keep 1 version of the thumbnail file. - # otherwise we hit the limit of 1000. - keep_count = 1 - - old_versions_to_purge = sorted( - thumbnail_file.versions, key=lambda v: v.last_modified, reverse=True - )[keep_count:] - - # Remove the N oldest - for old_version in old_versions_to_purge: - logger.info( - f'Purging {old_version.key=} {old_version.id=} as old version for "{thumbnail_file.latest.name}"...' - ) - - if old_version.is_latest: - # This is not supposed to happen, as versions were sorted above, - # but leaving it here as a security measure in case version - # ordering changes for some reason. - raise Exception("Trying to delete latest version") - - if not old_version.key or not re.match( - r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/meta/thumbnail.png$", - old_version.key, - ): - raise RuntimeError( - f"Suspicious S3 file version deletion {old_version.key=} {old_version.id=}" - ) - # TODO: any way to batch those ? will probaby get slow on production - delete_version_permanently(old_version) - # TODO: audit ? take implementation from files_views.py:211 - - -@legacy_only -def purge_old_file_versions_legacy( - project: qfieldcloud.core.models.Project, -) -> None: # noqa: F821 - """ - Deletes old versions of all files in the given project. Will keep __3__ - versions for COMMUNITY user accounts, and __10__ versions for PRO user - accounts - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - - keep_count = project.owner_aware_storage_keep_versions - - logger.info(f"Cleaning up old files for {project} to {keep_count} versions") - - # Process file by file - for file in qfieldcloud.core.utils.get_project_files_with_versions(project.pk): - # Skip the newest N - old_versions_to_purge = sorted( - file.versions, key=lambda v: v.last_modified, reverse=True - )[keep_count:] - - # Debug print - logger.info( - f'Purging {len(old_versions_to_purge)} out of {len(file.versions)} old versions for "{file.latest.name}"...' - ) - - # Remove the N oldest - for old_version in old_versions_to_purge: - logger.info( - f'Purging {old_version.key=} {old_version.id=} as old version for "{file.latest.name}"...' - ) - - if old_version.is_latest: - # This is not supposed to happen, as versions were sorted above, - # but leaving it here as a security measure in case version - # ordering changes for some reason. - raise Exception("Trying to delete latest version") - - if not old_version.key or not re.match( - r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/.+$", old_version.key - ): - raise RuntimeError( - f"Suspicious S3 file version deletion {old_version.key=} {old_version.id=}" - ) - # TODO: any way to batch those ? will probaby get slow on production - delete_version_permanently(old_version) - # TODO: audit ? take implementation from files_views.py:211 - - # Update the project size - project.save(recompute_storage=True) - - -def upload_file(file: IO, key: str): - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - bucket = qfieldcloud.core.utils.get_s3_bucket() - bucket.upload_fileobj( - file, - key, - ) - return key - - -@legacy_only -def upload_project_file( - project: qfieldcloud.core.models.Project, file: IO, filename: str -) -> str: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - key = f"projects/{project.id}/files/{filename}" - bucket = qfieldcloud.core.utils.get_s3_bucket() - bucket.upload_fileobj( - file, - key, - ) - return key - - -def delete_all_project_files_permanently(project_id: str) -> None: - """Deletes all project files permanently. - - Args: - project_id: the project which files shall be deleted. Note that the `project_id` might be a of a already deleted project which files are still dangling around. - - Raises: - RuntimeError: if the produced Object Storage key to delete is not in the right format - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - prefix = f"projects/{project_id}/" - - if not re.match(r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/$", prefix): - raise RuntimeError( - f"Suspicious S3 deletion of all project files with {prefix=}" - ) - - _delete_by_prefix_permanently(prefix) - - -@legacy_only -def delete_project_file_permanently( - project: qfieldcloud.core.models.Project, filename: str -): # noqa: F821 - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - logger.info(f"Requested delete (permanent) of project file {filename=}") - - file = qfieldcloud.core.utils.get_project_file_with_versions( - str(project.id), filename - ) - - if not file: - raise Exception( - f"No file with such name in the given project found {filename=}" - ) - - if not re.match(r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/.+$", file.latest.key): - raise RuntimeError(f"Suspicious S3 file deletion {file.latest.key=}") - - # NOTE the file operations depend on HTTP calls to the S3 storage and they might fail, - # we need to choose source of truth between DB and S3. - # For now the source of truth is what is on the S3 storage, - # because we do most of our file operations directly by calling the S3 API. - # 1. S3 storage modification. If it fails, it will cancel the transaction - # and do not update anything in the database. - # We assume S3 storage is transactional, while it might not be true. - # 2. DB modification. If it fails, the states betewen DB and S3 mismatch, - # but can be easyly synced from the S3 to DB with a manual script. - with transaction.atomic(): - _delete_by_key_permanently(file.latest.key) - - update_fields = ["data_last_updated_at"] - - now = timezone.now() - project.data_last_updated_at = now - - if is_admin_restricted_file(filename, project.the_qgis_file_name): - update_fields.append("restricted_data_last_updated_at") - project.restricted_data_last_updated_at = now - - if qfieldcloud.core.utils.is_the_qgis_file(filename): - update_fields.append("the_qgis_file_name") - project.the_qgis_file_name = None - - project.save(update_fields=update_fields, recompute_storage=True) - - # NOTE force audits to be required when deleting files - audit( - project, - LogEntry.Action.DELETE, - changes={f"{filename} ALL": [file.latest.e_tag, None]}, - ) - - -@legacy_only -def delete_project_file_version_permanently( - project: qfieldcloud.core.models.Project, - filename: str, - version_id: str, - include_older: bool = False, -) -> list[qfieldcloud.core.utils.S3ObjectVersion]: - """Deletes a specific version of given file. - - Args: - project: project the file belongs to - filename: filename the version belongs to - version_id: version id to delete - include_older: when True, versions older than the passed `version` will also be deleted. If the version_id is the latest version of a file, this parameter will treated as False. Defaults to False. - - Returns: - the number of versions deleted - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - project_id = str(project.id) - file = qfieldcloud.core.utils.get_project_file_with_versions(project_id, filename) - - if not file: - raise Exception( - f"No file with such name in the given project found {filename=} {version_id=}" - ) - - if file.latest.id == version_id: - include_older = False - - if len(file.versions) == 1: - raise RuntimeError( - "Forbidded attempt to delete a specific file version which is the only file version available." - ) - - versions_latest_first = list(reversed(file.versions)) - versions_to_delete: list[qfieldcloud.core.utils.S3ObjectVersion] = [] - - for file_version in versions_latest_first: - if file_version.id == version_id: - versions_to_delete.append(file_version) - - if include_older: - continue - else: - break - - if versions_to_delete: - assert include_older, ( - "We should continue to loop only if `include_older` is True" - ) - assert versions_to_delete[-1].last_modified > file_version.last_modified, ( - "Assert the other versions are really older than the requested one" - ) - - versions_to_delete.append(file_version) - - with transaction.atomic(): - update_fields = ["data_last_updated_at"] - for file_version in versions_to_delete: - if ( - not re.match( - r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/.+$", - file_version._data.key, - ) - or not file_version.id - ): - raise RuntimeError( - f"Suspicious S3 file version deletion {filename=} {version_id=} {include_older=}" - ) - - audit_suffix = file_version.display - - audit( - project, - LogEntry.Action.DELETE, - changes={f"{filename} {audit_suffix}": [file_version.e_tag, None]}, - ) - - delete_version_permanently(file_version) - - now = timezone.now() - project.data_last_updated_at = now - - if is_admin_restricted_file(filename, project.the_qgis_file_name): - update_fields.append("restricted_data_last_updated_at") - project.restricted_data_last_updated_at = now - - project.save(update_fields=update_fields, recompute_storage=True) - - return versions_to_delete - - -@legacy_only -def get_stored_package_ids(project: qfieldcloud.core.models.Project) -> set[str]: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - project_id = project.id - bucket = qfieldcloud.core.utils.get_s3_bucket() - prefix = f"projects/{project_id}/packages/" - root_path = PurePath(prefix) - package_ids = set() - - for file in bucket.objects.filter(Prefix=prefix): - file_path = PurePath(file.key) - parts = file_path.relative_to(root_path).parts - package_ids.add(parts[0]) - - return package_ids - - -@legacy_only -def delete_stored_package( - project: qfieldcloud.core.models.Project, package_id: str -) -> None: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - project_id = str(project.id) - prefix = f"projects/{project_id}/packages/{package_id}/" - - if not re.match( - # e.g. "projects/878039c4-b945-4356-a44e-a908fd3f2263/packages/633cd4f7-db14-4e6e-9b2b-c0ce98f9d338/" - r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/packages/[\w]{8}(-[\w]{4}){3}-[\w]{12}/$", - prefix, - ): - raise RuntimeError( - f"Suspicious S3 deletion on stored project package {project_id=} {package_id=}" - ) - - _delete_by_prefix_permanently(prefix) - - -@legacy_only -def get_project_file_storage_in_bytes(project: qfieldcloud.core.models.Project) -> int: - """Calculates the project files storage in bytes, including their versions. - - WARNING This function can be quite slow on projects with thousands of files. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - project_id = str(project.id) - bucket = qfieldcloud.core.utils.get_s3_bucket() - total_bytes = 0 - prefix = f"projects/{project_id}/files/" - - logger.info(f"Project file storage size requested for {project_id=}") - - if not re.match(r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/files/$", prefix): - raise RuntimeError( - f"Suspicious S3 calculation of all project files with {prefix=}" - ) - - for version in bucket.object_versions.filter(Prefix=prefix): - total_bytes += version.size or 0 - - return total_bytes - - -def calculate_checksums( - content: ContentFile, alrgorithms: tuple[str, ...], blocksize: int = 65536 -) -> tuple[bytes, ...]: - """Calculates checksums on given file for given algorithms.""" - hashers = [] - for alrgorithm in alrgorithms: - hashers.append(getattr(hashlib, alrgorithm)()) - - for chunk in content.chunks(blocksize): - for hasher in hashers: - hasher.update(chunk) - - content.seek(0) - - checksums = [] - for hasher in hashers: - checksums.append(hasher.digest()) - - return tuple(checksums) diff --git a/docker-app/qfieldcloud/core/views/files_views.py b/docker-app/qfieldcloud/core/views/files_views.py deleted file mode 100644 index 9d7e64197..000000000 --- a/docker-app/qfieldcloud/core/views/files_views.py +++ /dev/null @@ -1,493 +0,0 @@ -""" -Todo: - * Delete with QF-4963 Drop support for legacy storage -""" - -import copy -import io -import logging -from pathlib import PurePath -from traceback import print_stack - -import qfieldcloud.core.utils2 as utils2 -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist -from django.db import transaction -from django.utils import timezone -from drf_spectacular.utils import ( - OpenApiParameter, - OpenApiTypes, - extend_schema, - extend_schema_view, -) -from qfieldcloud.core import exceptions, permissions_utils, utils -from qfieldcloud.core.models import Job, ProcessProjectfileJob, Project -from qfieldcloud.core.serializers import FileWithVersionsSerializer -from qfieldcloud.core.utils import S3ObjectVersion, get_project_file_with_versions -from qfieldcloud.core.utils2.audit import LogEntry, audit -from qfieldcloud.core.utils2.sentry import report_serialization_diff_to_sentry -from qfieldcloud.core.utils2.storage import ( - get_attachment_dir_prefix, - purge_old_file_versions_legacy, -) -from qfieldcloud.filestorage.utils import is_admin_restricted_file -from rest_framework import permissions, serializers, status, views -from rest_framework.exceptions import NotFound -from rest_framework.parsers import DataAndFiles, MultiPartParser -from rest_framework.request import Request -from rest_framework.response import Response - -logger = logging.getLogger(__name__) - - -class ListFilesViewPermissions(permissions.BasePermission): - def has_permission(self, request, view): - if "projectid" not in request.parser_context["kwargs"]: - return False - - projectid = request.parser_context["kwargs"]["projectid"] - project = Project.objects.get(id=projectid) - - return permissions_utils.can_read_files(request.user, project) - - -@extend_schema_view( - get=extend_schema( - description="Get all the project's file versions", - responses={200: serializers.ListSerializer(child=FileWithVersionsSerializer())}, - parameters=[ - OpenApiParameter( - name="skip_metadata", - type=OpenApiTypes.INT, - required=False, - default=0, - enum=[1, 0], - description="Skip obtaining file metadata (e.g. `sha256`). Makes responses much faster. In the future `skip_metadata=1` might be default behaviour.", - ), - ], - ), -) -class ListFilesView(views.APIView): - # TODO: docstring - - permission_classes = [permissions.IsAuthenticated, ListFilesViewPermissions] - - def get(self, request: Request, projectid: str) -> Response: - try: - project = Project.objects.get(id=projectid) - except ObjectDoesNotExist: - raise NotFound(detail=projectid) - - bucket = utils.get_s3_bucket() - prefix = f"projects/{projectid}/files/" - - files = {} - for version in bucket.object_versions.filter(Prefix=prefix): - # Created the dict entry if doesn't exist - if version.key not in files: - files[version.key] = {"versions": []} - - path = PurePath(version.key) - filename = str(path.relative_to(*path.parts[:3])) - last_modified = version.last_modified.strftime( - settings.QFIELDCLOUD_STORAGE_DT_LAST_MODIFIED_FORMAT - ) - # NOTE ETag is a MD5. But for the multipart uploaded files, the MD5 is computed from the concatenation of the MD5s of each uploaded part. - # TODO make sure when file metadata is in the DB (QF-2760), this is a real md5sum of the current file. - md5sum = version.e_tag.replace('"', "") - - version_data = { - "size": version.size, - "md5sum": md5sum, - "version_id": version.version_id, - "last_modified": last_modified, - "is_latest": version.is_latest, - "display": S3ObjectVersion(version.key, version).display, - } - - # NOTE Some clients (e.g. QFieldSync) are still requiring the `sha256` key to check whether the files needs to be reuploaded. - # Since we do not have control on these old client versions, we need to keep the API backward compatible for some time and assume `skip_metadata=0` by default. - skip_metadata_param = request.GET.get("skip_metadata", "0") - if skip_metadata_param == "0": - skip_metadata = False - else: - skip_metadata = bool(skip_metadata_param) - - if not skip_metadata: - head = version.head() - # We cannot be sure of the metadata's first letter case - # https://github.com/boto/boto3/issues/1709 - metadata = head["Metadata"] - if "sha256sum" in metadata: - sha256sum = metadata["sha256sum"] - else: - sha256sum = metadata["Sha256sum"] - - version_data["sha256"] = sha256sum - - if version.is_latest: - is_attachment = get_attachment_dir_prefix(project, filename) != "" - - files[version.key]["name"] = filename - files[version.key]["size"] = version.size - files[version.key]["md5sum"] = md5sum - files[version.key]["last_modified"] = last_modified - files[version.key]["is_attachment"] = is_attachment - - if not skip_metadata: - files[version.key]["sha256"] = sha256sum - - files[version.key]["versions"].append(version_data) - - result_list = [files[key] for key in files] - return Response(result_list) - - -@extend_schema_view( - get=extend_schema( - description="Get the project's file metadata", - responses={200: FileWithVersionsSerializer()}, - ), -) -class LegacyFileMetadataView(views.APIView): - permission_classes = [permissions.IsAuthenticated, ListFilesViewPermissions] - - def get(self, request: Request, projectid: str, filename: str) -> Response: - try: - project = Project.objects.get(id=projectid) - except ObjectDoesNotExist: - raise NotFound(detail=projectid) - - file_obj = get_project_file_with_versions(projectid, filename) - - if not file_obj: - raise NotFound(detail=filename) - - versions_data = [] - file_data: dict = {} - - for version in file_obj.versions: - last_modified = version.last_modified.strftime( - settings.QFIELDCLOUD_STORAGE_DT_LAST_MODIFIED_FORMAT - ) - md5sum = version.md5sum - - head = version._data.head() - # We cannot be sure of the metadata's first letter case - # https://github.com/boto/boto3/issues/1709 - metadata = head["Metadata"] - sha256sum = metadata.get("sha256sum") or metadata.get("Sha256sum") - - version_data = { - "size": version.size, - "md5sum": md5sum, - "version_id": version.id, - "last_modified": last_modified, - "is_latest": version.is_latest, - "display": version.display, - "sha256": sha256sum, - } - - versions_data.append(version_data) - - if version.is_latest: - is_attachment = get_attachment_dir_prefix(project, filename) != "" - - file_data["name"] = filename - file_data["size"] = version.size - file_data["md5sum"] = md5sum - file_data["last_modified"] = last_modified - file_data["is_attachment"] = is_attachment - file_data["sha256"] = sha256sum - - file_data["versions"] = versions_data - return Response(file_data) - - -class DownloadPushDeleteFileViewPermissions(permissions.BasePermission): - def has_permission(self, request, view): - if "projectid" not in request.parser_context["kwargs"]: - return False - - projectid = request.parser_context["kwargs"]["projectid"] - project = Project.objects.get(id=projectid) - user = request.user - - if request.method == "GET": - return permissions_utils.can_read_files(user, project) - elif request.method == "DELETE": - return permissions_utils.can_delete_files(user, project) - elif request.method == "POST": - return permissions_utils.can_create_files(user, project) - - return False - - -class QfcMultiPartSerializer(MultiPartParser): - errors: list[str] = [] - - # QF-2540 - def parse(self, stream, media_type=None, parser_context=None) -> DataAndFiles: - """Substitute to MultiPartParser for debugging `EmptyContentError`""" - parsed: DataAndFiles = super().parse(stream, media_type, parser_context) - - if "file" not in parsed.files or not parsed.files["file"]: - self.errors.append( - f"QfcMultiPartParser was able to obtain `DataAndFiles` from the request's input stream, but `MultiValueDict` either lacks a `file` key or a value at `file`! parser_context: {parser_context}. `EmptyContentError` expected." - ) - - return parsed - - -@extend_schema_view( - get=extend_schema( - description="Download a file from a project", - responses={ - (200, "*/*"): OpenApiTypes.BINARY, - }, - ), - post=extend_schema( - description="Upload a file to the project", - parameters=[ - OpenApiParameter( - name="file", - type=OpenApiTypes.BINARY, - location=OpenApiParameter.QUERY, - required=True, - description="File to be uploaded", - ) - ], - ), - delete=extend_schema(description="Delete a file from a project"), -) -class DownloadPushDeleteFileView(views.APIView): - # TODO: swagger doc - # TODO: docstring - parser_classes = [QfcMultiPartSerializer] - permission_classes = [ - permissions.IsAuthenticated, - DownloadPushDeleteFileViewPermissions, - ] - - def get(self, request, projectid, filename): - Project.objects.get(id=projectid) - - version = None - if "version" in self.request.query_params: - version = self.request.query_params["version"] - - key = utils.safe_join(f"projects/{projectid}/files/", filename) - return utils2.storage.file_response( - request, - key, - expires=600, - version=version, - as_attachment=True, - ) - - # TODO refactor this function by moving the actual upload and Project model updates to library function outside the view - def post(self, request, projectid, filename, format=None): - if len(request.FILES.getlist("file")) > 1: - raise exceptions.MultipleContentsError() - - # QF-2540 - # Getting traceback in case the traceback provided by Sentry is too short - # Add post-serialization keys for diff-ing with pre-serialization keys - if "file" not in request.data and not request.FILES.getlist("file"): - if "file" not in request.data: - logger.warning( - 'The key "file" was not found in `request.data`. Sending report to Sentry.' - ) - - if not request.FILES.getlist("file"): - logger.warning( - 'The key "file" occurs in `request.data` but maps to an empty list. Sending report to Sentry.' - ) - - callstack_buffer = io.StringIO() - print_stack(limit=50, file=callstack_buffer) - - request_attributes = { - "data": str(copy.copy(self.request.data).keys()), - "files": str(self.request.FILES.keys()), - "meta": str(self.request.META), - } - - # QF-2540 - report_serialization_diff_to_sentry( - # using the 'X-Request-Id' added to the request by RequestIDMiddleware - name=f"{request.META.get('X-Request-Id')}_{projectid}", - pre_serialization=request.attached_keys, - post_serialization=",".join(QfcMultiPartSerializer.errors) - + str(request_attributes), - buffer=callstack_buffer, - body_stream=getattr(request, "body_stream", None), - ) - raise exceptions.EmptyContentError() - - # get project from request or db - project = Project.objects.get(id=projectid) - - is_the_qgis_file = utils.is_the_qgis_file(filename) - - if is_the_qgis_file and project.is_shared_datasets_project: - raise exceptions.QGISProjectFileNotAllowedError( - "QGIS project files are not allowed in shared datasets projects." - ) - - # check if the project restricts qgs/qgz file modification to admins - if is_the_qgis_file and not permissions_utils.can_modify_qgis_projectfile( - request.user, project - ): - raise exceptions.RestrictedProjectModificationError( - "The project restricts modification of the QGIS project file to managers and administrators." - ) - - # check only one qgs/qgz file per project - if ( - is_the_qgis_file - and project.has_the_qgis_file - and PurePath(filename) != PurePath(project.the_qgis_file_name) - ): - raise exceptions.MultipleProjectsError( - "Only one QGIS project per project allowed" - ) - - request_file = request.FILES.get("file") - - if hasattr(request, "auth") and hasattr(request.auth, "client_type"): - client_type = request.auth.client_type - else: - client_type = request.session.get("client_type") - - permissions_utils.check_can_upload_file(project, client_type, request_file.size) - - old_object = get_project_file_with_versions(project.id, filename) - sha256sum = utils.get_sha256(request_file) - bucket = utils.get_s3_bucket() - - key = utils.safe_join(f"projects/{projectid}/files/", filename) - metadata = {"Sha256sum": sha256sum} - - bucket.upload_fileobj(request_file, key, ExtraArgs={"Metadata": metadata}) - - new_object = get_project_file_with_versions(project.id, filename) - - assert new_object - - with transaction.atomic(): - # we only enter a transaction after the file is uploaded above because we do not - # want to lock the project row for way too long. If we reselect for update the - # project and update it now, it guarantees there will be no other file upload editing - # the same project row. - project = Project.objects.select_for_update().get(id=projectid) - update_fields = ["data_last_updated_at"] - - if get_attachment_dir_prefix(project, filename) == "" and ( - is_the_qgis_file or project.has_the_qgis_file - ): - if is_the_qgis_file: - project.the_qgis_file_name = filename - update_fields.append("the_qgis_file_name") - - running_jobs = ProcessProjectfileJob.objects.filter( - project=project, - created_by=self.request.user, - status__in=[ - Job.Status.PENDING, - Job.Status.QUEUED, - Job.Status.STARTED, - ], - ) - - if not running_jobs.exists(): - ProcessProjectfileJob.objects.create( - project=project, created_by=self.request.user - ) - - now = timezone.now() - project.data_last_updated_at = now - - if is_admin_restricted_file(filename, project.the_qgis_file_name): - update_fields.append("restricted_data_last_updated_at") - project.restricted_data_last_updated_at = now - - project.save(update_fields=update_fields, recompute_storage=True) - - if old_object: - audit( - project, - LogEntry.Action.UPDATE, - changes={filename: [old_object.latest.e_tag, new_object.latest.e_tag]}, - ) - else: - audit( - project, - LogEntry.Action.CREATE, - changes={filename: [None, new_object.latest.e_tag]}, - ) - - # Delete the old file versions - purge_old_file_versions_legacy(project) - - return Response(status=status.HTTP_201_CREATED) - - @transaction.atomic() - def delete(self, request, projectid, filename): - project = Project.objects.select_for_update().get(id=projectid) - version_id = request.GET.get("version", request.headers.get("x-file-version")) - - if version_id: - utils2.storage.delete_project_file_version_permanently( - project, filename, version_id, False - ) - else: - utils2.storage.delete_project_file_permanently(project, filename) - - return Response(status=status.HTTP_200_OK) - - -@extend_schema_view( - get=extend_schema( - description="Download the metadata of a project's file", - responses={ - (200, "*/*"): OpenApiTypes.BINARY, - }, - ) -) -class ProjectMetafilesView(views.APIView): - parser_classes = [MultiPartParser] - permission_classes = [ - permissions.IsAuthenticated, - DownloadPushDeleteFileViewPermissions, - ] - - def get(self, request, projectid, filename): - key = utils.safe_join(f"projects/{projectid}/meta/", filename) - return utils2.storage.file_response(request, key) - - -@extend_schema_view( - get=extend_schema( - description="Download a public file, e.g. user avatar.", - responses={ - (200, "*/*"): OpenApiTypes.BINARY, - }, - ) -) -class PublicFilesView(views.APIView): - parser_classes = [MultiPartParser] - permission_classes = [] - - def get(self, request, filename): - return utils2.storage.file_response(request, filename) - - -@extend_schema(exclude=True) -class AdminDownloadPushDeleteFileView(DownloadPushDeleteFileView): - """Allowing `DownloadPushDeleteFileView` to be excluded from the OpenAPI schema documentation""" - - -@extend_schema(exclude=True) -class AdminListFilesViews(ListFilesView): - """Allowing `ListFilesView` to be excluded from the OpenAPI schema documentation""" diff --git a/docker-app/qfieldcloud/core/views/package_views.py b/docker-app/qfieldcloud/core/views/package_views.py index 237698667..6b23052e0 100644 --- a/docker-app/qfieldcloud/core/views/package_views.py +++ b/docker-app/qfieldcloud/core/views/package_views.py @@ -436,72 +436,38 @@ def post( @csrf_exempt def compatibility_latest_package_view(request: Request, *args, **kwargs) -> Response: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ project_id: UUID = kwargs["project_id"] - project = get_object_or_404(Project, id=project_id) - - if project.uses_legacy_storage: - logger.debug( - f"Project {project_id=} will be using the legacy package file management." - ) - - return LegacyLatestPackageView.as_view()(request, *args, **kwargs) - else: - logger.debug( - f"Project {project_id=} will be using the regular package file management." - ) + _project = get_object_or_404(Project, id=project_id) + logger.debug( + f"Project {project_id=} will be using the regular package file management." + ) - return LatestPackageView.as_view()(request, *args, **kwargs) + return LatestPackageView.as_view()(request, *args, **kwargs) @csrf_exempt def compatibility_package_download_files_view( request: Request, *args, **kwargs ) -> Response: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ project_id: UUID = kwargs["project_id"] - project = get_object_or_404(Project, id=project_id) - - if project.uses_legacy_storage: - logger.debug( - f"Project {project_id=} will be using the legacy package file management." - ) + _project = get_object_or_404(Project, id=project_id) - return LegacyLatestPackageDownloadFilesView.as_view()(request, *args, **kwargs) - else: - logger.debug( - f"Project {project_id=} will be using the regular package file management." - ) + logger.debug( + f"Project {project_id=} will be using the regular package file management." + ) - return LatestPackageDownloadFilesView.as_view()(request, *args, **kwargs) + return LatestPackageDownloadFilesView.as_view()(request, *args, **kwargs) @csrf_exempt def compatibility_package_upload_files_view( request: Request, *args, **kwargs ) -> Response: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ project_id: UUID = kwargs["project_id"] - project = get_object_or_404(Project, id=project_id) - - if project.uses_legacy_storage: - logger.debug( - f"Project {project_id=} will be using the legacy package file management." - ) + _project = get_object_or_404(Project, id=project_id) - return LegacyPackageUploadFilesView.as_view()(request, *args, **kwargs) - else: - logger.debug( - f"Project {project_id=} will be using the regular package file management." - ) + logger.debug( + f"Project {project_id=} will be using the regular package file management." + ) - return PackageUploadFilesView.as_view()(request, *args, **kwargs) + return PackageUploadFilesView.as_view()(request, *args, **kwargs) diff --git a/docker-app/qfieldcloud/core/views/projects_views.py b/docker-app/qfieldcloud/core/views/projects_views.py index d43f4bd15..dd9c45b22 100644 --- a/docker-app/qfieldcloud/core/views/projects_views.py +++ b/docker-app/qfieldcloud/core/views/projects_views.py @@ -11,7 +11,6 @@ from qfieldcloud.core.filters import ProjectFilterSet from qfieldcloud.core.models import Project, ProjectQueryset from qfieldcloud.core.serializers import ProjectSerializer -from qfieldcloud.core.utils2 import storage from qfieldcloud.subscription.exceptions import QuotaError from rest_framework import filters as drf_filters from rest_framework import generics, permissions, viewsets @@ -141,10 +140,7 @@ def perform_update(self, serializer: ProjectSerializer) -> None: def destroy(self, request, projectid): # Delete files from storage - project = Project.objects.get(id=projectid) - - if project.uses_legacy_storage: - storage.delete_all_project_files_permanently(projectid) + _project = Project.objects.get(id=projectid) return super().destroy(request, projectid) From b829143dc4329a19e150b39faa7a16f35a56b6a6 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Wed, 11 Feb 2026 05:19:32 +0100 Subject: [PATCH 03/22] chore: get rid of legacy storage in the `filestorage` module --- .../filestorage/tests/test_files_api.py | 5 +- .../filestorage/tests/test_storage_usage.py | 21 ---- docker-app/qfieldcloud/filestorage/views.py | 100 +++--------------- 3 files changed, 15 insertions(+), 111 deletions(-) diff --git a/docker-app/qfieldcloud/filestorage/tests/test_files_api.py b/docker-app/qfieldcloud/filestorage/tests/test_files_api.py index 6370e616c..fc533db40 100644 --- a/docker-app/qfieldcloud/filestorage/tests/test_files_api.py +++ b/docker-app/qfieldcloud/filestorage/tests/test_files_api.py @@ -50,10 +50,7 @@ def setUp(self): def assertFileUploaded( self, user: User, project: Project, filename: str, content: IO ) -> HttpResponse | Response: - if project.uses_legacy_storage: - return self._assertFileUploadedLegacy(user, project, filename, content) - else: - return self._assertFileUploaded(user, project, filename, content) + return self._assertFileUploaded(user, project, filename, content) def _assertFileUploadedLegacy( self, user: User, project: Project, filename: str, content: IO diff --git a/docker-app/qfieldcloud/filestorage/tests/test_storage_usage.py b/docker-app/qfieldcloud/filestorage/tests/test_storage_usage.py index 8814173a0..63573b29e 100644 --- a/docker-app/qfieldcloud/filestorage/tests/test_storage_usage.py +++ b/docker-app/qfieldcloud/filestorage/tests/test_storage_usage.py @@ -57,30 +57,9 @@ def test_upload_file_succeeds(self): self.assertEqual(self.p1.file_storage_bytes, 13) self.assertEqual(self.u1.useraccount.storage_used_bytes, 13) - # 3) creating a second project with legacy storage backend - # TODO: Delete with QF-4963 Drop support for legacy storage - p2 = Project.objects.create( - owner=self.u1, - name="p2", - file_storage="legacy_storage", - ) - - # 4) adding a file in the legacy storage backend - response = self._upload_file(self.u1, p2, "file.name", StringIO("Hello3!")) - - self.p1.refresh_from_db() - p2.refresh_from_db() - self.u1.useraccount.refresh_from_db() - - self.assertEqual(response.status_code, status.HTTP_201_CREATED) # p1 checks self.assertEqual(self.p1.project_files.count(), 1) self.assertEqual(self.p1.get_file("file.name").versions.count(), 2) self.assertEqual(self.p1.file_storage_bytes, 13) - # TODO: Change the number of files with QF-4963 Drop support for legacy storage - # p2 checks - # since the project is in the legacy storage, no `File`` object is created. - self.assertEqual(p2.project_files.count(), 0) - self.assertEqual(p2.file_storage_bytes, 7) self.assertEqual(self.u1.useraccount.storage_used_bytes, 20) diff --git a/docker-app/qfieldcloud/filestorage/views.py b/docker-app/qfieldcloud/filestorage/views.py index be738ff8b..fdc80e904 100644 --- a/docker-app/qfieldcloud/filestorage/views.py +++ b/docker-app/qfieldcloud/filestorage/views.py @@ -28,18 +28,6 @@ Project, UserAccount, ) -from qfieldcloud.core.views.files_views import ( - DownloadPushDeleteFileView as LegacyFileCrudView, -) -from qfieldcloud.core.views.files_views import ( - LegacyFileMetadataView, -) -from qfieldcloud.core.views.files_views import ( - ListFilesView as LegacyFileListView, -) -from qfieldcloud.core.views.files_views import ( - ProjectMetafilesView as LegacyProjectMetaFileReadView, -) from qfieldcloud.filestorage.models import ( File, ) @@ -275,135 +263,75 @@ def get( def compatibility_file_list_view( request: Request, *args, **kwargs ) -> Response | HttpResponse: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ # let's assume that `kwargs["project_id"]` will no throw a `KeyError` project_id: UUID = kwargs["project_id"] view_kwargs = kwargs.pop("view_kwargs", {}) try: - project = Project.objects.get(id=project_id) + _project = Project.objects.get(id=project_id) except Project.DoesNotExist: # if the project does not exist, we just fallback to the new view, which will return JSON formatted 404 later return FileListView.as_view(**view_kwargs)(request, *args, **kwargs) - if project.uses_legacy_storage: - # rename the `project_id` to previously used `projectid`, so we don't change anything in the legacy code - kwargs["projectid"] = kwargs.pop("project_id") - - logger.debug(f"Project {project_id=} will be using the legacy file management.") - - return LegacyFileListView.as_view(**view_kwargs)(request, *args, **kwargs) - else: - logger.debug( - f"Project {project_id=} will be using the regular file management." - ) + logger.debug(f"Project {project_id=} will be using the regular file management.") - return FileListView.as_view(**view_kwargs)(request, *args, **kwargs) + return FileListView.as_view(**view_kwargs)(request, *args, **kwargs) @csrf_exempt def compatibility_file_metadata_view( request: Request, *args, **kwargs ) -> Response | HttpResponse: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ # let's assume that `kwargs["project_id"]` will no throw a `KeyError` project_id: UUID = kwargs["project_id"] view_kwargs = kwargs.pop("view_kwargs", {}) try: - project = Project.objects.get(id=project_id) + _project = Project.objects.get(id=project_id) except Project.DoesNotExist: # if the project does not exist, we just fallback to the new view, which will return JSON formatted 404 later return FileMetadataView.as_view(**view_kwargs)(request, *args, **kwargs) - if project.uses_legacy_storage: - # rename the `project_id` to previously used `projectid`, so we don't change anything in the legacy code - kwargs["projectid"] = kwargs.pop("project_id") - - logger.debug(f"Project {project_id=} will be using the legacy file management.") + logger.debug(f"Project {project_id=} will be using the regular file management.") - return LegacyFileMetadataView.as_view(**view_kwargs)(request, *args, **kwargs) - else: - logger.debug( - f"Project {project_id=} will be using the regular file management." - ) - - return FileMetadataView.as_view(**view_kwargs)(request, *args, **kwargs) + return FileMetadataView.as_view(**view_kwargs)(request, *args, **kwargs) @csrf_exempt def compatibility_file_crud_view( request: Request, *args, **kwargs ) -> Response | HttpResponse: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ # let's assume that `kwargs["project_id"]` will no throw a `KeyError` project_id: UUID = kwargs["project_id"] view_kwargs = kwargs.pop("view_kwargs", {}) try: - project = Project.objects.get(id=project_id) + _project = Project.objects.get(id=project_id) except Project.DoesNotExist: # if the project does not exist, we just fallback to the new view, which will return JSON formatted 404 later return FileCrudView.as_view(**view_kwargs)(request, *args, **kwargs) - if project.uses_legacy_storage: - # rename the `project_id` to previously used `projectid`, so we don't change anything in the legacy code - kwargs["projectid"] = kwargs.pop("project_id") + logger.debug(f"Project {project_id=} will be using the regular file management.") - logger.debug(f"Project {project_id=} will be using the legacy file management.") - - return LegacyFileCrudView.as_view(**view_kwargs)(request, *args, **kwargs) - else: - logger.debug( - f"Project {project_id=} will be using the regular file management." - ) - - return FileCrudView.as_view(**view_kwargs)(request, *args, **kwargs) + return FileCrudView.as_view(**view_kwargs)(request, *args, **kwargs) @csrf_exempt def compatibility_project_meta_file_read_view( request: Request, *args, **kwargs ) -> Response | HttpResponse: - """ - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ # let's assume that `kwargs["project_id"]` will no throw a `KeyError` project_id: UUID = kwargs["project_id"] view_kwargs = kwargs.pop("view_kwargs", {}) try: - project = Project.objects.get(id=project_id) + _project = Project.objects.get(id=project_id) except Project.DoesNotExist: # if the project does not exist, we just fallback to the new view, which will return JSON formatted 404 later return ProjectMetaFileReadView.as_view(**view_kwargs)(request, *args, **kwargs) - if project.uses_legacy_storage: - # rename the `project_id` to previously used `projectid`, so we don't change anything in the legacy code - kwargs["projectid"] = kwargs.pop("project_id") - # hardcode the thumbnail file name - kwargs["filename"] = "thumbnail.png" - - logger.debug( - f"Project {project_id=} will be using the legacy file management for meta files." - ) - - return LegacyProjectMetaFileReadView.as_view(**view_kwargs)( - request, *args, **kwargs - ) - else: - logger.debug( - f"Project {project_id=} will be using the regular file management for meta files." - ) + logger.debug( + f"Project {project_id=} will be using the regular file management for meta files." + ) - return ProjectMetaFileReadView.as_view(**view_kwargs)(request, *args, **kwargs) + return ProjectMetaFileReadView.as_view(**view_kwargs)(request, *args, **kwargs) From 1c99eb485fbe55012375963c0f08dd72c29ec497 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Wed, 11 Feb 2026 05:19:47 +0100 Subject: [PATCH 04/22] chore: get rid of legacy storage in the `subscription` module --- .../subscription/tests/test_job_creation.py | 20 +++++++------------ .../subscription/tests/test_package.py | 18 +++-------------- 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/docker-app/qfieldcloud/subscription/tests/test_job_creation.py b/docker-app/qfieldcloud/subscription/tests/test_job_creation.py index 3ceb1379f..be6ee3f63 100644 --- a/docker-app/qfieldcloud/subscription/tests/test_job_creation.py +++ b/docker-app/qfieldcloud/subscription/tests/test_job_creation.py @@ -95,19 +95,13 @@ def test_create_job_if_project_owner_is_over_quota(self): plan = self.user1.useraccount.current_subscription.plan more_bytes_than_plan = (plan.storage_mb * 1000 * 1000) + 1 - # TODO Delete with QF-4963 Drop support for legacy storage - if self.project1.uses_legacy_storage: - # Create a project that uses all the storage - self.project1.file_storage_bytes = more_bytes_than_plan - self.project1.save() - else: - FileVersion.objects.add_version( - project=self.project1, - filename="bigfile.name", - content=ContentFile(b"x" * more_bytes_than_plan, "dummy.name"), - file_type=File.FileType.PROJECT_FILE, - uploaded_by=self.user1, - ) + FileVersion.objects.add_version( + project=self.project1, + filename="bigfile.name", + content=ContentFile(b"x" * more_bytes_than_plan, "dummy.name"), + file_type=File.FileType.PROJECT_FILE, + uploaded_by=self.user1, + ) self.check_cannot_create_jobs(QuotaError) self.check_can_update_existing_jobs() diff --git a/docker-app/qfieldcloud/subscription/tests/test_package.py b/docker-app/qfieldcloud/subscription/tests/test_package.py index a904db362..c25adacaf 100644 --- a/docker-app/qfieldcloud/subscription/tests/test_package.py +++ b/docker-app/qfieldcloud/subscription/tests/test_package.py @@ -639,21 +639,13 @@ def test_used_storage_changes_when_uploading_and_deleting_files_and_versions(sel storage_free_mb=0.6, ) - # TODO Delete with QF-4963 Drop support for legacy storage - if p1.uses_legacy_storage: - version = p1.legacy_files[0].versions[0] - else: - version = p1.project_files[0].versions.all().reverse()[0] + version = p1.project_files[0].versions.all().reverse()[0] response = self.client.delete( f"/api/v1/files/{p1.id}/file.name/?version={str(version.id)}", ) - # TODO Delete with QF-4963 Drop support for legacy storage - if p1.uses_legacy_storage: - self.assertEqual(response.status_code, status.HTTP_200_OK) - else: - self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) p1.save(recompute_storage=True) @@ -673,11 +665,7 @@ def test_used_storage_changes_when_uploading_and_deleting_files_and_versions(sel response = self.client.delete(f"/api/v1/files/{p1.id}/file.name/") - # TODO Delete with QF-4963 Drop support for legacy storage - if p1.uses_legacy_storage: - self.assertEqual(response.status_code, status.HTTP_200_OK) - else: - self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) p1.save(recompute_storage=True) From e5736ffc39437a6f132ce133b13e50b98ab6eaa7 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Wed, 11 Feb 2026 05:20:08 +0100 Subject: [PATCH 05/22] chore: get rid of legacy storage in the worker_wrapper --- docker-app/worker_wrapper/wrapper.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/docker-app/worker_wrapper/wrapper.py b/docker-app/worker_wrapper/wrapper.py index 5174ea68c..b928275f4 100644 --- a/docker-app/worker_wrapper/wrapper.py +++ b/docker-app/worker_wrapper/wrapper.py @@ -634,16 +634,7 @@ def after_docker_run(self) -> None: thumbnail_filename = self.shared_tempdir.joinpath("thumbnail.png") with open(thumbnail_filename, "rb") as f: - # TODO Delete with QF-4963 Drop support for legacy storage - if project.uses_legacy_storage: - legacy_thumbnail_uri = storage.upload_project_thumbail( - project, f, "image/png", "thumbnail" - ) - project.legacy_thumbnail_uri = ( - project.legacy_thumbnail_uri or legacy_thumbnail_uri - ) - else: - project.thumbnail = ContentFile(f.read(), "dummy_thumbnail_name.png") + project.thumbnail = ContentFile(f.read(), "dummy_thumbnail_name.png") project.save( update_fields=( @@ -653,8 +644,8 @@ def after_docker_run(self) -> None: ) ) - # for non-legacy storage, keep only one thumbnail version if so. - if not project.uses_legacy_storage and project.thumbnail: + # keep only one thumbnail version if so. + if project.thumbnail: storage.purge_previous_thumbnails_versions(project) def after_docker_exception(self) -> None: From 7f78a8dbb82e6c8422d066e800a59c270aef92cf Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Wed, 11 Feb 2026 05:20:48 +0100 Subject: [PATCH 06/22] chore: get rid of legacy storage in the CI test workflow --- .github/workflows/test.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e6671dded..e0e842392 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,7 +47,6 @@ jobs: - __flaky__ storage: - default - - legacy_storage continue-on-error: true steps: - name: Checkout repo @@ -68,17 +67,6 @@ jobs: cat <> .env STORAGES='{ - "legacy_storage": { - "BACKEND": "qfieldcloud.filestorage.backend.QfcS3Boto3Storage", - "OPTIONS": { - "access_key": "minioadmin", - "secret_key": "minioadmin", - "bucket_name": "qfieldcloud-local-legacy", - "region_name": "", - "endpoint_url": "http://172.17.0.1:8009" - }, - "QFC_IS_LEGACY": true - }, "webdav": { "BACKEND": "qfieldcloud.filestorage.backend.QfcWebDavStorage", "OPTIONS": { From e3327918f7f109adee2ef026a71cdd41235b3dd5 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Wed, 11 Feb 2026 08:18:12 +0100 Subject: [PATCH 07/22] fix: reintroduce storage utils functions --- docker-app/qfieldcloud/core/utils2/storage.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 docker-app/qfieldcloud/core/utils2/storage.py diff --git a/docker-app/qfieldcloud/core/utils2/storage.py b/docker-app/qfieldcloud/core/utils2/storage.py new file mode 100644 index 000000000..cda76fa49 --- /dev/null +++ b/docker-app/qfieldcloud/core/utils2/storage.py @@ -0,0 +1,20 @@ +import qfieldcloud.core.models + + +def get_attachment_dir_prefix( + project: qfieldcloud.core.models.Project, filename: str +) -> str: # noqa: F821 + """Returns the attachment dir where the file belongs to or empty string if it does not. + + Args: + project: project to check + filename: filename to check + + Returns: + the attachment dir or empty string if no match found + """ + for attachment_dir in project.attachment_dirs: + if filename.startswith(attachment_dir): + return attachment_dir + + return "" From 1b9c61d93e0734261aca4d2a28634c0dbd254f39 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Wed, 11 Feb 2026 08:18:56 +0100 Subject: [PATCH 08/22] chore: drop support of legacy storage in filestorage views --- docker-app/qfieldcloud/filestorage/views.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/docker-app/qfieldcloud/filestorage/views.py b/docker-app/qfieldcloud/filestorage/views.py index fdc80e904..c2aaf0c92 100644 --- a/docker-app/qfieldcloud/filestorage/views.py +++ b/docker-app/qfieldcloud/filestorage/views.py @@ -22,7 +22,6 @@ from qfieldcloud.core import ( pagination, permissions_utils, - utils2, ) from qfieldcloud.core.models import ( Project, @@ -250,13 +249,7 @@ def get( str(useraccount.avatar), ) else: - if useraccount.legacy_avatar_uri: - return utils2.storage.file_response( - request._request, - useraccount.legacy_avatar_uri, - ) - else: - return redirect(staticfiles_storage.url("logo.svg")) + return redirect(staticfiles_storage.url("logo.svg")) @csrf_exempt From 5232624de5ca53bd52112d54c8b5cb36c57f47db Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Wed, 11 Feb 2026 08:20:39 +0100 Subject: [PATCH 09/22] doc: add comment to show deprecation of avatar migration command --- .../filestorage/management/commands/migrateavatars.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-app/qfieldcloud/filestorage/management/commands/migrateavatars.py b/docker-app/qfieldcloud/filestorage/management/commands/migrateavatars.py index 5b230797c..a48bfa889 100644 --- a/docker-app/qfieldcloud/filestorage/management/commands/migrateavatars.py +++ b/docker-app/qfieldcloud/filestorage/management/commands/migrateavatars.py @@ -1,3 +1,8 @@ +""" +Migrate avatars from the legacy storage to the `default` storage. +NOTE: this command is legacy and deprecated, it should not be used anymore. It is only kept for historical reference and should be removed in the future. +""" + import logging from datetime import datetime From 5c41641ba71991cc8b9264b90bc1ae412eaec860 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Wed, 11 Feb 2026 08:41:23 +0100 Subject: [PATCH 10/22] fix: temporary reintroduce utils functions to prepare migration --- docker-app/qfieldcloud/core/utils.py | 291 ++++++++++++++++++++++++++- 1 file changed, 290 insertions(+), 1 deletion(-) diff --git a/docker-app/qfieldcloud/core/utils.py b/docker-app/qfieldcloud/core/utils.py index 474cceb8a..193fd18e9 100644 --- a/docker-app/qfieldcloud/core/utils.py +++ b/docker-app/qfieldcloud/core/utils.py @@ -3,15 +3,171 @@ import json import logging import os -from typing import IO +from datetime import datetime +from pathlib import PurePath +from typing import IO, NamedTuple +import boto3 import jsonschema +import mypy_boto3_s3 +from botocore.errorfactory import ClientError from django.conf import settings from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile +from qfieldcloud.settings_utils import DjangoStorages + logger = logging.getLogger(__name__) +class S3PrefixPath(NamedTuple): + """ + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + + Key: str + + +class S3Object(NamedTuple): + """ + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + + name: str + key: str + last_modified: datetime + size: int + etag: str + md5sum: str + + +class S3ObjectVersion: + """ + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + + def __init__( + self, name: str, data: mypy_boto3_s3.service_resource.ObjectVersion + ) -> None: + self.name = name + self._data = data + + @property + def id(self) -> str: + """Returns the version id""" + # NOTE id and version_id are the same thing + return self._data.id + + @property + def key(self) -> str: + return self._data.key + + @property + def last_modified(self) -> datetime: + return self._data.last_modified + + @property + def size(self) -> int: + return self._data.size + + @property + def e_tag(self) -> str: + return self._data.e_tag + + @property + def md5sum(self) -> str: + return self._data.e_tag.replace('"', "") + + @property + def is_latest(self) -> bool: + return self._data.is_latest + + @property + def display(self) -> str: + return self.last_modified.strftime("v%Y%m%d%H%M%S") + + +class S3ObjectWithVersions(NamedTuple): + """ + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + + latest: S3ObjectVersion + versions: list[S3ObjectVersion] + + @property + def total_size(self) -> int: + """Total size of all versions""" + # latest is also in versions + return sum(v.size for v in self.versions if v.size is not None) + + +def get_legacy_s3_credentials() -> DjangoStorages: + """ + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + assert settings.LEGACY_STORAGE_NAME + + return settings.STORAGES[settings.LEGACY_STORAGE_NAME] + + +def get_s3_session() -> boto3.Session: + """Get a new S3 Session instance using Django settings + + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + session = boto3.Session( + aws_access_key_id=get_legacy_s3_credentials()["OPTIONS"]["access_key"], + aws_secret_access_key=get_legacy_s3_credentials()["OPTIONS"]["secret_key"], + region_name=get_legacy_s3_credentials()["OPTIONS"]["region_name"], + ) + return session + + +def get_s3_bucket() -> mypy_boto3_s3.service_resource.Bucket: + """ + Get a new S3 Bucket instance using Django settings. + + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + bucket_name = get_legacy_s3_credentials()["OPTIONS"]["bucket_name"] + + assert bucket_name, "Expected `bucket_name` to be non-empty string!" + + session = get_s3_session() + s3 = session.resource( + "s3", + endpoint_url=get_legacy_s3_credentials()["OPTIONS"]["endpoint_url"], + ) + + # Ensure the bucket exists + s3.meta.client.head_bucket(Bucket=bucket_name) + + # Get the bucket resource + return s3.Bucket(bucket_name) + + +def get_s3_client() -> mypy_boto3_s3.Client: + """Get a new S3 client instance using Django settings + + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + + s3_session = get_s3_session() + s3_client = s3_session.client( + "s3", + endpoint_url=get_legacy_s3_credentials()["OPTIONS"]["endpoint_url"], + ) + return s3_client + + def _get_sha256_memory_file(file: InMemoryUploadedFile | TemporaryUploadedFile) -> str: BLOCKSIZE = 65536 hasher = hashlib.sha256() @@ -72,6 +228,67 @@ def strip_json_null_bytes(file: IO) -> IO: return result +def is_the_qgis_file(filename: str) -> bool: + """Returns whether the filename seems to be a QGIS project file by checking the file extension. + + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + path = PurePath(filename) + + if path.suffix.lower() in (".qgs", ".qgz"): + return True + + return False + + +def get_qgis_project_file(project_id: str) -> str | None: + """Return the relative path inside the project of the qgs/qgz file or + None if no qgs/qgz file is present + + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + + bucket = get_s3_bucket() + + prefix = f"projects/{project_id}/files/" + + for obj in bucket.objects.filter(Prefix=prefix): + if is_the_qgis_file(obj.key): + path = PurePath(obj.key) + return str(path.relative_to(*path.parts[:3])) + + return None + + +def check_s3_key(key: str) -> str | None: + """Check to see if an object exists on S3. It it exists, the function + returns the sha256 of the file from the metadata + + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + + client = get_s3_client() + try: + head = client.head_object( + Bucket=get_legacy_s3_credentials()["OPTIONS"]["bucket_name"], + Key=key, + ) + except ClientError as e: + if e.response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404: + return None + else: + raise e + + metadata = head["Metadata"] + if "sha256sum" in metadata: + return metadata["sha256sum"] + else: + return metadata["Sha256sum"] + + def get_deltafile_schema_validator() -> jsonschema.Draft7Validator: """Creates a JSON schema validator to check whether the provided delta file is valid. @@ -91,6 +308,78 @@ def get_deltafile_schema_validator() -> jsonschema.Draft7Validator: return jsonschema.Draft7Validator(schema_dict) +def get_project_files(project_id: str, path: str = "") -> list[S3Object]: + """Returns a list of files and their versions. + + Args: + project_id: the project id + path: additional filter prefix + + Returns: + the list of files + + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + bucket = get_s3_bucket() + root_prefix = f"projects/{project_id}/files/" + prefix = f"projects/{project_id}/files/{path}" + + return list_files(bucket, prefix, root_prefix) + + +def get_project_package_files(project_id: str, package_id: str) -> list[S3Object]: + """Returns a list of package files. + + Args: + project_id: the project id + + Returns: + the list of package files + + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + bucket = get_s3_bucket() + prefix = f"projects/{project_id}/packages/{package_id}/" + + return list_files(bucket, prefix, prefix) + + +def list_files( + bucket: mypy_boto3_s3.service_resource.Bucket, + prefix: str, + strip_prefix: str = "", +) -> list[S3Object]: + """List a bucket's objects under prefix. + + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + files = [] + for f in bucket.objects.filter(Prefix=prefix): + if strip_prefix: + start_idx = len(strip_prefix) + name = f.key[start_idx:] + else: + name = f.key + + files.append( + S3Object( + name=name, + key=f.key, + last_modified=f.last_modified, + size=f.size, + etag=f.e_tag, + md5sum=f.e_tag.replace('"', ""), + ) + ) + + files.sort(key=lambda f: f.name) + + return files + + def get_file_storage_choices() -> list[tuple[str, str]]: """ Returns configured storages keys. From 19669a555ea13b611b2b5c13bc5be60bc4f01ae4 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Wed, 11 Feb 2026 08:42:05 +0100 Subject: [PATCH 11/22] chore: add django migration for removing projects' legacy fields --- ...e_project_legacy_thumbnail_uri_and_more.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 docker-app/qfieldcloud/core/migrations/0095_remove_project_legacy_thumbnail_uri_and_more.py diff --git a/docker-app/qfieldcloud/core/migrations/0095_remove_project_legacy_thumbnail_uri_and_more.py b/docker-app/qfieldcloud/core/migrations/0095_remove_project_legacy_thumbnail_uri_and_more.py new file mode 100644 index 000000000..f060fa4a9 --- /dev/null +++ b/docker-app/qfieldcloud/core/migrations/0095_remove_project_legacy_thumbnail_uri_and_more.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.28 on 2026-02-11 07:31 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0094_alter_project_are_attachments_versioned_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="project", + name="legacy_thumbnail_uri", + ), + migrations.RemoveField( + model_name="useraccount", + name="legacy_avatar_uri", + ), + ] From c7fd333b2ac91f666bd54a9bb8c6824c54f5bf3c Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Wed, 11 Feb 2026 08:50:48 +0100 Subject: [PATCH 12/22] chore: fix 0081 migration to drop legacy storage support --- .../core/migrations/0081_file_storage_project_and_more.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-app/qfieldcloud/core/migrations/0081_file_storage_project_and_more.py b/docker-app/qfieldcloud/core/migrations/0081_file_storage_project_and_more.py index 9e27829d9..0c34b1369 100644 --- a/docker-app/qfieldcloud/core/migrations/0081_file_storage_project_and_more.py +++ b/docker-app/qfieldcloud/core/migrations/0081_file_storage_project_and_more.py @@ -1,7 +1,6 @@ # Generated by Django 3.2.25 on 2024-06-18 01:02 import django.core.validators -from django.conf import settings from django.db import migrations, models import qfieldcloud.core.models @@ -10,7 +9,7 @@ def get_file_storage_name(): - return settings.LEGACY_STORAGE_NAME or "default" + return "default" class Migration(migrations.Migration): From 012501fcd036873f9f2875dfb2ed7d548a48bee7 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Wed, 11 Feb 2026 10:32:22 +0100 Subject: [PATCH 13/22] chore: get rid of legacy compatibility views --- docker-app/qfieldcloud/core/urls.py | 6 +- .../qfieldcloud/core/views/package_views.py | 40 --------- docker-app/qfieldcloud/filestorage/urls.py | 16 ++-- docker-app/qfieldcloud/filestorage/views.py | 81 +------------------ docker-app/qfieldcloud/urls.py | 29 +------ 5 files changed, 16 insertions(+), 156 deletions(-) diff --git a/docker-app/qfieldcloud/core/urls.py b/docker-app/qfieldcloud/core/urls.py index fd5bc4cf8..b3860113f 100644 --- a/docker-app/qfieldcloud/core/urls.py +++ b/docker-app/qfieldcloud/core/urls.py @@ -61,15 +61,15 @@ ), path( "packages//latest/", - package_views.compatibility_latest_package_view, + package_views.LatestPackageView.as_view(), ), path( "packages//latest/files//", - package_views.compatibility_package_download_files_view, + package_views.LatestPackageDownloadFilesView.as_view(), ), path( "packages///files//", - package_views.compatibility_package_upload_files_view, + package_views.PackageUploadFilesView.as_view(), ), path("members//", members_views.ListCreateMembersView.as_view()), path( diff --git a/docker-app/qfieldcloud/core/views/package_views.py b/docker-app/qfieldcloud/core/views/package_views.py index 6b23052e0..f4797c2e4 100644 --- a/docker-app/qfieldcloud/core/views/package_views.py +++ b/docker-app/qfieldcloud/core/views/package_views.py @@ -4,7 +4,6 @@ from django.core.exceptions import ObjectDoesNotExist from django.shortcuts import get_object_or_404 from django.urls import reverse -from django.views.decorators.csrf import csrf_exempt from drf_spectacular.utils import ( OpenApiParameter, OpenApiTypes, @@ -432,42 +431,3 @@ def post( status=status.HTTP_201_CREATED, headers=headers, ) - - -@csrf_exempt -def compatibility_latest_package_view(request: Request, *args, **kwargs) -> Response: - project_id: UUID = kwargs["project_id"] - _project = get_object_or_404(Project, id=project_id) - logger.debug( - f"Project {project_id=} will be using the regular package file management." - ) - - return LatestPackageView.as_view()(request, *args, **kwargs) - - -@csrf_exempt -def compatibility_package_download_files_view( - request: Request, *args, **kwargs -) -> Response: - project_id: UUID = kwargs["project_id"] - _project = get_object_or_404(Project, id=project_id) - - logger.debug( - f"Project {project_id=} will be using the regular package file management." - ) - - return LatestPackageDownloadFilesView.as_view()(request, *args, **kwargs) - - -@csrf_exempt -def compatibility_package_upload_files_view( - request: Request, *args, **kwargs -) -> Response: - project_id: UUID = kwargs["project_id"] - _project = get_object_or_404(Project, id=project_id) - - logger.debug( - f"Project {project_id=} will be using the regular package file management." - ) - - return PackageUploadFilesView.as_view()(request, *args, **kwargs) diff --git a/docker-app/qfieldcloud/filestorage/urls.py b/docker-app/qfieldcloud/filestorage/urls.py index 18763a751..240807d89 100644 --- a/docker-app/qfieldcloud/filestorage/urls.py +++ b/docker-app/qfieldcloud/filestorage/urls.py @@ -2,31 +2,31 @@ from .views import ( AvatarFileReadView, - compatibility_file_crud_view, - compatibility_file_list_view, - compatibility_file_metadata_view, - compatibility_project_meta_file_read_view, + FileCrudView, + FileListView, + FileMetadataView, + ProjectMetaFileReadView, ) urlpatterns = [ path( "files//", - compatibility_file_list_view, + FileListView.as_view(), name="filestorage_list_files", ), path( "files///", - compatibility_file_crud_view, + FileCrudView.as_view(), name="filestorage_crud_file", ), path( "files/metadata///", - compatibility_file_metadata_view, + FileMetadataView.as_view(), name="filestorage_file_metadata", ), path( "files/thumbnails//", - compatibility_project_meta_file_read_view, + ProjectMetaFileReadView.as_view(), name="filestorage_project_thumbnails", ), path( diff --git a/docker-app/qfieldcloud/filestorage/views.py b/docker-app/qfieldcloud/filestorage/views.py index c2aaf0c92..0ff532787 100644 --- a/docker-app/qfieldcloud/filestorage/views.py +++ b/docker-app/qfieldcloud/filestorage/views.py @@ -5,10 +5,9 @@ from django.core import signing from django.db.models import Q, QuerySet from django.http import Http404 -from django.http.response import HttpResponse, HttpResponseBase +from django.http.response import HttpResponseBase from django.shortcuts import get_object_or_404, redirect from django.urls import reverse -from django.views.decorators.csrf import csrf_exempt from drf_spectacular.utils import ( OpenApiParameter, OpenApiTypes, @@ -250,81 +249,3 @@ def get( ) else: return redirect(staticfiles_storage.url("logo.svg")) - - -@csrf_exempt -def compatibility_file_list_view( - request: Request, *args, **kwargs -) -> Response | HttpResponse: - # let's assume that `kwargs["project_id"]` will no throw a `KeyError` - project_id: UUID = kwargs["project_id"] - view_kwargs = kwargs.pop("view_kwargs", {}) - - try: - _project = Project.objects.get(id=project_id) - except Project.DoesNotExist: - # if the project does not exist, we just fallback to the new view, which will return JSON formatted 404 later - return FileListView.as_view(**view_kwargs)(request, *args, **kwargs) - - logger.debug(f"Project {project_id=} will be using the regular file management.") - - return FileListView.as_view(**view_kwargs)(request, *args, **kwargs) - - -@csrf_exempt -def compatibility_file_metadata_view( - request: Request, *args, **kwargs -) -> Response | HttpResponse: - # let's assume that `kwargs["project_id"]` will no throw a `KeyError` - project_id: UUID = kwargs["project_id"] - view_kwargs = kwargs.pop("view_kwargs", {}) - - try: - _project = Project.objects.get(id=project_id) - except Project.DoesNotExist: - # if the project does not exist, we just fallback to the new view, which will return JSON formatted 404 later - return FileMetadataView.as_view(**view_kwargs)(request, *args, **kwargs) - - logger.debug(f"Project {project_id=} will be using the regular file management.") - - return FileMetadataView.as_view(**view_kwargs)(request, *args, **kwargs) - - -@csrf_exempt -def compatibility_file_crud_view( - request: Request, *args, **kwargs -) -> Response | HttpResponse: - # let's assume that `kwargs["project_id"]` will no throw a `KeyError` - project_id: UUID = kwargs["project_id"] - view_kwargs = kwargs.pop("view_kwargs", {}) - - try: - _project = Project.objects.get(id=project_id) - except Project.DoesNotExist: - # if the project does not exist, we just fallback to the new view, which will return JSON formatted 404 later - return FileCrudView.as_view(**view_kwargs)(request, *args, **kwargs) - - logger.debug(f"Project {project_id=} will be using the regular file management.") - - return FileCrudView.as_view(**view_kwargs)(request, *args, **kwargs) - - -@csrf_exempt -def compatibility_project_meta_file_read_view( - request: Request, *args, **kwargs -) -> Response | HttpResponse: - # let's assume that `kwargs["project_id"]` will no throw a `KeyError` - project_id: UUID = kwargs["project_id"] - view_kwargs = kwargs.pop("view_kwargs", {}) - - try: - _project = Project.objects.get(id=project_id) - except Project.DoesNotExist: - # if the project does not exist, we just fallback to the new view, which will return JSON formatted 404 later - return ProjectMetaFileReadView.as_view(**view_kwargs)(request, *args, **kwargs) - - logger.debug( - f"Project {project_id=} will be using the regular file management for meta files." - ) - - return ProjectMetaFileReadView.as_view(**view_kwargs)(request, *args, **kwargs) diff --git a/docker-app/qfieldcloud/urls.py b/docker-app/qfieldcloud/urls.py index ffb8c6646..ee49ed3ab 100644 --- a/docker-app/qfieldcloud/urls.py +++ b/docker-app/qfieldcloud/urls.py @@ -17,8 +17,6 @@ 'blog/', include('blog.urls')) """ -from functools import wraps - from django.conf import settings from django.contrib import admin from django.urls import include, path @@ -35,8 +33,8 @@ from qfieldcloud.core.admin import qfc_admin_site from qfieldcloud.core.views.redirect_views import redirect_to_admin_project_view from qfieldcloud.filestorage.views import ( - compatibility_file_crud_view, - compatibility_file_list_view, + FileCrudView, + FileListView, ) admin.site.site_header = _("QFieldCloud Admin") @@ -44,21 +42,6 @@ admin.site.index_title = _("Welcome to QFieldCloud Admin") -def add_view_kwargs(view, **view_kwargs): - """Adds kwargs to DRF views in the `.as_view()` call. - - Todo: - * Delete with QF-4963 Drop support for legacy storage - """ - - @wraps(view) - def wrapper(request, *args, **kwargs): - kwargs["view_kwargs"] = view_kwargs - return view(request, *args, **kwargs) - - return wrapper - - urlpatterns = [ path( "", @@ -82,15 +65,11 @@ def wrapper(request, *args, **kwargs): ), path( settings.QFIELDCLOUD_ADMIN_URI + "api/files//", - add_view_kwargs( - compatibility_file_list_view, permission_classes=[permissions.IsAdminUser] - ), + FileListView.as_view(permission_classes=[permissions.IsAdminUser]), ), path( settings.QFIELDCLOUD_ADMIN_URI + "api/files///", - add_view_kwargs( - compatibility_file_crud_view, permission_classes=[permissions.IsAdminUser] - ), + FileCrudView.as_view(permission_classes=[permissions.IsAdminUser]), name="project_file_download", ), path(settings.QFIELDCLOUD_ADMIN_URI, qfc_admin_site.urls), From 24e8a366283502a6e2503236a211abf1d2b40b69 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Wed, 11 Feb 2026 10:32:36 +0100 Subject: [PATCH 14/22] chore: drop legacy fields in serializers --- docker-app/qfieldcloud/filestorage/serializers.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/docker-app/qfieldcloud/filestorage/serializers.py b/docker-app/qfieldcloud/filestorage/serializers.py index 74c86f259..2af4273fc 100644 --- a/docker-app/qfieldcloud/filestorage/serializers.py +++ b/docker-app/qfieldcloud/filestorage/serializers.py @@ -46,9 +46,6 @@ class Meta: "uploaded_at", "display", "is_latest", - # TODO delete the fields below, they are deprecated and exists only as a compatibility layer for the legacy storage - "last_modified", - "sha256", ) read_only_fields = ( @@ -58,9 +55,6 @@ class Meta: "uploaded_at", "display", "is_latest", - # TODO delete the fields below, they are deprecated and exists only as a compatibility layer for the legacy storage - "last_modified", - "sha256", ) @@ -97,9 +91,6 @@ class Meta: "uploaded_at", "is_attachment", "md5sum", - # TODO delete the fields below, they are deprecated and exists only as a compatibility layer for the legacy storage - "last_modified", - "sha256", ] read_only_fields = [ "name", @@ -107,9 +98,6 @@ class Meta: "uploaded_at", "is_attachment", "md5sum", - # TODO delete the fields below, they are deprecated and exists only as a compatibility layer for the legacy storage - "last_modified", - "sha256", ] order_by = "name" From d572c5682168f3aaea5a896637864af4448234b7 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Wed, 11 Feb 2026 10:38:29 +0100 Subject: [PATCH 15/22] ci: remove storage matrix since there is only default --- .github/workflows/test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e0e842392..e40018673 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,8 +45,6 @@ jobs: - core - filestorage - __flaky__ - storage: - - default continue-on-error: true steps: - name: Checkout repo @@ -90,7 +88,7 @@ jobs: }' EOF - echo "STORAGES_PROJECT_DEFAULT_STORAGE=${{ matrix.storage }}" >> .env + echo "STORAGES_PROJECT_DEFAULT_STORAGE=default" >> .env - name: Pull docker containers run: docker compose pull From 8267cc5257f10355ba89361083508aac6f34f5fe Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Tue, 17 Feb 2026 09:56:50 +0100 Subject: [PATCH 16/22] fix: fix wrongly declared queryset --- docker-app/qfieldcloud/core/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-app/qfieldcloud/core/models.py b/docker-app/qfieldcloud/core/models.py index 50dcb9b74..a918fbe55 100644 --- a/docker-app/qfieldcloud/core/models.py +++ b/docker-app/qfieldcloud/core/models.py @@ -509,7 +509,7 @@ def storage_used_bytes(self) -> float: project_files_used_quota = ( FileVersion.objects.filter( file__file_type=File.FileType.PROJECT_FILE, - file__project__in=self.user.projects, + file__project__in=self.user.projects.all(), ).aggregate(sum_bytes=Sum("size"))["sum_bytes"] or 0 ) From 805cdb7f681b864ea458ad2eebdd71623102fd44 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Tue, 17 Feb 2026 09:58:00 +0100 Subject: [PATCH 17/22] chore: drop legacy package views not used anymore --- .../qfieldcloud/core/views/package_views.py | 194 +----------------- 1 file changed, 1 insertion(+), 193 deletions(-) diff --git a/docker-app/qfieldcloud/core/views/package_views.py b/docker-app/qfieldcloud/core/views/package_views.py index f4797c2e4..00b02c835 100644 --- a/docker-app/qfieldcloud/core/views/package_views.py +++ b/docker-app/qfieldcloud/core/views/package_views.py @@ -11,15 +11,10 @@ extend_schema_view, ) from qfieldcloud.authentication.models import AuthToken -from qfieldcloud.core import exceptions, utils +from qfieldcloud.core import exceptions from qfieldcloud.core import permissions_utils as perms from qfieldcloud.core.models import PackageJob, Project from qfieldcloud.core.serializers import LatestPackageSerializer -from qfieldcloud.core.utils import ( - check_s3_key, - get_project_files, - get_project_package_files, -) from qfieldcloud.core.utils2 import storage from qfieldcloud.filestorage.models import ( File, @@ -75,193 +70,6 @@ def has_permission(self, request, view): return False -@extend_schema_view( - get=extend_schema( - description="Get all the files in a project package with files.", - responses={200: LatestPackageSerializer()}, - ), -) -class LegacyLatestPackageView(views.APIView): - permission_classes = [permissions.IsAuthenticated, PackageViewPermissions] - - def get(self, request, project_id): - """Get last project package status and file list.""" - project = Project.objects.get(id=project_id) - latest_finished_package_job = project.latest_finished_package_job_for_user( - request.user - ) - - # Check if the project was packaged at least once - if not latest_finished_package_job: - raise exceptions.InvalidJobError( - "Packaging has never been triggered or successful for this project." - ) - - filenames = set() - files = [] - - # NOTE Some clients (e.g. QFieldSync) are still requiring the `sha256` key to check whether the files needs to be reuploaded. - # Since we do not have control on these old client versions, we need to keep the API backward compatible for some time and assume `skip_metadata=0` by default. - skip_metadata_param = request.GET.get("skip_metadata", "0") - if skip_metadata_param == "0": - skip_metadata = False - else: - skip_metadata = bool(skip_metadata_param) - - for f in get_project_package_files( - project_id, str(latest_finished_package_job.id) - ): - file_data = { - "name": f.name, - "size": f.size, - "last_modified": f.last_modified, - "md5sum": f.md5sum, - "is_attachment": False, - } - - if not skip_metadata: - file_data["sha256"] = check_s3_key(f.key) - - filenames.add(f.name) - files.append(file_data) - - # get attachment files directly from the original project files, not from the package - for attachment_dir in project.attachment_dirs: - for f in get_project_files(project_id, attachment_dir): - # skip files that are part of the package - if f.name in filenames: - continue - - file_data = { - "name": f.name, - "size": f.size, - "last_modified": f.last_modified, - "md5sum": f.md5sum, - "is_attachment": True, - } - - if not skip_metadata: - file_data["sha256"] = check_s3_key(f.key) - - filenames.add(f.name) - files.append(file_data) - - if not files: - raise exceptions.InvalidJobError("Empty project package.") - - assert latest_finished_package_job.feedback - - feedback_version = latest_finished_package_job.feedback.get("feedback_version") - - # version 2 and 3 have the same format - if feedback_version in ["2.0", "3.0"]: - layers = latest_finished_package_job.feedback["outputs"][ - "qgis_layers_data" - ]["layers_by_id"] - # support some ancient QFieldCloud job data - elif feedback_version is None: - steps = latest_finished_package_job.feedback.get("steps", []) - - layers = None - if len(steps) > 2 and steps[1].get("stage", 1) == 2: - layers = steps[1]["outputs"]["layer_checks"] - - # be paranoid and raise for newer versions - else: - raise NotImplementedError() - - return Response( - { - "files": files, - "layers": layers, - "status": latest_finished_package_job.status, - "package_id": latest_finished_package_job.pk, - "packaged_at": latest_finished_package_job.project.data_last_packaged_at, - "data_last_updated_at": latest_finished_package_job.project.data_last_updated_at, - } - ) - - -@extend_schema_view( - get=extend_schema( - description="Download a file from a project package.", - responses={ - (200, "*/*"): OpenApiTypes.BINARY, - }, - ), -) -class LegacyLatestPackageDownloadFilesView(views.APIView): - permission_classes = [permissions.IsAuthenticated, PackageViewPermissions] - - def get(self, request, project_id, filename): - """Download package file. - - Raises: - exceptions.InvalidJobError: [description] - """ - project = Project.objects.get(id=project_id) - latest_finished_package_job = project.latest_finished_package_job_for_user( - request.user - ) - - # Check if the project was packaged at least once - if not latest_finished_package_job: - raise exceptions.InvalidJobError( - "Packaging has never been triggered or successful for this project." - ) - - key = f"projects/{project_id}/packages/{latest_finished_package_job.id}/{filename}" - - # files within attachment dirs that do not exist is the packaged files should be served - # directly from the original data storage - if storage.get_attachment_dir_prefix(project, filename) and not check_s3_key( - key - ): - key = f"projects/{project_id}/files/{filename}" - - # NOTE the `expires` kwarg is sending the `Expires` header to the client, keep it a low value (in seconds). - return storage.file_response(request, key, expires=10, as_attachment=True) - - -@extend_schema_view( - post=extend_schema( - description="Upload a file to the package", - parameters=[ - OpenApiParameter( - name="file", - type=OpenApiTypes.BINARY, - location=OpenApiParameter.QUERY, - required=True, - description="File to be uploaded", - ) - ], - ) -) -class LegacyPackageUploadFilesView(views.APIView): - permission_classes = [permissions.IsAuthenticated, PackageUploadViewPermissions] - - def post(self, request, project_id, job_id, filename): - """Upload the package files.""" - key = utils.safe_join(f"projects/{project_id}/packages/{job_id}/", filename) - - request_file = request.FILES.get("file") - sha256sum = utils.get_sha256(request_file) - md5sum = utils.get_md5sum(request_file) - metadata = {"Sha256sum": sha256sum} - - bucket = utils.get_s3_bucket() - bucket.upload_fileobj(request_file, key, ExtraArgs={"Metadata": metadata}) - - return Response( - { - "name": filename, - "size": request_file.size, - "sha256": sha256sum, - "md5sum": md5sum, - } - ) - - @extend_schema_view( get=extend_schema( description="Get all the files in a project package with files.", From da4e80fbae9033647259e8a141a0fe04abca806c Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Tue, 17 Feb 2026 09:58:36 +0100 Subject: [PATCH 18/22] chore: reintroduce checksums utils function --- docker-app/qfieldcloud/core/utils2/storage.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docker-app/qfieldcloud/core/utils2/storage.py b/docker-app/qfieldcloud/core/utils2/storage.py index cda76fa49..06d90f44a 100644 --- a/docker-app/qfieldcloud/core/utils2/storage.py +++ b/docker-app/qfieldcloud/core/utils2/storage.py @@ -1,3 +1,7 @@ +import hashlib + +from django.core.files.base import ContentFile + import qfieldcloud.core.models @@ -18,3 +22,24 @@ def get_attachment_dir_prefix( return attachment_dir return "" + + +def calculate_checksums( + content: ContentFile, alrgorithms: tuple[str, ...], blocksize: int = 65536 +) -> tuple[bytes, ...]: + """Calculates checksums on given file for given algorithms.""" + hashers = [] + for alrgorithm in alrgorithms: + hashers.append(getattr(hashlib, alrgorithm)()) + + for chunk in content.chunks(blocksize): + for hasher in hashers: + hasher.update(chunk) + + content.seek(0) + + checksums = [] + for hasher in hashers: + checksums.append(hasher.digest()) + + return tuple(checksums) From 7ad4799b216cbfb014e54046988699c9f34b0796 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Tue, 17 Feb 2026 09:59:08 +0100 Subject: [PATCH 19/22] fix: add file metadata fields to file serializers --- docker-app/qfieldcloud/filestorage/serializers.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker-app/qfieldcloud/filestorage/serializers.py b/docker-app/qfieldcloud/filestorage/serializers.py index 2af4273fc..88a3c7870 100644 --- a/docker-app/qfieldcloud/filestorage/serializers.py +++ b/docker-app/qfieldcloud/filestorage/serializers.py @@ -46,6 +46,8 @@ class Meta: "uploaded_at", "display", "is_latest", + "last_modified", + "sha256", ) read_only_fields = ( @@ -55,6 +57,8 @@ class Meta: "uploaded_at", "display", "is_latest", + "last_modified", + "sha256", ) @@ -91,6 +95,8 @@ class Meta: "uploaded_at", "is_attachment", "md5sum", + "last_modified", + "sha256", ] read_only_fields = [ "name", @@ -98,6 +104,8 @@ class Meta: "uploaded_at", "is_attachment", "md5sum", + "last_modified", + "sha256", ] order_by = "name" From 91c5b7ab79af0e2e43ad256d5655d7505b7508f8 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Tue, 17 Feb 2026 11:46:06 +0100 Subject: [PATCH 20/22] fix: remove legacy thumbnail fields when saving in worker wrapper --- docker-app/worker_wrapper/wrapper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-app/worker_wrapper/wrapper.py b/docker-app/worker_wrapper/wrapper.py index b928275f4..7ad1ee546 100644 --- a/docker-app/worker_wrapper/wrapper.py +++ b/docker-app/worker_wrapper/wrapper.py @@ -639,7 +639,6 @@ def after_docker_run(self) -> None: project.save( update_fields=( "project_details", - "legacy_thumbnail_uri", "thumbnail", ) ) From c1d38645fef47faeaa04cd59f54c8d6a1be9ac54 Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Tue, 17 Feb 2026 14:06:36 +0100 Subject: [PATCH 21/22] fix: reintroduce deleted utils functions --- docker-app/qfieldcloud/core/utils.py | 107 +++++++++++++++++- docker-app/qfieldcloud/core/utils2/storage.py | 76 +++++++++++++ 2 files changed, 182 insertions(+), 1 deletion(-) diff --git a/docker-app/qfieldcloud/core/utils.py b/docker-app/qfieldcloud/core/utils.py index 193fd18e9..ae00523de 100644 --- a/docker-app/qfieldcloud/core/utils.py +++ b/docker-app/qfieldcloud/core/utils.py @@ -3,9 +3,10 @@ import json import logging import os +import posixpath from datetime import datetime from pathlib import PurePath -from typing import IO, NamedTuple +from typing import IO, Generator, NamedTuple import boto3 import jsonschema @@ -228,6 +229,48 @@ def strip_json_null_bytes(file: IO) -> IO: return result +def safe_join(base: str, *paths: str) -> str: + """ + A version of django.utils._os.safe_join for S3 paths. + Joins one or more path components to the base path component + intelligently. Returns a normalized version of the final path. + The final path must be located inside of the base path component + (otherwise a ValueError is raised). + Paths outside the base path indicate a possible security + sensitive operation. + + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + base_path = base + base_path = base_path.rstrip("/") + paths = tuple(paths) + + final_path = base_path + "/" + + for path in paths: + _final_path = posixpath.normpath(posixpath.join(final_path, path)) + + # posixpath.normpath() strips the trailing /. Add it back. + if path.endswith("/") or _final_path + "/" == final_path: + _final_path += "/" + + final_path = _final_path + + if final_path == base_path: + final_path += "/" + + # Ensure final_path starts with base_path and that the next character after + # the base path is /. + base_path_len = len(base_path) + if not final_path.startswith(base_path) or final_path[base_path_len] != "/": + raise ValueError( + "the joined path is located outside of the base path component" + ) + + return final_path.lstrip("/") + + def is_the_qgis_file(filename: str) -> bool: """Returns whether the filename seems to be a QGIS project file by checking the file extension. @@ -380,6 +423,68 @@ def list_files( return files +def list_versions( + bucket: mypy_boto3_s3.service_resource.Bucket, + prefix: str, + strip_prefix: str = "", +) -> list[S3ObjectVersion]: + """Iterator that lists a bucket's objects under prefix. + + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + versions = [] + for v in bucket.object_versions.filter(Prefix=prefix): + if strip_prefix: + start_idx = len(prefix) + name = v.key[start_idx:] + else: + name = v.key + + versions.append(S3ObjectVersion(name, v)) + + versions.sort(key=lambda v: (v.key, v.last_modified)) + + return versions + + +def list_files_with_versions( + bucket: mypy_boto3_s3.service_resource.Bucket, + prefix: str, + strip_prefix: str = "", +) -> Generator[S3ObjectWithVersions, None, None]: + """Yields an object with all it's versions + Yields: + Generator[S3ObjectWithVersions]: the object with its versions + + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + last_key = None + versions: list[S3ObjectVersion] = [] + latest: S3ObjectVersion | None = None + + for v in list_versions(bucket, prefix, strip_prefix): + if last_key != v.key: + if last_key: + assert latest + + yield S3ObjectWithVersions(latest, versions) + + latest = None + versions = [] + last_key = v.key + + versions.append(v) + + if v.is_latest: + latest = v + + if last_key: + assert latest + yield S3ObjectWithVersions(latest, versions) + + def get_file_storage_choices() -> list[tuple[str, str]]: """ Returns configured storages keys. diff --git a/docker-app/qfieldcloud/core/utils2/storage.py b/docker-app/qfieldcloud/core/utils2/storage.py index 06d90f44a..c5d3ec0cb 100644 --- a/docker-app/qfieldcloud/core/utils2/storage.py +++ b/docker-app/qfieldcloud/core/utils2/storage.py @@ -1,8 +1,27 @@ import hashlib +import logging +import re from django.core.files.base import ContentFile +from django.core.files.storage import storages import qfieldcloud.core.models +import qfieldcloud.core.utils +from qfieldcloud.filestorage.backend import QfcS3Boto3Storage + +logger = logging.getLogger(__name__) + + +def delete_version_permanently(version_obj: qfieldcloud.core.utils.S3ObjectVersion): + """ + Todo: + * Delete with QF-4963 Drop support for legacy storage + """ + logging.info( + f'S3 object version deletion (permanent) with "{version_obj.key=}" and "{version_obj.id=}"' + ) + + version_obj._data.delete() def get_attachment_dir_prefix( @@ -24,6 +43,63 @@ def get_attachment_dir_prefix( return "" +def purge_previous_thumbnails_versions( + project: qfieldcloud.core.models.Project, +) -> None: + # this method applies only to S3 storage + if not isinstance(project.file_storage, QfcS3Boto3Storage): + return + + bucket = storages[project.file_storage].bucket # type: ignore + prefix = project.thumbnail.name + + if not prefix: + return + + thumbnail_files = list( + qfieldcloud.core.utils.list_files_with_versions(bucket, prefix) + ) + + if len(thumbnail_files) == 0: + logger.info(f'No thumbnail found to delete for project "{project.id}"!') + return + + assert len(thumbnail_files) == 1 + + thumbnail_file = thumbnail_files[0] + + # we only keep 1 version of the thumbnail file. + # otherwise we hit the limit of 1000. + keep_count = 1 + + old_versions_to_purge = sorted( + thumbnail_file.versions, key=lambda v: v.last_modified, reverse=True + )[keep_count:] + + # Remove the N oldest + for old_version in old_versions_to_purge: + logger.info( + f'Purging {old_version.key=} {old_version.id=} as old version for "{thumbnail_file.latest.name}"...' + ) + + if old_version.is_latest: + # This is not supposed to happen, as versions were sorted above, + # but leaving it here as a security measure in case version + # ordering changes for some reason. + raise Exception("Trying to delete latest version") + + if not old_version.key or not re.match( + r"^projects/[\w]{8}(-[\w]{4}){3}-[\w]{12}/meta/thumbnail.png$", + old_version.key, + ): + raise RuntimeError( + f"Suspicious S3 file version deletion {old_version.key=} {old_version.id=}" + ) + # TODO: any way to batch those ? will probaby get slow on production + delete_version_permanently(old_version) + # TODO: audit ? take implementation from files_views.py:211 + + def calculate_checksums( content: ContentFile, alrgorithms: tuple[str, ...], blocksize: int = 65536 ) -> tuple[bytes, ...]: From d33f703b465a6b7514da74ca98ebeaa88570dd1e Mon Sep 17 00:00:00 2001 From: Guilhem Allaman Date: Tue, 17 Feb 2026 14:37:32 +0100 Subject: [PATCH 22/22] test: fix failing test --- .../qfieldcloud/filestorage/tests/test_storage_usage.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docker-app/qfieldcloud/filestorage/tests/test_storage_usage.py b/docker-app/qfieldcloud/filestorage/tests/test_storage_usage.py index 63573b29e..b8cf74353 100644 --- a/docker-app/qfieldcloud/filestorage/tests/test_storage_usage.py +++ b/docker-app/qfieldcloud/filestorage/tests/test_storage_usage.py @@ -56,10 +56,3 @@ def test_upload_file_succeeds(self): self.assertEqual(self.p1.get_file("file.name").versions.count(), 2) self.assertEqual(self.p1.file_storage_bytes, 13) self.assertEqual(self.u1.useraccount.storage_used_bytes, 13) - - # p1 checks - self.assertEqual(self.p1.project_files.count(), 1) - self.assertEqual(self.p1.get_file("file.name").versions.count(), 2) - self.assertEqual(self.p1.file_storage_bytes, 13) - - self.assertEqual(self.u1.useraccount.storage_used_bytes, 20)