Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
2cb732a
chore: get rid of legacy storage
gounux Feb 11, 2026
a71022f
chore: get rid of legacy storage in the `core` module
gounux Feb 11, 2026
b829143
chore: get rid of legacy storage in the `filestorage` module
gounux Feb 11, 2026
1c99eb4
chore: get rid of legacy storage in the `subscription` module
gounux Feb 11, 2026
e5736ff
chore: get rid of legacy storage in the worker_wrapper
gounux Feb 11, 2026
7f78a8d
chore: get rid of legacy storage in the CI test workflow
gounux Feb 11, 2026
e332791
fix: reintroduce storage utils functions
gounux Feb 11, 2026
1b9c61d
chore: drop support of legacy storage in filestorage views
gounux Feb 11, 2026
5232624
doc: add comment to show deprecation of avatar migration command
gounux Feb 11, 2026
5c41641
fix: temporary reintroduce utils functions to prepare migration
gounux Feb 11, 2026
19669a5
chore: add django migration for removing projects' legacy fields
gounux Feb 11, 2026
c7fd333
chore: fix 0081 migration to drop legacy storage support
gounux Feb 11, 2026
012501f
chore: get rid of legacy compatibility views
gounux Feb 11, 2026
24e8a36
chore: drop legacy fields in serializers
gounux Feb 11, 2026
d572c56
ci: remove storage matrix since there is only default
gounux Feb 11, 2026
8267cc5
fix: fix wrongly declared queryset
gounux Feb 17, 2026
805cdb7
chore: drop legacy package views not used anymore
gounux Feb 17, 2026
da4e80f
chore: reintroduce checksums utils function
gounux Feb 17, 2026
7ad4799
fix: add file metadata fields to file serializers
gounux Feb 17, 2026
91c5b7a
fix: remove legacy thumbnail fields when saving in worker wrapper
gounux Feb 17, 2026
c1d3864
fix: reintroduce deleted utils functions
gounux Feb 17, 2026
d33f703
test: fix failing test
gounux Feb 17, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 1 addition & 15 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,6 @@ jobs:
- core
- filestorage
- __flaky__
storage:
- default
- legacy_storage
continue-on-error: true
steps:
- name: Checkout repo
Expand All @@ -68,17 +65,6 @@ jobs:

cat <<EOF >> .env
STORAGES='{
"legacy_storage": {
"BACKEND": "qfieldcloud.filestorage.backend.QfcS3Boto3Storage",
"OPTIONS": {
"access_key": "minioadmin",
"secret_key": "minioadmin",
"bucket_name": "qfieldcloud-local-legacy",
"region_name": "",
"endpoint_url": "http://172.17.0.1:8009"
},
"QFC_IS_LEGACY": true
},
"webdav": {
"BACKEND": "qfieldcloud.filestorage.backend.QfcWebDavStorage",
"OPTIONS": {
Expand All @@ -102,7 +88,7 @@ jobs:
}'
EOF

echo "STORAGES_PROJECT_DEFAULT_STORAGE=${{ matrix.storage }}" >> .env
echo "STORAGES_PROJECT_DEFAULT_STORAGE=default" >> .env

- name: Pull docker containers
run: docker compose pull
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# Generated by Django 3.2.25 on 2024-06-18 01:02

import django.core.validators
from django.conf import settings
from django.db import migrations, models

import qfieldcloud.core.models
Expand All @@ -10,7 +9,7 @@


def get_file_storage_name():
return settings.LEGACY_STORAGE_NAME or "default"
return "default"


class Migration(migrations.Migration):
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Generated by Django 4.2.28 on 2026-02-11 07:31

from django.db import migrations


class Migration(migrations.Migration):
dependencies = [
("core", "0094_alter_project_are_attachments_versioned_and_more"),
]

operations = [
migrations.RemoveField(
model_name="project",
name="legacy_thumbnail_uri",
),
migrations.RemoveField(
model_name="useraccount",
name="legacy_avatar_uri",
),
]
111 changes: 12 additions & 99 deletions docker-app/qfieldcloud/core/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
from typing import TYPE_CHECKING, Any, cast
from uuid import uuid4

from deprecated import deprecated
from django.conf import settings
from django.contrib.auth.models import AbstractUser
from django.contrib.auth.models import UserManager as DjangoUserManager
Expand Down Expand Up @@ -39,9 +38,8 @@
)
from timezone_field import TimeZoneField

from qfieldcloud.core import utils, validators
from qfieldcloud.core import validators
from qfieldcloud.core.fields import DynamicStorageFileField, QfcImageField, QfcImageFile
from qfieldcloud.core.utils2 import storage
from qfieldcloud.subscription.exceptions import ReachedMaxOrganizationMembersError

if TYPE_CHECKING:
Expand Down Expand Up @@ -467,13 +465,6 @@ class UserAccount(models.Model):
twitter = models.CharField(max_length=255, default="", blank=True)
is_email_public = models.BooleanField(default=False)

# TODO Delete with QF-4963 Drop support for legacy storage
legacy_avatar_uri = models.CharField(
_("Legacy Profile Picture URI"),
max_length=255,
blank=True,
)

avatar = QfcImageField(
_("Avatar Picture"),
upload_to=get_user_account_avatar_upload_to,
Expand Down Expand Up @@ -518,23 +509,12 @@ def storage_used_bytes(self) -> float:
project_files_used_quota = (
FileVersion.objects.filter(
file__file_type=File.FileType.PROJECT_FILE,
file__project__in=self.user.projects.exclude(
file_storage=settings.LEGACY_STORAGE_NAME
),
file__project__in=self.user.projects.all(),
).aggregate(sum_bytes=Sum("size"))["sum_bytes"]
or 0
)

# TODO: Delete with QF-4963 Drop support for legacy storage
legacy_used_quota = (
self.user.projects.filter(
file_storage=settings.LEGACY_STORAGE_NAME
).aggregate(sum_bytes=Sum("file_storage_bytes"))["sum_bytes"]
# if there are no projects, the value will be `None`
or 0
)

used_quota = project_files_used_quota + legacy_used_quota
used_quota = project_files_used_quota

return used_quota

Expand Down Expand Up @@ -1211,11 +1191,6 @@ class Meta:
),
)

# TODO: Delete with QF-4963 Drop support for legacy storage
legacy_thumbnail_uri = models.CharField(
_("Legacy Thumbnail Picture URI"), max_length=255, blank=True
)

thumbnail = DynamicStorageFileField(
_("Thumbnail Picture"),
upload_to=get_project_thumbnail_upload_to,
Expand Down Expand Up @@ -1491,17 +1466,9 @@ def owner_aware_storage_keep_versions(self) -> int:

@property
def thumbnail_url(self) -> StrOrPromise:
"""Returns the url to the project's thumbnail or empty string if no URL provided.

Todo:
* Delete with QF-4963 Drop support for legacy storage
"""
if self.uses_legacy_storage:
if not self.legacy_thumbnail_uri:
return ""
else:
if not self.thumbnail:
return ""
"""Returns the url to the project's thumbnail or empty string if no URL provided."""
if not self.thumbnail:
return ""

return reverse_lazy(
"filestorage_project_thumbnails",
Expand Down Expand Up @@ -1575,45 +1542,14 @@ def private(self) -> bool:
# still used in the project serializer
return not self.is_public

@property
def uses_legacy_storage(self) -> bool:
"""Whether the storage of the project is legacy.

Todo:
* Delete with QF-4963 Drop support for legacy storage
"""
return self.file_storage == settings.LEGACY_STORAGE_NAME

@cached_property
@deprecated
def legacy_files(self) -> list[utils.S3ObjectWithVersions]:
"""Gets all the files from S3 storage. This is potentially slow. Results are cached on the instance.

Todo:
* Delete with QF-4963 Drop support for legacy storage
"""
if self.uses_legacy_storage:
return list(utils.get_project_files_with_versions(str(self.id)))
else:
raise NotImplementedError(
"The `Project.legacy_files` method is not implemented for projects stored in non-legacy storage"
)

@property
def project_files(self) -> "FileQueryset":
"""Returns the files of type PROJECT related to the project."""
return self.all_files.with_type_project()

@property
def project_files_count(self) -> int:
"""
Todo:
* Delete with QF-4963 Drop support for legacy storage
"""
if self.uses_legacy_storage:
return len(self.legacy_files)
else:
return self.project_files.count()
return self.project_files.count()

@property
def users(self):
Expand Down Expand Up @@ -1852,14 +1788,7 @@ def direct_collaborators(self) -> ProjectCollaboratorQueryset:
)

def delete(self, *args, **kwargs):
"""Deletes the project and the thumbnail for the legacy storage.

Todo:
* Delete with QF-4963 Drop support for legacy storage
"""
if self.uses_legacy_storage:
storage.delete_project_thumbnail(self)

"""Deletes the project and the thumbnail for the legacy storage."""
return super().delete(*args, **kwargs)

@property
Expand All @@ -1878,15 +1807,9 @@ def save(self, recompute_storage=False, *args, **kwargs):
additional_update_fields = set()

if recompute_storage:
# TODO Delete with QF-4963 Drop support for legacy storage
if self.uses_legacy_storage:
self.file_storage_bytes = storage.get_project_file_storage_in_bytes(
self
)
else:
self.file_storage_bytes = self.project_files.aggregate(
file_storage_bytes=Sum("versions__size", default=0)
)["file_storage_bytes"]
self.file_storage_bytes = self.project_files.aggregate(
file_storage_bytes=Sum("versions__size", default=0)
)["file_storage_bytes"]

additional_update_fields.add("file_storage_bytes")

Expand All @@ -1912,11 +1835,6 @@ def save(self, recompute_storage=False, *args, **kwargs):
def get_file(self, filename: str) -> File:
return self.project_files.get_by_name(filename) # type: ignore

def legacy_get_file(self, filename: str) -> utils.S3ObjectWithVersions:
files = filter(lambda f: f.latest.name == filename, self.legacy_files)

return next(files)


class ProjectCollaboratorQueryset(models.QuerySet):
def validated(self, skip_invalid=False):
Expand Down Expand Up @@ -2710,10 +2628,5 @@ class Meta:
)

def _get_file_storage_name(self) -> str:
# TODO Delete with QF-4963 Drop support for legacy storage
# Legacy storage - use default storage
if self.project.uses_legacy_storage:
return "default"

# Non-legacy storage - use same storage as project
# Use same storage as project
return self.project.file_storage
24 changes: 8 additions & 16 deletions docker-app/qfieldcloud/core/tests/test_commands.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import csv
import io
import os

from django.core.files.base import ContentFile
Expand All @@ -8,7 +7,6 @@

from qfieldcloud.core.models import Person, Project
from qfieldcloud.core.tests.utils import set_subscription, setup_subscription_plans
from qfieldcloud.core.utils2 import storage
from qfieldcloud.filestorage.models import File, FileVersion


Expand All @@ -23,20 +21,14 @@ def setUpTestData(cls):
# Project
cls.p1 = Project.objects.create(name="test_project", owner=user)

# TODO Delete with QF-4963 Drop support for legacy storage
if cls.p1.uses_legacy_storage:
storage.upload_project_file(
cls.p1, io.BytesIO(b"Hello world!"), "project.qgs"
)
else:
FileVersion.objects.add_version(
project=cls.p1,
filename="file.name",
# NOTE the dummy name is required when running tests on GitHub CI, but not locally. Spent few hours before I isolated this...
content=ContentFile(b"Hello world!", "dummy.name"),
file_type=File.FileType.PROJECT_FILE,
uploaded_by=user,
)
FileVersion.objects.add_version(
project=cls.p1,
filename="file.name",
# NOTE the dummy name is required when running tests on GitHub CI, but not locally. Spent few hours before I isolated this...
content=ContentFile(b"Hello world!", "dummy.name"),
file_type=File.FileType.PROJECT_FILE,
uploaded_by=user,
)

def test_extracts3data_output_to_file(self):
output_file = "extracted.csv"
Expand Down
Loading
Loading