From 0fc5d0d78935a995967ace299f1a654948f12969 Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Mon, 16 Feb 2026 12:28:12 +0200 Subject: [PATCH 01/18] feat: update model verbose names and translations Refs: RATYK-154 --- address/locale/fi/LC_MESSAGES/django.po | 52 +++++- .../0009_update_model_verbose_names.py | 166 ++++++++++++++++++ address/models.py | 49 ++++-- 3 files changed, 249 insertions(+), 18 deletions(-) create mode 100644 address/migrations/0009_update_model_verbose_names.py diff --git a/address/locale/fi/LC_MESSAGES/django.po b/address/locale/fi/LC_MESSAGES/django.po index 5ae187a..77540da 100644 --- a/address/locale/fi/LC_MESSAGES/django.po +++ b/address/locale/fi/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2021-09-13 20:28+0300\n" +"POT-Creation-Date: 2026-02-16 11:27+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -18,11 +18,53 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" +msgid "Id" +msgstr "Tunniste" + +msgid "Municipality code" +msgstr "Kuntakoodi" + msgid "Name" msgstr "Nimi" -msgid "municipalities" -msgstr "kunnat" +msgid "Area" +msgstr "Alue" + +msgid "Municipality" +msgstr "Kunta" + +msgid "Municipalities" +msgstr "Kunnat" + +msgid "Street" +msgstr "Katu" + +msgid "Streets" +msgstr "Kadut" + +msgid "Postal code" +msgstr "Postinumero" + +msgid "Postal code area" +msgstr "Postinumeroalue" + +msgid "Postal code areas" +msgstr "Postinumeroalueet" + +msgid "Number" +msgstr "Numero" + +msgid "Number end" +msgstr "Numeron loppu" + +msgid "Letter" +msgstr "Kirjain" + +msgid "Location" +msgstr "Sijainti" + +msgid "Address" +msgstr "Osoite" -msgid "addresses" -msgstr "osoitteet" +msgid "Addresses" +msgstr "Osoitteet" diff --git a/address/migrations/0009_update_model_verbose_names.py b/address/migrations/0009_update_model_verbose_names.py new file mode 100644 index 0000000..2b04c54 --- /dev/null +++ b/address/migrations/0009_update_model_verbose_names.py @@ -0,0 +1,166 @@ +# Generated by Django 6.0.2 on 2026-02-16 10:07 + +import django.contrib.gis.db.models.fields +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("address", "0008_address_idx_address_municipality_id"), + ] + + operations = [ + migrations.AlterModelOptions( + name="address", + options={"verbose_name": "Address", "verbose_name_plural": "Addresses"}, + ), + migrations.AlterModelOptions( + name="municipality", + options={ + "verbose_name": "Municipality", + "verbose_name_plural": "Municipalities", + }, + ), + migrations.AlterModelOptions( + name="municipalitytranslation", + options={ + "default_permissions": (), + "managed": True, + "verbose_name": "Municipality Translation", + }, + ), + migrations.AlterModelOptions( + name="postalcodearea", + options={ + "verbose_name": "Postal code area", + "verbose_name_plural": "Postal code areas", + }, + ), + migrations.AlterModelOptions( + name="postalcodeareatranslation", + options={ + "default_permissions": (), + "managed": True, + "verbose_name": "Postal code area Translation", + }, + ), + migrations.AlterModelOptions( + name="street", + options={"verbose_name": "Street", "verbose_name_plural": "Streets"}, + ), + migrations.AlterModelOptions( + name="streettranslation", + options={ + "default_permissions": (), + "managed": True, + "verbose_name": "Street Translation", + }, + ), + migrations.AlterField( + model_name="address", + name="letter", + field=models.CharField(blank=True, max_length=2, verbose_name="Letter"), + ), + migrations.AlterField( + model_name="address", + name="location", + field=django.contrib.gis.db.models.fields.PointField( + srid=4326, verbose_name="Location" + ), + ), + migrations.AlterField( + model_name="address", + name="municipality", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="address.municipality", + verbose_name="Municipality", + ), + ), + migrations.AlterField( + model_name="address", + name="number", + field=models.CharField(blank=True, max_length=6, verbose_name="Number"), + ), + migrations.AlterField( + model_name="address", + name="number_end", + field=models.CharField(blank=True, max_length=6, verbose_name="Number end"), + ), + migrations.AlterField( + model_name="address", + name="postal_code_area", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="addresses", + to="address.postalcodearea", + verbose_name="Postal code area", + ), + ), + migrations.AlterField( + model_name="address", + name="street", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="addresses", + to="address.street", + verbose_name="Street", + ), + ), + migrations.AlterField( + model_name="municipality", + name="area", + field=django.contrib.gis.db.models.fields.MultiPolygonField( + blank=True, null=True, srid=4326, verbose_name="Area" + ), + ), + migrations.AlterField( + model_name="municipality", + name="code", + field=models.CharField(max_length=3, verbose_name="Municipality code"), + ), + migrations.AlterField( + model_name="municipality", + name="id", + field=models.CharField( + max_length=100, primary_key=True, serialize=False, verbose_name="Id" + ), + ), + migrations.AlterField( + model_name="postalcodearea", + name="area", + field=django.contrib.gis.db.models.fields.MultiPolygonField( + blank=True, null=True, srid=4326, verbose_name="Area" + ), + ), + migrations.AlterField( + model_name="postalcodearea", + name="municipality", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="address.municipality", + verbose_name="Municipality", + ), + ), + migrations.AlterField( + model_name="postalcodearea", + name="postal_code", + field=models.CharField( + blank=True, max_length=5, null=True, verbose_name="Postal code" + ), + ), + migrations.AlterField( + model_name="street", + name="municipality", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="address.municipality", + verbose_name="Municipality", + ), + ), + ] diff --git a/address/models.py b/address/models.py index b95c876..acac48c 100644 --- a/address/models.py +++ b/address/models.py @@ -5,24 +5,27 @@ class Municipality(TranslatableModel): - id = models.CharField(max_length=100, primary_key=True) - code = models.CharField(max_length=3) + id = models.CharField(_("Id"), max_length=100, primary_key=True) + code = models.CharField(_("Municipality code"), max_length=3) translations = TranslatedFields( name=models.CharField(_("Name"), max_length=100, db_index=True) ) area = models.MultiPolygonField( - srid=settings.PROJECTION_SRID, null=True, blank=True + _("Area"), srid=settings.PROJECTION_SRID, null=True, blank=True ) def __str__(self) -> str: return self.name class Meta: + verbose_name = _("Municipality") verbose_name_plural = _("Municipalities") class Street(TranslatableModel): - municipality = models.ForeignKey(Municipality, models.CASCADE, db_index=True) + municipality = models.ForeignKey( + Municipality, models.CASCADE, db_index=True, verbose_name=_("Municipality") + ) modified_at = models.DateTimeField(auto_now=True) translations = TranslatedFields( name=models.CharField(_("Name"), max_length=100, db_index=True), @@ -31,42 +34,61 @@ class Street(TranslatableModel): def __str__(self) -> str: return self.name + class Meta: + verbose_name = _("Street") + verbose_name_plural = _("Streets") + class PostalCodeArea(TranslatableModel): - postal_code = models.CharField(max_length=5, null=True, blank=True) + postal_code = models.CharField( + _("Postal code"), max_length=5, null=True, blank=True + ) municipality = models.ForeignKey( - Municipality, models.CASCADE, db_index=True, null=True, blank=True + Municipality, + models.CASCADE, + db_index=True, + null=True, + blank=True, + verbose_name=_("Municipality"), ) translations = TranslatedFields( name=models.CharField(_("Name"), max_length=100, null=True, blank=True) ) area = models.MultiPolygonField( - srid=settings.PROJECTION_SRID, null=True, blank=True + _("Area"), srid=settings.PROJECTION_SRID, null=True, blank=True ) def __str__(self) -> str: return self.postal_code class Meta: + verbose_name = _("Postal code area") verbose_name_plural = _("Postal code areas") class Address(models.Model): - municipality = models.ForeignKey(Municipality, models.CASCADE, db_index=True) + municipality = models.ForeignKey( + Municipality, models.CASCADE, db_index=True, verbose_name=_("Municipality") + ) street = models.ForeignKey( - Street, models.CASCADE, db_index=True, related_name="addresses" + Street, + models.CASCADE, + db_index=True, + related_name="addresses", + verbose_name=_("Street"), ) - number = models.CharField(max_length=6, blank=True) - number_end = models.CharField(max_length=6, blank=True) - letter = models.CharField(max_length=2, blank=True) + number = models.CharField(_("Number"), max_length=6, blank=True) + number_end = models.CharField(_("Number end"), max_length=6, blank=True) + letter = models.CharField(_("Letter"), max_length=2, blank=True) postal_code_area = models.ForeignKey( PostalCodeArea, models.CASCADE, null=True, blank=True, related_name="addresses", + verbose_name=_("Postal code area"), ) - location = models.PointField(srid=settings.PROJECTION_SRID) + location = models.PointField(_("Location"), srid=settings.PROJECTION_SRID) modified_at = models.DateTimeField(auto_now=True) def __str__(self) -> str: @@ -78,6 +100,7 @@ def __str__(self) -> str: return f"{s}, {self.municipality}" class Meta: + verbose_name = _("Address") verbose_name_plural = _("Addresses") indexes = [ # This index speeds up address?municipality and address?municipalitycode From 6650fd2ff7425c61b3123cdbc851e3ea407d2324 Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Mon, 16 Feb 2026 12:30:49 +0200 Subject: [PATCH 02/18] feat: update Django Admin configs Update list display, ordering and search fields for: - MunicipalityAdmin - StreetAdmin - PostalCodeAreaAdmin - AddressAdmin Refs: RATYK-154 --- address/admin.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/address/admin.py b/address/admin.py index d1b9ff3..007ca1a 100644 --- a/address/admin.py +++ b/address/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin +from django.utils.translation import gettext_lazy as _ from parler.admin import TranslatableAdmin from .models import Address, Municipality, PostalCodeArea, Street @@ -6,19 +7,40 @@ @admin.register(Municipality) class MunicipalityAdmin(TranslatableAdmin): - pass + list_display = ("id", "code") + ordering = ("id",) + search_fields = ("id",) @admin.register(Street) class StreetAdmin(TranslatableAdmin): - pass + list_display = ("name_column", "municipality") + search_fields = ("translations__name",) + + @admin.display(description=_("Name")) + def name_column(self, object): + return object.name + + def get_queryset(self, request): + language_code = self.get_queryset_language(request) + return ( + super() + .get_queryset(request) + .translated(language_code) + .order_by("translations__name") + ) @admin.register(PostalCodeArea) class PostalCodeAreaAdmin(TranslatableAdmin): - pass + list_display = ("postal_code", "name", "municipality") + ordering = ("postal_code",) + search_fields = ("postal_code",) @admin.register(Address) class AddressAdmin(admin.ModelAdmin): + list_display = ("street", "number", "municipality", "postal_code_area") raw_id_fields = ["street"] + ordering = ("street", "number", "municipality") + search_fields = ("street__translations__name",) From 136bac8cdefecc01149d28f9ee4dff04dacdd68b Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Mon, 16 Feb 2026 15:11:56 +0200 Subject: [PATCH 03/18] feat: add english language support Update importers to use english language for: - Street - Municipality - Postal code area Fallback-logic explained: - sv: use finnish language if swedish translation is empty - fi: use swedish language if finnish translation is empty - en: use finnish or swedish version as english Refs: RATYK-154 --- address/services/address_import.py | 47 ++++++++------- address/services/import_utils.py | 32 +++++++++++ address/services/municipality_import.py | 31 +++++----- address/services/postal_code_import.py | 18 ++++-- address/tests/factories.py | 6 ++ address/tests/test_models.py | 76 ++++++++++++++++++++++++- 6 files changed, 168 insertions(+), 42 deletions(-) create mode 100644 address/services/import_utils.py diff --git a/address/services/address_import.py b/address/services/address_import.py index 5b53d76..9c4b621 100644 --- a/address/services/address_import.py +++ b/address/services/address_import.py @@ -9,6 +9,7 @@ from address.constants import MUNICIPALITIES from address.models import Address, Municipality, Street +from address.services.import_utils import create_municipality, value_or_empty # Offset (in meters) from the middle of the street to the address at perpendicular to # the street. Setting this to zero makes the address locations lie exactly on the street @@ -76,11 +77,23 @@ def _build_addresses_from_feature(self, feature: Feature) -> list[Address]: return [] # Create the municipality and street if they don't exist yet - municipality = self._create_municipality(feature["KUNTAKOODI"].value) + municipality_code = int(value_or_empty(feature, "KUNTAKOODI")) + municipality_fi, municipality_sv = MUNICIPALITIES[self.province][ + municipality_code + ] + municipality = create_municipality( + municipality_code, municipality_fi, municipality_sv, municipality_fi + ) + + street_name_fi = value_or_empty(feature, "TIENIMI_SU") + street_name_sv = value_or_empty(feature, "TIENIMI_RU") or street_name_fi + if not street_name_fi: + street_name_fi = street_name_sv street = self._create_street( - feature["TIENIMI_SU"].value or "", - feature["TIENIMI_RU"].value or "", + street_name_fi, + street_name_sv, + street_name_fi, municipality, ) @@ -216,30 +229,16 @@ def _find_normal( return -x, -y return x, y - @lru_cache(maxsize=None) # noqa: B019 - def _create_municipality(self, municipality_id: int) -> Municipality: - """Create a new municipality if it does not exist already, and return it.""" - municipality_fi, municipality_sv = MUNICIPALITIES[self.province][ - municipality_id - ] - municipality, _ = Municipality.objects.get_or_create(id=municipality_fi.lower()) - municipality.set_current_language("sv") - municipality.name = municipality_sv - municipality.set_current_language("fi") - municipality.name = municipality_fi - municipality.code = municipality_id - - municipality.save() - return municipality - @lru_cache(maxsize=None) # noqa: B019 def _create_street( - self, name_fi: str, name_sv: str, municipality: Municipality + self, name_fi: str, name_sv: str, name_en: str, municipality: Municipality ) -> Street: """Create a new street if it does not exist already, and return it.""" street, _ = Street.objects.translated(name=name_fi).get_or_create( municipality=municipality ) + street.set_current_language("en") + street.name = name_en street.set_current_language("sv") street.name = name_sv street.set_current_language("fi") @@ -248,9 +247,13 @@ def _create_street( return street def _has_required_fields(self, feature: Feature) -> bool: - """Check whether the feature contains street name and first/last numbers.""" + """Check whether the feature contains street name, first/last numbers + and municipality code. + """ street_keys = ["TIENIMI_SU", "TIENIMI_RU"] number_keys = ["ENS_TALO_O", "ENS_TALO_V", "VIIM_TAL_O", "VIIM_TAL_V"] + municipality_key = "KUNTAKOODI" has_street = any(feature[key].value for key in street_keys) has_number = any(feature[key].value for key in number_keys) - return has_street and has_number + has_municipality = feature[municipality_key].value + return has_street and has_number and has_municipality diff --git a/address/services/import_utils.py b/address/services/import_utils.py new file mode 100644 index 0000000..58d560d --- /dev/null +++ b/address/services/import_utils.py @@ -0,0 +1,32 @@ +from functools import lru_cache + +from django.contrib.gis.gdal.feature import Feature + +from address.models import Municipality + + +def value_or_empty(feature: Feature, key: str) -> str: + field = feature[key] + if not field or field.value is None: + return "" + value = field.value + if isinstance(value, int): + return str(value) + return value.strip() if isinstance(value, str) else str(value) + + +@lru_cache(maxsize=None) # noqa: B019 +def create_municipality( + code: int, municipality_fi: str, municipality_sv: str, municipality_en: str +) -> Municipality: + """Create a new municipality if it does not exist already, and return it.""" + municipality, _ = Municipality.objects.get_or_create(id=municipality_fi.lower()) + municipality.set_current_language("en") + municipality.name = municipality_en + municipality.set_current_language("sv") + municipality.name = municipality_sv + municipality.set_current_language("fi") + municipality.name = municipality_fi + municipality.code = code + municipality.save() + return municipality diff --git a/address/services/municipality_import.py b/address/services/municipality_import.py index 83506ec..9029be8 100644 --- a/address/services/municipality_import.py +++ b/address/services/municipality_import.py @@ -5,7 +5,8 @@ from django.contrib.gis.gdal.feature import Feature from django.contrib.gis.geos import MultiPolygon -from ..models import Address, Municipality +from ..models import Address +from .import_utils import create_municipality, value_or_empty logger = logging.getLogger(__name__) @@ -22,18 +23,22 @@ def import_municipalities(self, features: Iterable[Feature]) -> int: # Update municipality for all addresses within each postal code area for feature in features: geometry = feature.geom.geos - code = feature["NATCODE"].value - name_fi = feature["NAMEFIN"].value - name_sv = feature["NAMESWE"].value - municipality_id = name_fi.lower() - - municipality, _ = Municipality.objects.get_or_create(pk=municipality_id) - municipality.set_current_language("sv") - municipality.name = name_sv - municipality.set_current_language("fi") - municipality.name = name_fi - municipality.id = municipality_id - municipality.code = code + code = int(value_or_empty(feature, "NATCODE")) + name_fi = value_or_empty(feature, "NAMEFIN") + name_sv = value_or_empty(feature, "NAMESWE") or name_fi + if not name_fi and not name_sv: + logger.warning(f"Municipality with code {code} has no name, skipping") + continue + + if not name_fi: + name_fi = name_sv + + municipality = create_municipality( + code=code, + municipality_fi=name_fi, + municipality_sv=name_sv, + municipality_en=name_fi, + ) if geometry.geom_type == "Polygon": area = MultiPolygon(geometry, srid=3067) else: diff --git a/address/services/postal_code_import.py b/address/services/postal_code_import.py index 0a9625e..e18ed48 100644 --- a/address/services/postal_code_import.py +++ b/address/services/postal_code_import.py @@ -8,6 +8,7 @@ from address.constants import MUNICIPALITIES from ..models import Address, Municipality, PostalCodeArea +from .import_utils import value_or_empty logger = logging.getLogger(__name__) @@ -43,16 +44,21 @@ def import_postal_codes(self, features: Iterable[Feature]) -> int: # Update postal code for all addresses within each postal code area for feature in features: geometry = feature.geom.geos - postal_code = ( - feature["posti_alue"].value.strip() if feature["posti_alue"] else None - ) - post_office_fi = feature["nimi"].value.strip() if feature["nimi"] else None - post_office_sv = feature["namn"].value.strip() if feature["namn"] else None - municipality_id = feature["kuntanro"].value + postal_code = value_or_empty(feature, "posti_alue") + if not postal_code: + logger.warning("Postal code area with no postal code, skipping") + continue + post_office_fi = value_or_empty(feature, "nimi") + post_office_sv = value_or_empty(feature, "namn") or post_office_fi + if not post_office_fi: + post_office_fi = post_office_sv + municipality_id = int(value_or_empty(feature, "kuntanro")) postal_code_area, _ = PostalCodeArea.objects.get_or_create( postal_code=postal_code ) + postal_code_area.set_current_language("en") + postal_code_area.name = post_office_fi postal_code_area.set_current_language("sv") postal_code_area.name = post_office_sv postal_code_area.set_current_language("fi") diff --git a/address/tests/factories.py b/address/tests/factories.py index c1f8bd7..4c32832 100644 --- a/address/tests/factories.py +++ b/address/tests/factories.py @@ -14,6 +14,8 @@ def post(self, *_, **__): name = self.name self.set_current_language("sv") self.name = name + self.set_current_language("en") + self.name = name self.save() class Meta: @@ -31,6 +33,8 @@ def post(self, *_, **__): name = self.name self.set_current_language("sv") self.name = name + self.set_current_language("en") + self.name = name self.save() class Meta: @@ -48,6 +52,8 @@ def post(self, *_, **__): name = self.name self.set_current_language("sv") self.name = name + self.set_current_language("en") + self.name = name self.save() class Meta: diff --git a/address/tests/test_models.py b/address/tests/test_models.py index 3a218ce..dd08b90 100644 --- a/address/tests/test_models.py +++ b/address/tests/test_models.py @@ -1,6 +1,11 @@ from pytest import mark -from .factories import AddressFactory, MunicipalityFactory, StreetFactory +from .factories import ( + AddressFactory, + MunicipalityFactory, + PostalCodeAreaFactory, + StreetFactory, +) @mark.django_db @@ -9,12 +14,81 @@ def test_municipality_string_has_name(): assert str(municipality) == municipality.name +@mark.django_db +def test_municipality_has_english_translation(): + municipality = MunicipalityFactory() + municipality.set_current_language("en") + assert municipality.name is not None + assert municipality.has_translation("en") + + +@mark.django_db +def test_municipality_english_translation_can_be_different(): + municipality = MunicipalityFactory() + municipality.set_current_language("fi") + municipality.name = "Helsinki" + municipality.set_current_language("en") + municipality.name = "Helsinki City" + municipality.save() + + municipality.set_current_language("fi") + assert municipality.name == "Helsinki" + municipality.set_current_language("en") + assert municipality.name == "Helsinki City" + + @mark.django_db def test_street_string_has_name(): street = StreetFactory() assert str(street) == street.name +@mark.django_db +def test_street_has_english_translation(): + street = StreetFactory() + street.set_current_language("en") + assert street.name is not None + assert street.has_translation("en") + + +@mark.django_db +def test_street_english_translation_can_be_different(): + street = StreetFactory() + street.set_current_language("fi") + street.name = "Mannerheimintie" + street.set_current_language("en") + street.name = "Mannerheim Road" + street.save() + + street.set_current_language("fi") + assert street.name == "Mannerheimintie" + street.set_current_language("en") + assert street.name == "Mannerheim Road" + + +@mark.django_db +def test_postal_code_area_has_english_translation(): + postal_code_area = PostalCodeAreaFactory() + postal_code_area.set_current_language("en") + assert postal_code_area.name is not None + assert postal_code_area.has_translation("en") + + +@mark.django_db +def test_postal_code_area_english_translation_can_be_different(): + postal_code_area = PostalCodeAreaFactory() + postal_code_area.set_current_language("fi") + postal_code_area.name = "Keskusta" + postal_code_area.set_current_language("en") + postal_code_area.name = "City Center" + postal_code_area.save() + + postal_code_area.set_current_language("fi") + assert postal_code_area.name == "Keskusta" + postal_code_area.set_current_language("en") + assert postal_code_area.name == "City Center" + + @mark.django_db def test_address_string_has_street_number_number_end_letter_municipality(): address = AddressFactory() From 0b4c7f175475fe0654ed7129b02e85b631432e58 Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Wed, 18 Feb 2026 13:37:05 +0200 Subject: [PATCH 04/18] feat: use explicit source srids Updates: - Use explicit source srid constants to make code more readable. - Rely on address data update only by default. - Rename "postal code import" to "postal code area import". Refs: RATYK-163 --- .../management/commands/import_addresses.py | 5 ++- ...l_codes.py => import_postal_code_areas.py} | 8 ++-- address/services/address_import.py | 10 ++--- address/services/municipality_import.py | 4 +- ...e_import.py => postal_code_area_import.py} | 37 ++++++------------- address/tests/test_address_import_service.py | 10 ++--- .../tests/test_postal_code_import_service.py | 4 +- 7 files changed, 32 insertions(+), 46 deletions(-) rename address/management/commands/{import_postal_codes.py => import_postal_code_areas.py} (81%) rename address/services/{postal_code_import.py => postal_code_area_import.py} (68%) diff --git a/address/management/commands/import_addresses.py b/address/management/commands/import_addresses.py index f9be8fc..a6fd37e 100644 --- a/address/management/commands/import_addresses.py +++ b/address/management/commands/import_addresses.py @@ -26,7 +26,10 @@ class Command(BaseCommand): - help = "Imports addresses from the given Digiroad shapefiles." + help = ( + "Imports addresses from the given Digiroad shapefiles" + "for the specified province." + ) def add_arguments(self, parser) -> None: parser.add_argument("files", nargs="+", type=Path) diff --git a/address/management/commands/import_postal_codes.py b/address/management/commands/import_postal_code_areas.py similarity index 81% rename from address/management/commands/import_postal_codes.py rename to address/management/commands/import_postal_code_areas.py index 70115ca..f02a8f3 100644 --- a/address/management/commands/import_postal_codes.py +++ b/address/management/commands/import_postal_code_areas.py @@ -9,11 +9,11 @@ from django.contrib.gis.gdal import DataSource from django.core.management.base import BaseCommand -from ...services.postal_code_import import PostalCodeImporter +from ...services.postal_code_area_import import PostalCodeAreaImporter class Command(BaseCommand): - help = "Imports postal codes from the given Paavo shapefiles." + help = "Imports postal code areas from the given Paavo shapefiles." def add_arguments(self, parser) -> None: parser.add_argument("province") @@ -21,13 +21,13 @@ def add_arguments(self, parser) -> None: def handle(self, *args, **options) -> None: start_time = time() - importer = PostalCodeImporter(options["province"]) + importer = PostalCodeAreaImporter(options["province"]) paths = options["files"] num_addresses_updated = 0 for path in paths: self.stdout.write(f"Reading data from {path}.") for layer in DataSource(path, encoding="latin-1"): - num_addresses_updated += importer.import_postal_codes(layer) + num_addresses_updated += importer.import_postal_code_areas(layer) self.stdout.write( self.style.SUCCESS( f"{num_addresses_updated} addresses updated " diff --git a/address/services/address_import.py b/address/services/address_import.py index 9c4b621..a703736 100644 --- a/address/services/address_import.py +++ b/address/services/address_import.py @@ -20,6 +20,8 @@ # Write addresses to database after this many instances have been generated ADDRESS_BATCH_SIZE = 1000 +ADDRESS_SOURCE_SRID = 3067 + class AddressImporter: def __init__(self, province: str = None): @@ -31,13 +33,8 @@ def delete_address_data(self) -> None: muni_ids = [ muni[1][0].lower() for muni in MUNICIPALITIES[self.province].items() ] - Municipality.objects.filter(id__in=muni_ids).delete() Street.objects.filter(addresses__municipality_id__in=muni_ids).delete() Address.objects.filter(municipality_id__in=muni_ids).delete() - else: - Municipality.objects.all().delete() - Street.objects.all().delete() - Address.objects.all().delete() def import_addresses(self, features: Iterable[Feature]) -> int: """Create addresses from the given features.""" @@ -65,8 +62,7 @@ def _transform_address_locations(self, addresses: list[Address]) -> None: if not addresses: return locations = [address.location for address in addresses] - srid = locations[0].srid # Each location has the same SRID - transformed_points = MultiPoint(locations, srid=srid) + transformed_points = MultiPoint(locations, srid=ADDRESS_SOURCE_SRID) transformed_points.transform(settings.PROJECTION_SRID) for i, address in enumerate(addresses): address.location = transformed_points[i] diff --git a/address/services/municipality_import.py b/address/services/municipality_import.py index 9029be8..1c00b05 100644 --- a/address/services/municipality_import.py +++ b/address/services/municipality_import.py @@ -10,6 +10,8 @@ logger = logging.getLogger(__name__) +MUNICIPALITY_SOURCE_SRID = 3067 + class MunicipalityImporter: def import_municipalities(self, features: Iterable[Feature]) -> int: @@ -40,7 +42,7 @@ def import_municipalities(self, features: Iterable[Feature]) -> int: municipality_en=name_fi, ) if geometry.geom_type == "Polygon": - area = MultiPolygon(geometry, srid=3067) + area = MultiPolygon(geometry, srid=MUNICIPALITY_SOURCE_SRID) else: area = geometry area.transform(settings.PROJECTION_SRID) diff --git a/address/services/postal_code_import.py b/address/services/postal_code_area_import.py similarity index 68% rename from address/services/postal_code_import.py rename to address/services/postal_code_area_import.py index e18ed48..07e9eb1 100644 --- a/address/services/postal_code_import.py +++ b/address/services/postal_code_area_import.py @@ -5,43 +5,29 @@ from django.contrib.gis.gdal.feature import Feature from django.contrib.gis.geos import MultiPolygon -from address.constants import MUNICIPALITIES - from ..models import Address, Municipality, PostalCodeArea from .import_utils import value_or_empty logger = logging.getLogger(__name__) +POSTAL_CODE_AREA_SOURCE_SRID = 3067 + -class PostalCodeImporter: +class PostalCodeAreaImporter: def __init__(self, province: str = None): self.province = province - def import_postal_codes(self, features: Iterable[Feature]) -> int: + def import_postal_code_areas(self, features: Iterable[Feature]) -> int: """ - Go through the given postal code area features, find all addresses - that are within each area, and update the postal code of the address - accordingly. + Go through the given postal code area features + and create/update PostalCodeArea objects for each area. + + Also find all addresses that are within each area + and update the postal code area of the address. """ total_addresses_updated = 0 - if self.province in MUNICIPALITIES.keys(): - muni_ids = [ - muni[1][0].lower() for muni in MUNICIPALITIES[self.province].items() - ] - # Clear existing postal code areas for one province - Address.objects.filter( - postal_code_area__isnull=False, municipality__id__in=muni_ids - ).update( - postal_code_area=None, - ) - else: - # Clear existing postal code areas for all provinces - Address.objects.filter(postal_code_area__isnull=False).update( - postal_code_area=None, - ) - - # Update postal code for all addresses within each postal code area + # Update postal code area for all addresses within each postal code area for feature in features: geometry = feature.geom.geos postal_code = value_or_empty(feature, "posti_alue") @@ -71,7 +57,7 @@ def import_postal_codes(self, features: Iterable[Feature]) -> int: postal_code_area.municipality = municipality if geometry.geom_type == "Polygon": - area = MultiPolygon(geometry, srid=3067) + area = MultiPolygon(geometry, srid=POSTAL_CODE_AREA_SOURCE_SRID) else: area = geometry area.transform(settings.PROJECTION_SRID) @@ -79,7 +65,6 @@ def import_postal_codes(self, features: Iterable[Feature]) -> int: postal_code_area.save() addresses = Address.objects.filter( - postal_code_area__isnull=True, location__intersects=geometry, ) diff --git a/address/tests/test_address_import_service.py b/address/tests/test_address_import_service.py index da84e88..52ac2b2 100644 --- a/address/tests/test_address_import_service.py +++ b/address/tests/test_address_import_service.py @@ -32,11 +32,11 @@ @mark.django_db(transaction=True) def test_delete_address_data_removes_municipalities_streets_and_addresses(): - MunicipalityFactory() - StreetFactory() - AddressFactory() - AddressImporter().delete_address_data() - assert not Municipality.objects.exists() + # Create municipality with ID matching one from uusimaa province + municipality = MunicipalityFactory(id="helsinki", code="91") + street = StreetFactory(municipality=municipality) + AddressFactory(municipality=municipality, street=street) + AddressImporter(province=TEST_PROVINCE).delete_address_data() assert not Street.objects.exists() assert not Address.objects.exists() diff --git a/address/tests/test_postal_code_import_service.py b/address/tests/test_postal_code_import_service.py index 9e6c4cc..0ad640b 100644 --- a/address/tests/test_postal_code_import_service.py +++ b/address/tests/test_postal_code_import_service.py @@ -33,7 +33,7 @@ def test_import_postal_codes(): "kuntanro": municipality.code, } ) - PostalCodeImporter().import_postal_codes([feature]) + PostalCodeImporter().import_postal_code_areas([feature]) address.refresh_from_db() assert address.postal_code_area.postal_code == postal_code address.postal_code_area.set_current_language("sv") @@ -57,7 +57,7 @@ def test_import_postal_codes_does_not_update_postal_code_if_outside(paavo_shapef "kuntanro": 91, } ) - PostalCodeImporter().import_postal_codes([feature]) + PostalCodeImporter().import_postal_code_areas([feature]) address.refresh_from_db() assert not address.postal_code_area From f6b45fe2343e65d20deff2b7a5714b8bbe4f572f Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Wed, 18 Feb 2026 13:39:27 +0200 Subject: [PATCH 05/18] feat: add delete address data management command Add a new Django management command to delete all address data from the database (municipalities, streets, addresses and postal code areas). Refs: RATYK-163 --- .../commands/delete_address_data.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 address/management/commands/delete_address_data.py diff --git a/address/management/commands/delete_address_data.py b/address/management/commands/delete_address_data.py new file mode 100644 index 0000000..d9c2af4 --- /dev/null +++ b/address/management/commands/delete_address_data.py @@ -0,0 +1,26 @@ +""" +Deletes all address data from the database. +""" + +from time import time + +from django.core.management.base import BaseCommand + +from address.models import Address, Municipality, PostalCodeArea, Street + + +class Command(BaseCommand): + help = "Deletes all address data from the database." + + def handle(self, *args, **options) -> None: + start_time = time() + self.stdout.write("Deleting all address data...") + Municipality.objects.all().delete() + Street.objects.all().delete() + Address.objects.all().delete() + PostalCodeArea.objects.all().delete() + self.stdout.write( + self.style.SUCCESS( + f"Address data deleted in {time() - start_time:.0f} seconds." + ) + ) From f730fd028a3f6fa2c1aba4adfbf1baa84df9bbda Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Fri, 20 Feb 2026 08:11:37 +0200 Subject: [PATCH 06/18] feat: add delete-address-data.sh convenience script - Add shell script wrapper for delete_address_data management command - Include safety confirmation prompt before deletion - Follow existing script conventions and shellcheck compliance - Make script executable with proper permissions Refs: RATYK-163 --- scripts/delete-address-data.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 scripts/delete-address-data.sh diff --git a/scripts/delete-address-data.sh b/scripts/delete-address-data.sh new file mode 100755 index 0000000..a496c51 --- /dev/null +++ b/scripts/delete-address-data.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +set -e + +echo "WARNING: This will delete all address data from the database!" +echo "This includes:" +echo " - All addresses" +echo " - All streets" +echo " - All municipalities" +echo "" +echo "Press Ctrl+C to cancel, or press Enter to continue..." +read -r _ + +SCRIPT_DIR="$(cd "$(dirname "$0")"; pwd)" +python "$SCRIPT_DIR/../manage.py" delete_address_data From 698d3958dd41c16ddc9bf446e90d2eb98b561020 Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Wed, 18 Feb 2026 13:43:27 +0200 Subject: [PATCH 07/18] fix: update import scripts Digiroad-import: - Update URL - Allow curl to follow redirects - Fix multiple shapefile argument support Paavo-import: - Use WFS 2.0 - Use typeName 'pno_meri_2026' - Update management command name Update documentation from the underlying management commands. Refs: RATYK-163 --- address/management/commands/import_municipalities.py | 10 +++++++--- .../management/commands/import_postal_code_areas.py | 5 ++++- scripts/import-digiroad-data.sh | 7 ++++--- scripts/import-paavo-data.sh | 6 +++--- 4 files changed, 18 insertions(+), 10 deletions(-) mode change 100644 => 100755 scripts/import-digiroad-data.sh mode change 100644 => 100755 scripts/import-paavo-data.sh diff --git a/address/management/commands/import_municipalities.py b/address/management/commands/import_municipalities.py index f26e11d..186c278 100644 --- a/address/management/commands/import_municipalities.py +++ b/address/management/commands/import_municipalities.py @@ -1,6 +1,10 @@ """ -This management command imports municipalities from MML data: -https://www.maanmittauslaitos.fi/kartat-ja-paikkatieto/ammattilaiskayttajille/tuotekuvaukset/hallinnolliset-aluejaot-vektori +This management command imports municipalities from NLS data: +https://www.maanmittauslaitos.fi/en/maps-and-spatial-data/datasets-and-interfaces/product-descriptions/division-administrative-areas-vector + +Data requires manual downloading and unzipping. The shapefiles can be given to +the management command as arguments: + python manage.py import_municipalities /SuomenKuntajako__10k.shp """ from pathlib import Path @@ -13,7 +17,7 @@ class Command(BaseCommand): - help = "Import municipalities from the given MML shapefiles." + help = "Import municipalities from the given NLS shapefiles." def add_arguments(self, parser) -> None: parser.add_argument("files", nargs="+", type=Path) diff --git a/address/management/commands/import_postal_code_areas.py b/address/management/commands/import_postal_code_areas.py index f02a8f3..0d9713b 100644 --- a/address/management/commands/import_postal_code_areas.py +++ b/address/management/commands/import_postal_code_areas.py @@ -1,5 +1,8 @@ """ -This management command imports addresses from Paavo data: +This management command imports addresses from Paavo data +(Statistics Finland's open data on postal code areas). + +The data can be downloaded from: https://www.stat.fi/org/avoindata/paikkatietoaineistot/paavo.html """ diff --git a/scripts/import-digiroad-data.sh b/scripts/import-digiroad-data.sh old mode 100644 new mode 100755 index e266219..421a932 --- a/scripts/import-digiroad-data.sh +++ b/scripts/import-digiroad-data.sh @@ -23,7 +23,7 @@ fi echo "Importing Digiroad data for province $1."; -DATA_URL="https://ava.vaylapilvi.fi/ava/Tie/Digiroad/Aineistojulkaisut/latest/Maakuntajako_digiroad_R/${package_name}" +DATA_URL="https://aineistot.vayla.fi/spa/ava/Tie/Digiroad/Aineistojulkaisut/latest/Maakuntajako_digiroad_R/${package_name}" # Directory where the data will be downloaded and extracted DATA_DIR=/tmp/digiroad @@ -40,7 +40,7 @@ CONVERTED_DIR=$DATA_DIR/converted # Download the source data mkdir -p $DATA_DIR -curl -o $DATA_DIR/data.zip $DATA_URL +curl --proto "=https" --tlsv1.2 -sSf -L -o $DATA_DIR/data.zip $DATA_URL # Extract the shapefiles from the archive rm -rf $EXTRACTED_DIR @@ -61,4 +61,5 @@ done # Run the management command with all the shapefiles as arguments SCRIPT_DIR="$(cd "$(dirname "$0")"; pwd)" -python "$SCRIPT_DIR/../manage.py" import_addresses "$(find $CONVERTED_DIR -type f -name "DR_LINKK*.shp")" "$1" +# shellcheck disable=SC2046 +python "$SCRIPT_DIR/../manage.py" import_addresses $(find $CONVERTED_DIR -type f -name "DR_LINKK*.shp") "$1" diff --git a/scripts/import-paavo-data.sh b/scripts/import-paavo-data.sh old mode 100644 new mode 100755 index 0a8893f..61fa313 --- a/scripts/import-paavo-data.sh +++ b/scripts/import-paavo-data.sh @@ -24,7 +24,7 @@ fi echo "Importing Paavo data for province $1."; # URL to Paavo WFS service -DATA_URL="https://geo.stat.fi/geoserver/wfs?SERVICE=wfs&version=1.0.0&request=GetFeature&srsName=EPSG:3067&outputFormat=SHAPE-ZIP&typeNames=pno_meri_2024&bbox=${bbox}" +DATA_URL="https://geo.stat.fi/geoserver/wfs?SERVICE=wfs&version=2.0.0&request=GetFeature&srsName=EPSG:3067&outputFormat=SHAPE-ZIP&typeNames=pno_meri_2026&bbox=${bbox}" DATA_DIR=/tmp/paavo @@ -33,7 +33,7 @@ EXTRACTED_DIR=$DATA_DIR/extracted # Download the source data mkdir -p $DATA_DIR -curl "$DATA_URL" -o $DATA_DIR/data.zip +curl --proto "=https" --tlsv1.2 -sSf "$DATA_URL" -o $DATA_DIR/data.zip # Extract the files from the archive rm -rf $EXTRACTED_DIR @@ -41,4 +41,4 @@ unzip $DATA_DIR/data.zip -d $EXTRACTED_DIR # Run the management command with all the shapefiles as arguments SCRIPT_DIR="$(cd "$(dirname "$0")"; pwd)" -python "$SCRIPT_DIR/../manage.py" import_postal_codes "$1" "$(find $EXTRACTED_DIR -type f -name "*.shp")" +python "$SCRIPT_DIR/../manage.py" import_postal_code_areas "$1" "$(find $EXTRACTED_DIR -type f -name "*.shp")" From 56ffacb7fd767670cceb09e54fb1c97c6f8a04b6 Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Wed, 18 Feb 2026 14:01:17 +0200 Subject: [PATCH 08/18] feat!: remove municipality from PostalCodeArea BREAKING CHANGE: municipality-field is no longer accessible in PostalCodeArea after this change. Details: - Update PostalCodeArea-model - Add migration - Update API - Update imports - Update tests - Update Django Admin Refs: RATYK-159 --- address/admin.py | 2 +- address/api/serializers.py | 2 +- address/api/views.py | 20 +---------- ...0010_remove_postalcodearea_municipality.py | 16 +++++++++ address/models.py | 8 ----- address/services/postal_code_area_import.py | 9 +---- address/tests/factories.py | 1 - address/tests/test_api_serializers.py | 2 -- address/tests/test_api_views.py | 34 ------------------- .../tests/test_import_postal_codes_command.py | 4 +-- .../tests/test_postal_code_import_service.py | 8 ++--- 11 files changed, 25 insertions(+), 81 deletions(-) create mode 100644 address/migrations/0010_remove_postalcodearea_municipality.py diff --git a/address/admin.py b/address/admin.py index 007ca1a..63157c4 100644 --- a/address/admin.py +++ b/address/admin.py @@ -33,7 +33,7 @@ def get_queryset(self, request): @admin.register(PostalCodeArea) class PostalCodeAreaAdmin(TranslatableAdmin): - list_display = ("postal_code", "name", "municipality") + list_display = ("postal_code", "name") ordering = ("postal_code",) search_fields = ("postal_code",) diff --git a/address/api/serializers.py b/address/api/serializers.py index 0d70138..91e30c2 100644 --- a/address/api/serializers.py +++ b/address/api/serializers.py @@ -79,7 +79,7 @@ class Meta: class PostalCodeAreaSerializer(TranslatedAreaModelSerializer): class Meta: model = PostalCodeArea - fields = ["postal_code", "translations", "area", "municipality"] + fields = ["postal_code", "translations", "area"] class AddressSerializer(serializers.ModelSerializer): diff --git a/address/api/views.py b/address/api/views.py index 8ade9f0..aa50ec9 100644 --- a/address/api/views.py +++ b/address/api/views.py @@ -276,9 +276,7 @@ def _filter_by_location(self, addresses: QuerySet) -> QuerySet: @extend_schema_view( - list=extend_schema( - parameters=_area_parameters + _postal_code_parameters + _municipality_parameters - ), + list=extend_schema(parameters=_area_parameters + _postal_code_parameters), retrieve=extend_schema(parameters=_area_parameters), ) class PostalCodeAreaViewSet(ReadOnlyModelViewSet): @@ -287,26 +285,10 @@ class PostalCodeAreaViewSet(ReadOnlyModelViewSet): def get_queryset(self) -> QuerySet: areas = self.queryset - areas = self._filter_by_municipality(areas) - areas = self._filter_by_municipality_code(areas) areas = self._filter_by_postal_code(areas) areas = self._filter_by_post_office(areas) return areas - def _filter_by_municipality(self, areas: QuerySet) -> QuerySet: - municipality = self.request.query_params.get("municipality") - if municipality is None: - return areas - return areas.filter( - municipality__translations__name__iexact=municipality - ).distinct() - - def _filter_by_municipality_code(self, areas: QuerySet) -> QuerySet: - municipality_code = self.request.query_params.get("municipalitycode") - if municipality_code is None: - return areas - return areas.filter(municipality__code__iexact=municipality_code).distinct() - def _filter_by_postal_code(self, areas: QuerySet) -> QuerySet: postal_code = self.request.query_params.get("postalcode") if postal_code is None: diff --git a/address/migrations/0010_remove_postalcodearea_municipality.py b/address/migrations/0010_remove_postalcodearea_municipality.py new file mode 100644 index 0000000..a3b8283 --- /dev/null +++ b/address/migrations/0010_remove_postalcodearea_municipality.py @@ -0,0 +1,16 @@ +# Generated by Django on 2026-02-18 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("address", "0009_update_model_verbose_names"), + ] + + operations = [ + migrations.RemoveField( + model_name="postalcodearea", + name="municipality", + ), + ] diff --git a/address/models.py b/address/models.py index acac48c..f17458d 100644 --- a/address/models.py +++ b/address/models.py @@ -43,14 +43,6 @@ class PostalCodeArea(TranslatableModel): postal_code = models.CharField( _("Postal code"), max_length=5, null=True, blank=True ) - municipality = models.ForeignKey( - Municipality, - models.CASCADE, - db_index=True, - null=True, - blank=True, - verbose_name=_("Municipality"), - ) translations = TranslatedFields( name=models.CharField(_("Name"), max_length=100, null=True, blank=True) ) diff --git a/address/services/postal_code_area_import.py b/address/services/postal_code_area_import.py index 07e9eb1..2072090 100644 --- a/address/services/postal_code_area_import.py +++ b/address/services/postal_code_area_import.py @@ -5,7 +5,7 @@ from django.contrib.gis.gdal.feature import Feature from django.contrib.gis.geos import MultiPolygon -from ..models import Address, Municipality, PostalCodeArea +from ..models import Address, PostalCodeArea from .import_utils import value_or_empty logger = logging.getLogger(__name__) @@ -38,7 +38,6 @@ def import_postal_code_areas(self, features: Iterable[Feature]) -> int: post_office_sv = value_or_empty(feature, "namn") or post_office_fi if not post_office_fi: post_office_fi = post_office_sv - municipality_id = int(value_or_empty(feature, "kuntanro")) postal_code_area, _ = PostalCodeArea.objects.get_or_create( postal_code=postal_code @@ -50,12 +49,6 @@ def import_postal_code_areas(self, features: Iterable[Feature]) -> int: postal_code_area.set_current_language("fi") postal_code_area.name = post_office_fi - try: - municipality = Municipality.objects.filter(code=municipality_id).first() - except Municipality.DoesNotExist: - municipality = None - postal_code_area.municipality = municipality - if geometry.geom_type == "Polygon": area = MultiPolygon(geometry, srid=POSTAL_CODE_AREA_SOURCE_SRID) else: diff --git a/address/tests/factories.py b/address/tests/factories.py index 4c32832..21784ba 100644 --- a/address/tests/factories.py +++ b/address/tests/factories.py @@ -45,7 +45,6 @@ class Meta: class PostalCodeAreaFactory(DjangoModelFactory): postal_code = Faker("postcode", locale="fi_FI") name = Faker("street_name", locale="fi_FI") - municipality = SubFactory(MunicipalityFactory) @post_generation def post(self, *_, **__): diff --git a/address/tests/test_api_serializers.py b/address/tests/test_api_serializers.py index e85253a..5b93b24 100644 --- a/address/tests/test_api_serializers.py +++ b/address/tests/test_api_serializers.py @@ -43,7 +43,6 @@ def test_postal_code_area_serializer(): actual = serializer.to_representation(postal_code_area) assert actual == { "postal_code": postal_code_area.postal_code, - "municipality": postal_code_area.municipality.id, "area": None, "name": {t.language_code: t.name for t in postal_code_area.translations.all()}, } @@ -65,7 +64,6 @@ def test_address_serializer(): "letter": address.letter, "postal_code_area": { "postal_code": address.postal_code_area.postal_code, - "municipality": address.postal_code_area.municipality.id, "area": None, "name": { t.language_code: t.name diff --git a/address/tests/test_api_views.py b/address/tests/test_api_views.py index 331e751..a7b0284 100644 --- a/address/tests/test_api_views.py +++ b/address/tests/test_api_views.py @@ -252,40 +252,6 @@ def test_filter_addresses_returns_bad_request_if_distance_is_invalid( assert response.status_code == 400 -@mark.django_db -def test_filter_postal_area_codes_by_municipality(api_client: APIClient): - municipality = MunicipalityFactory(name="Helsinki") - match = PostalCodeAreaFactory(municipality=municipality) - serializer = PostalCodeAreaSerializer() - response = api_client.get( - reverse("address:postalcodearea-list"), {"municipality": municipality.name} - ) - assert response.status_code == 200 - assert response.data == { - "count": 1, - "next": None, - "previous": None, - "results": [serializer.to_representation(match)], - } - - -@mark.django_db -def test_filter_postal_area_codes_by_municipality_code(api_client: APIClient): - municipality = MunicipalityFactory(code="91") - match = PostalCodeAreaFactory(municipality=municipality) - serializer = PostalCodeAreaSerializer() - response = api_client.get( - reverse("address:postalcodearea-list"), {"municipalitycode": municipality.code} - ) - assert response.status_code == 200 - assert response.data == { - "count": 1, - "next": None, - "previous": None, - "results": [serializer.to_representation(match)], - } - - @mark.django_db def test_filter_postal_area_codes_by_postal_code(api_client: APIClient): PostalCodeAreaFactory(postal_code="99999") diff --git a/address/tests/test_import_postal_codes_command.py b/address/tests/test_import_postal_codes_command.py index 31f1d1e..552c500 100644 --- a/address/tests/test_import_postal_codes_command.py +++ b/address/tests/test_import_postal_codes_command.py @@ -13,10 +13,8 @@ def test_import_postal_codes_updates_postal_codes_from_shapefile(paavo_shapefile location=Point(x=24.9428, y=60.1666, srid=settings.PROJECTION_SRID), municipality=municipality, ) - call_command("import_postal_codes", None, [paavo_shapefile]) + call_command("import_postal_code_areas", None, [paavo_shapefile]) address.refresh_from_db() assert address.postal_code_area.postal_code == "00100" assert address.postal_code_area.name == "Helsinki Keskusta - Etu-Töölö" - assert address.postal_code_area.municipality is not None - assert address.postal_code_area.municipality.code == "91" assert address.postal_code_area.area is not None diff --git a/address/tests/test_postal_code_import_service.py b/address/tests/test_postal_code_import_service.py index 0ad640b..13206da 100644 --- a/address/tests/test_postal_code_import_service.py +++ b/address/tests/test_postal_code_import_service.py @@ -7,7 +7,7 @@ from pytest import mark from ..models import Municipality -from ..services.postal_code_import import PostalCodeImporter +from ..services.postal_code_area_import import PostalCodeAreaImporter from ..tests.factories import AddressFactory TEST_GEOMETRY = Polygon.from_bbox([24.9427, 60.1665, 24.9430, 60.1667]) @@ -33,14 +33,13 @@ def test_import_postal_codes(): "kuntanro": municipality.code, } ) - PostalCodeImporter().import_postal_code_areas([feature]) + PostalCodeAreaImporter().import_postal_code_areas([feature]) address.refresh_from_db() assert address.postal_code_area.postal_code == postal_code address.postal_code_area.set_current_language("sv") assert address.postal_code_area.name == post_office_sv address.postal_code_area.set_current_language("fi") assert address.postal_code_area.name == post_office - assert address.postal_code_area.municipality.code == str(municipality.code) @mark.django_db(transaction=True) @@ -48,6 +47,7 @@ def test_import_postal_codes_does_not_update_postal_code_if_outside(paavo_shapef address = AddressFactory( # Not within the 00100 postal code area location=Point(x=27, y=61, srid=settings.PROJECTION_SRID), + postal_code_area=None, ) feature = _mock_feature( { @@ -57,7 +57,7 @@ def test_import_postal_codes_does_not_update_postal_code_if_outside(paavo_shapef "kuntanro": 91, } ) - PostalCodeImporter().import_postal_code_areas([feature]) + PostalCodeAreaImporter().import_postal_code_areas([feature]) address.refresh_from_db() assert not address.postal_code_area From 78c37e173a8ba077705f2cfc0b006901c8257d97 Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Thu, 19 Feb 2026 14:11:37 +0200 Subject: [PATCH 09/18] deps: add requests-package Refs: RATYK-157 --- requirements.in | 1 + requirements.txt | 128 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 128 insertions(+), 1 deletion(-) diff --git a/requirements.in b/requirements.in index f3f805e..d51f375 100644 --- a/requirements.in +++ b/requirements.in @@ -9,6 +9,7 @@ djangorestframework-api-key djangorestframework-xml drf-spectacular psycopg[c] +requests sentry-sdk[django] whitenoise uwsgi diff --git a/requirements.txt b/requirements.txt index e3112de..31f1fda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,123 @@ certifi==2025.11.12 \ --hash=sha256:d8ab5478f2ecd78af242878415affce761ca6bc54a22a27e026d7c25357c3316 # via # pyproj + # requests # sentry-sdk +charset-normalizer==3.4.4 \ + --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad \ + --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 \ + --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 \ + --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 \ + --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc \ + --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 \ + --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 \ + --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d \ + --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f \ + --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 \ + --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 \ + --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 \ + --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 \ + --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af \ + --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 \ + --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 \ + --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 \ + --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 \ + --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e \ + --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 \ + --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 \ + --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c \ + --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 \ + --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 \ + --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 \ + --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 \ + --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 \ + --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 \ + --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 \ + --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 \ + --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede \ + --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf \ + --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a \ + --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc \ + --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 \ + --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 \ + --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db \ + --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 \ + --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 \ + --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed \ + --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 \ + --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 \ + --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e \ + --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef \ + --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 \ + --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 \ + --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 \ + --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d \ + --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 \ + --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f \ + --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf \ + --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 \ + --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 \ + --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 \ + --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa \ + --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 \ + --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c \ + --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb \ + --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc \ + --hash=sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a \ + --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec \ + --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc \ + --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac \ + --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e \ + --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 \ + --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 \ + --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 \ + --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d \ + --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 \ + --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 \ + --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 \ + --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 \ + --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a \ + --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 \ + --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 \ + --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 \ + --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 \ + --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf \ + --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 \ + --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 \ + --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac \ + --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e \ + --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 \ + --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c \ + --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 \ + --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 \ + --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e \ + --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 \ + --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 \ + --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 \ + --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 \ + --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 \ + --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 \ + --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 \ + --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd \ + --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 \ + --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 \ + --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d \ + --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 \ + --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 \ + --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 \ + --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 \ + --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a \ + --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d \ + --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d \ + --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f \ + --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 \ + --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 \ + --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 \ + --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 \ + --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 \ + --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \ + --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 + # via requests defusedxml==0.7.1 \ --hash=sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69 \ --hash=sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61 @@ -81,6 +197,10 @@ drf-spectacular==0.29.0 \ --hash=sha256:0a069339ea390ce7f14a75e8b5af4a0860a46e833fd4af027411a3e94fc1a0cc \ --hash=sha256:d1ee7c9535d89848affb4427347f7c4a22c5d22530b8842ef133d7b72e19b41a # via -r requirements.in +idna==3.11 \ + --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ + --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 + # via requests inflection==0.5.1 \ --hash=sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417 \ --hash=sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2 @@ -429,6 +549,10 @@ referencing==0.37.0 \ # via # jsonschema # jsonschema-specifications +requests==2.32.5 \ + --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ + --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf + # via -r requirements.in rpds-py==0.30.0 \ --hash=sha256:07ae8a593e1c3c6b82ca3292efbe73c30b61332fd612e05abee07c79359f292f \ --hash=sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136 \ @@ -569,7 +693,9 @@ uritemplate==4.2.0 \ urllib3==2.6.3 \ --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 - # via sentry-sdk + # via + # requests + # sentry-sdk uwsgi==2.0.31 \ --hash=sha256:e8f8b350ccc106ff93a65247b9136f529c14bf96b936ac5b264c6ff9d0c76257 # via -r requirements.in From 73db9d3ae6531577c5c6ac5b8e1cf64999ef59f1 Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Thu, 19 Feb 2026 14:23:21 +0200 Subject: [PATCH 10/18] perf(postal code areas search): use subquery instead of distinct Replace distinct() with subqueries for performance. Also update OpenApiParameters. Refs: RATYK-160 --- address/api/views.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/address/api/views.py b/address/api/views.py index aa50ec9..a25c0e5 100644 --- a/address/api/views.py +++ b/address/api/views.py @@ -22,7 +22,7 @@ name="streetname", location=OpenApiParameter.QUERY, description=( - "Street name in Finnish or Swedish. " + "Street name in Finnish, Swedish or English. " 'E.g. "Mannerheimintie" or "Mannerheimvägen".' ), required=False, @@ -97,8 +97,9 @@ OpenApiParameter( name="geom_format", location=OpenApiParameter.QUERY, - description="Area geometry format. " - "Available values : geojson, ewkt. Default : geojson", + description=( + "Area geometry format. Available values : geojson, ewkt. Default : geojson" + ), required=False, type=str, ), @@ -115,8 +116,13 @@ OpenApiParameter( name="postalcodearea", location=OpenApiParameter.QUERY, - description='Postal code area name in Finnish or Swedish. "' - 'E.g. "Lappohja" or "Lappvik"', + description=( + 'Postal code area name in Finnish, Swedish or English. "' + 'E.g. "Lappohja" or "Lappvik"' + ), + required=False, + type=str, + ), required=False, type=str, ), @@ -127,7 +133,8 @@ name="municipality", location=OpenApiParameter.QUERY, description=( - 'Municipality name in Finnish or Swedish. E.g. "Helsinki" or "Helsingfors".' + "Municipality name in Finnish, Swedish or English. " + 'E.g. "Helsinki" or "Helsingfors".' ), required=False, type=str, @@ -299,7 +306,12 @@ def _filter_by_post_office(self, areas: QuerySet) -> QuerySet: postal_code_area = self.request.query_params.get("postalcodearea") if postal_code_area is None: return areas - return areas.filter(translations__name__iexact=postal_code_area).distinct() + area_ids = list( + PostalCodeArea.objects.filter( + translations__name__iexact=postal_code_area + ).values_list("id", flat=True) + ) + return areas.filter(id__in=area_ids) @extend_schema_view( From 63ba27d78c32eddee34eb4ed839e2715aa663c0e Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Thu, 19 Feb 2026 14:42:00 +0200 Subject: [PATCH 11/18] feat: add multilingual post_office field and API filtering Details: - Add post_office field with fi/sv/en translations to PostalCodeArea model - Implement import command for Posti ZIP/DAT format with fallback logic - Add postoffice query parameter to Address and PostalCodeArea APIs - Update finnish translations - Add comprehensive test coverage Refs: RATYK-157 --- address/api/views.py | 44 +++- address/locale/fi/LC_MESSAGES/django.po | 11 +- .../commands/import_post_offices.py | 205 ++++++++++++++++++ .../0011_postalcodearea_post_office.py | 19 ++ address/models.py | 5 +- address/tests/test_api_serializers.py | 7 + .../tests/test_import_post_offices_command.py | 160 ++++++++++++++ address/tests/test_post_office_filtering.py | 116 ++++++++++ address/tests/test_postal_code_area_api.py | 47 ++++ scripts/import-post-office-data.sh | 27 +++ 10 files changed, 631 insertions(+), 10 deletions(-) create mode 100644 address/management/commands/import_post_offices.py create mode 100644 address/migrations/0011_postalcodearea_post_office.py create mode 100644 address/tests/test_import_post_offices_command.py create mode 100644 address/tests/test_post_office_filtering.py create mode 100644 address/tests/test_postal_code_area_api.py create mode 100755 scripts/import-post-office-data.sh diff --git a/address/api/views.py b/address/api/views.py index a25c0e5..cf87d70 100644 --- a/address/api/views.py +++ b/address/api/views.py @@ -123,6 +123,13 @@ required=False, type=str, ), + OpenApiParameter( + name="postoffice", + location=OpenApiParameter.QUERY, + description=( + "Post office name in Finnish, Swedish or English. " + 'E.g. "HELSINKI", "HELSINGFORS".' + ), required=False, type=str, ), @@ -174,6 +181,7 @@ def get_queryset(self) -> QuerySet: addresses = self._filter_by_municipality(addresses) addresses = self._filter_by_municipality_code(addresses) addresses = self._filter_by_postal_code(addresses) + addresses = self._filter_by_postal_code_area(addresses) addresses = self._filter_by_post_office(addresses) addresses = self._filter_by_bbox(addresses) addresses = self._filter_by_location(addresses) @@ -232,13 +240,27 @@ def _filter_by_postal_code(self, addresses: QuerySet) -> QuerySet: return addresses return addresses.filter(postal_code_area__postal_code__iexact=postal_code) - def _filter_by_post_office(self, addresses: QuerySet) -> QuerySet: + def _filter_by_postal_code_area(self, addresses: QuerySet) -> QuerySet: postal_code_area = self.request.query_params.get("postalcodearea") if postal_code_area is None: return addresses - return addresses.filter( - postal_code_area__translations__name__iexact=postal_code_area - ).distinct() + postal_code_area_ids = list( + PostalCodeArea.objects.filter( + translations__name__iexact=postal_code_area + ).values_list("id", flat=True) + ) + return addresses.filter(postal_code_area_id__in=postal_code_area_ids) + + def _filter_by_post_office(self, addresses: QuerySet) -> QuerySet: + post_office = self.request.query_params.get("postoffice") + if post_office is None: + return addresses + postal_code_area_ids = list( + PostalCodeArea.objects.filter( + translations__post_office__iexact=post_office + ).values_list("id", flat=True) + ) + return addresses.filter(postal_code_area_id__in=postal_code_area_ids) def _filter_by_bbox(self, addresses: QuerySet) -> QuerySet: bbox = self.request.query_params.get("bbox") @@ -293,6 +315,7 @@ class PostalCodeAreaViewSet(ReadOnlyModelViewSet): def get_queryset(self) -> QuerySet: areas = self.queryset areas = self._filter_by_postal_code(areas) + areas = self._filter_by_postal_code_area(areas) areas = self._filter_by_post_office(areas) return areas @@ -302,7 +325,7 @@ def _filter_by_postal_code(self, areas: QuerySet) -> QuerySet: return areas return areas.filter(postal_code__iexact=postal_code) - def _filter_by_post_office(self, areas: QuerySet) -> QuerySet: + def _filter_by_postal_code_area(self, areas: QuerySet) -> QuerySet: postal_code_area = self.request.query_params.get("postalcodearea") if postal_code_area is None: return areas @@ -313,6 +336,17 @@ def _filter_by_post_office(self, areas: QuerySet) -> QuerySet: ) return areas.filter(id__in=area_ids) + def _filter_by_post_office(self, areas: QuerySet) -> QuerySet: + post_office = self.request.query_params.get("postoffice") + if post_office is None: + return areas + area_ids = list( + PostalCodeArea.objects.filter( + translations__post_office__iexact=post_office + ).values_list("id", flat=True) + ) + return areas.filter(id__in=area_ids) + @extend_schema_view( list=extend_schema(parameters=_area_parameters), diff --git a/address/locale/fi/LC_MESSAGES/django.po b/address/locale/fi/LC_MESSAGES/django.po index 77540da..a2adfdd 100644 --- a/address/locale/fi/LC_MESSAGES/django.po +++ b/address/locale/fi/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-02-16 11:27+0200\n" +"POT-Creation-Date: 2026-02-19 11:39+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -18,15 +18,15 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" +msgid "Name" +msgstr "Nimi" + msgid "Id" msgstr "Tunniste" msgid "Municipality code" msgstr "Kuntakoodi" -msgid "Name" -msgstr "Nimi" - msgid "Area" msgstr "Alue" @@ -45,6 +45,9 @@ msgstr "Kadut" msgid "Postal code" msgstr "Postinumero" +msgid "Post office" +msgstr "Postitoimipaikka" + msgid "Postal code area" msgstr "Postinumeroalue" diff --git a/address/management/commands/import_post_offices.py b/address/management/commands/import_post_offices.py new file mode 100644 index 0000000..8490387 --- /dev/null +++ b/address/management/commands/import_post_offices.py @@ -0,0 +1,205 @@ +""" +This management command imports post office names from Posti's postal code data. + +Posti provides postal code data in a ZIP file containing a fixed-width DAT file. +The format is documented at: https://www.posti.fi/en/for-businesses/customer-support/postal-code-services + +Example URL: https://www.posti.fi/webpcode/PCF_20260218.zip + +Usage: + python manage.py import_post_offices path/to/PCF_YYYYMMDD.zip + +Or provide a URL to download the file: + python manage.py import_post_offices --url https://www.posti.fi/webpcode/PCF_20260218.zip +""" + +import logging +import zipfile +from io import BytesIO +from pathlib import Path +from time import time + +import requests +from django.core.management.base import BaseCommand, CommandError + +from address.models import PostalCodeArea + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Imports post office names from Posti's postal code data." + + def add_arguments(self, parser) -> None: + parser.add_argument( + "file", + nargs="?", + type=Path, + help="Path to the ZIP file containing postal code data", + ) + parser.add_argument( + "--url", + type=str, + help="URL to download the postal code data from", + ) + + def handle(self, *args, **options) -> None: + start_time = time() + + if options["url"]: + zip_data = self._download_file(options["url"]) + num_updated = self._import_from_zip_bytes(zip_data) + elif options["file"]: + file_path = options["file"] + if not file_path.exists(): + raise CommandError(f"File not found: {file_path}") + self.stdout.write(f"Reading data from {file_path}.") + num_updated = self._import_from_zip_file(file_path) + else: + raise CommandError( + "You must provide either a file path or a URL using --url option" + ) + + self.stdout.write( + self.style.SUCCESS( + f"{num_updated} postal code areas updated " + f"in {time() - start_time:.0f} seconds." + ) + ) + + def _download_file(self, url: str) -> bytes: + """Download the file from the given URL.""" + self.stdout.write(f"Downloading data from {url}...") + try: + response = requests.get(url, timeout=60) + response.raise_for_status() + return response.content + except requests.RequestException as e: + raise CommandError(f"Failed to download file from {url}: {e}") + + def _import_from_zip_file(self, zip_path: Path) -> int: + """Import post office names from a ZIP file.""" + try: + with zipfile.ZipFile(zip_path, "r") as zip_file: + return self._import_from_zip(zip_file) + except zipfile.BadZipFile as e: + raise CommandError(f"Invalid ZIP file: {e}") + + def _import_from_zip_bytes(self, zip_data: bytes) -> int: + """Import post office names from ZIP file bytes.""" + try: + with zipfile.ZipFile(BytesIO(zip_data), "r") as zip_file: + return self._import_from_zip(zip_file) + except zipfile.BadZipFile as e: + raise CommandError(f"Invalid ZIP file: {e}") + + def _import_from_zip(self, zip_file: zipfile.ZipFile) -> int: + """Import post office names from an open ZIP file.""" + dat_files = [name for name in zip_file.namelist() if name.endswith(".dat")] + + if not dat_files: + raise CommandError("No .dat file found in the ZIP archive") + + if len(dat_files) > 1: + self.stdout.write( + self.style.WARNING(f"Multiple .dat files found, using {dat_files[0]}") + ) + + dat_file = dat_files[0] + self.stdout.write(f"Processing {dat_file}...") + + with zip_file.open(dat_file) as f: + content = f.read() + text = content.decode("latin-1") + return self._import_post_offices(text) + + def _import_post_offices(self, content: str) -> int: + """Import post office names from the DAT file content.""" + num_updated = 0 + + for line_num, line in enumerate(content.splitlines(), start=1): + if len(line) < 78: + logger.debug(f"Line {line_num} too short, skipping") + continue + + parsed_data = self._parse_line(line, line_num) + if not parsed_data: + continue + + postal_code, post_office_fi, post_office_sv, post_office_en = parsed_data + num_updated += self._update_postal_code_areas( + postal_code, post_office_fi, post_office_sv, post_office_en + ) + + return num_updated + + def _parse_line(self, line: str, line_num: int) -> tuple[str, str, str, str] | None: + """Parse a line from the DAT file and return postal code + and post office names.""" + # Parse fixed-width format + # Positions 13-17: Postal code (5 digits) + # Positions 18-47: Post office name in Finnish (30 chars) + # Positions 48-77: Post office name in Swedish (30 chars) + postal_code = line[13:18].strip() + post_office_fi = line[18:48].strip() + post_office_sv = line[48:78].strip() + + if not postal_code: + logger.debug(f"Line {line_num}: Missing postal code") + return None + + if not post_office_fi and not post_office_sv: + logger.debug( + f"Line {line_num}: Missing both Finnish and Swedish post office names" + ) + return None + + # Use fallbacks: if Swedish is missing, use Finnish and vice versa + if not post_office_fi: + post_office_fi = post_office_sv + if not post_office_sv: + post_office_sv = post_office_fi + + # For English, use Finnish version + post_office_en = post_office_fi + + return postal_code, post_office_fi, post_office_sv, post_office_en + + def _update_postal_code_areas( + self, + postal_code: str, + post_office_fi: str, + post_office_sv: str, + post_office_en: str, + ) -> int: + """Update all postal code areas matching the given postal code.""" + try: + postal_code_areas = PostalCodeArea.objects.filter(postal_code=postal_code) + + if not postal_code_areas.exists(): + logger.debug(f"No postal code area found for postal code {postal_code}") + return 0 + + num_updated = 0 + for area in postal_code_areas: + area.set_current_language("fi") + area.post_office = post_office_fi + + area.set_current_language("sv") + area.post_office = post_office_sv + + area.set_current_language("en") + area.post_office = post_office_en + + area.save() + num_updated += 1 + logger.debug( + f"Updated postal code {postal_code} with post office " + f"FI:{post_office_fi} SV:{post_office_sv} EN:{post_office_en}" + ) + + return num_updated + + except Exception as e: + logger.error(f"Error updating postal code {postal_code}: {e}") + return 0 diff --git a/address/migrations/0011_postalcodearea_post_office.py b/address/migrations/0011_postalcodearea_post_office.py new file mode 100644 index 0000000..f210f3c --- /dev/null +++ b/address/migrations/0011_postalcodearea_post_office.py @@ -0,0 +1,19 @@ +# Generated by Django 6.0.2 on 2026-02-19 04:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("address", "0010_remove_postalcodearea_municipality"), + ] + + operations = [ + migrations.AddField( + model_name="postalcodeareatranslation", + name="post_office", + field=models.CharField( + blank=True, max_length=100, null=True, verbose_name="Post office" + ), + ), + ] diff --git a/address/models.py b/address/models.py index f17458d..2a4e7ca 100644 --- a/address/models.py +++ b/address/models.py @@ -44,7 +44,10 @@ class PostalCodeArea(TranslatableModel): _("Postal code"), max_length=5, null=True, blank=True ) translations = TranslatedFields( - name=models.CharField(_("Name"), max_length=100, null=True, blank=True) + name=models.CharField(_("Name"), max_length=100, null=True, blank=True), + post_office=models.CharField( + _("Post office"), max_length=100, null=True, blank=True + ), ) area = models.MultiPolygonField( _("Area"), srid=settings.PROJECTION_SRID, null=True, blank=True diff --git a/address/tests/test_api_serializers.py b/address/tests/test_api_serializers.py index 5b93b24..da0d9c6 100644 --- a/address/tests/test_api_serializers.py +++ b/address/tests/test_api_serializers.py @@ -45,6 +45,9 @@ def test_postal_code_area_serializer(): "postal_code": postal_code_area.postal_code, "area": None, "name": {t.language_code: t.name for t in postal_code_area.translations.all()}, + "post_office": { + t.language_code: t.post_office for t in postal_code_area.translations.all() + }, } @@ -69,6 +72,10 @@ def test_address_serializer(): t.language_code: t.name for t in address.postal_code_area.translations.all() }, + "post_office": { + t.language_code: t.post_office + for t in address.postal_code_area.translations.all() + }, }, "location": { "type": "Point", diff --git a/address/tests/test_import_post_offices_command.py b/address/tests/test_import_post_offices_command.py new file mode 100644 index 0000000..609f48a --- /dev/null +++ b/address/tests/test_import_post_offices_command.py @@ -0,0 +1,160 @@ +""" +Tests for the import_post_offices management command. +""" + +import zipfile +from pathlib import Path + +from django.core.management import call_command +from pytest import mark + +from address.models import PostalCodeArea +from address.tests.factories import PostalCodeAreaFactory + + +def create_test_zip(zip_path: Path, postal_data: list[tuple[str, str, str]]) -> None: + """ + Create a test ZIP file with Posti fixed-width format DAT file. + + Args: + zip_path: Path where the ZIP file should be created + postal_data: List of tuples (postal_code, post_office_fi, post_office_sv) + """ + lines = [] + for postal_code, post_office_fi, post_office_sv in postal_data: + # Build a fixed-width line + # Positions 0-12: Header (PONOT20260218) + # Positions 13-17: Postal code (5 chars) + # Positions 18-47: Finnish post office (30 chars) + # Positions 48-77: Swedish post office (30 chars) + line = ( + "PONOT20260218" + + postal_code.ljust(5) + + post_office_fi.ljust(30) + + post_office_sv.ljust(30) + ) + # Pad to minimum length + line = line.ljust(200) + lines.append(line) + + dat_content = "\n".join(lines) + + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("PCF_20260218.dat", dat_content) + + +@mark.django_db +def test_import_post_offices_from_zip_file(tmp_path): + """Test importing post office names from a ZIP file with Finnish and Swedish.""" + area1 = PostalCodeAreaFactory(postal_code="00900") + area2 = PostalCodeAreaFactory(postal_code="02760") + area3 = PostalCodeAreaFactory(postal_code="07590") + + zip_file = tmp_path / "postal_codes.zip" + create_test_zip( + zip_file, + [ + ("00900", "HELSINKI", "HELSINGFORS"), + ("02760", "ESPOO", "ESBO"), + ("07590", "ASKOLA", "ASKOLA"), + ], + ) + + call_command("import_post_offices", str(zip_file)) + + area1.refresh_from_db() + area1.set_current_language("fi") + assert area1.post_office == "HELSINKI" + area1.set_current_language("sv") + assert area1.post_office == "HELSINGFORS" + area1.set_current_language("en") + assert area1.post_office == "HELSINKI" + + area2.refresh_from_db() + area2.set_current_language("fi") + assert area2.post_office == "ESPOO" + area2.set_current_language("sv") + assert area2.post_office == "ESBO" + area2.set_current_language("en") + assert area2.post_office == "ESPOO" + + area3.refresh_from_db() + area3.set_current_language("fi") + assert area3.post_office == "ASKOLA" + area3.set_current_language("sv") + assert area3.post_office == "ASKOLA" + area3.set_current_language("en") + assert area3.post_office == "ASKOLA" + + +@mark.django_db +def test_import_post_offices_uses_finnish_fallback_for_swedish(tmp_path): + """Test that Finnish is used as fallback when Swedish is missing.""" + area = PostalCodeAreaFactory(postal_code="00900") + + zip_file = tmp_path / "postal_codes.zip" + create_test_zip(zip_file, [("00900", "HELSINKI", "")]) + call_command("import_post_offices", str(zip_file)) + + area.refresh_from_db() + area.set_current_language("fi") + assert area.post_office == "HELSINKI" + area.set_current_language("sv") + assert area.post_office == "HELSINKI" + + +@mark.django_db +def test_import_post_offices_uses_swedish_fallback_for_finnish(tmp_path): + """Test that Swedish is used as fallback when Finnish is missing.""" + area = PostalCodeAreaFactory(postal_code="00900") + + zip_file = tmp_path / "postal_codes.zip" + create_test_zip(zip_file, [("00900", "", "HELSINGFORS")]) + call_command("import_post_offices", str(zip_file)) + + area.refresh_from_db() + area.set_current_language("fi") + assert area.post_office == "HELSINGFORS" + area.set_current_language("sv") + assert area.post_office == "HELSINGFORS" + + +@mark.django_db +def test_import_post_offices_skips_missing_postal_codes(tmp_path): + """Test that the command skips postal codes that don't exist in the database.""" + area = PostalCodeAreaFactory(postal_code="00900") + + zip_file = tmp_path / "postal_codes.zip" + create_test_zip( + zip_file, + [ + ("00900", "HELSINKI", "HELSINGFORS"), + ("99999", "NONEXISTENT", "EXISTERAR INTE"), + ], + ) + call_command("import_post_offices", str(zip_file)) + + area.refresh_from_db() + area.set_current_language("fi") + assert area.post_office == "HELSINKI" + + assert not PostalCodeArea.objects.filter(postal_code="99999").exists() + + +@mark.django_db +def test_import_post_offices_handles_whitespace(tmp_path): + """Test that the command properly handles whitespace in post office names.""" + area = PostalCodeAreaFactory(postal_code="00900") + + zip_file = tmp_path / "postal_codes.zip" + create_test_zip( + zip_file, + [("00900", "HELSINKI ", "HELSINGFORS ")], + ) + call_command("import_post_offices", str(zip_file)) + + area.refresh_from_db() + area.set_current_language("fi") + assert area.post_office == "HELSINKI" + area.set_current_language("sv") + assert area.post_office == "HELSINGFORS" diff --git a/address/tests/test_post_office_filtering.py b/address/tests/test_post_office_filtering.py new file mode 100644 index 0000000..dac9026 --- /dev/null +++ b/address/tests/test_post_office_filtering.py @@ -0,0 +1,116 @@ +""" +Tests for querying addresses by post_office field. +""" + +from django.urls import reverse +from pytest import mark +from rest_framework.test import APIClient + +from address.tests.factories import AddressFactory, PostalCodeAreaFactory + + +@mark.django_db +def test_filter_addresses_by_post_office_finnish(api_client: APIClient): + """Test filtering addresses by Finnish post office name.""" + postal_code_area = PostalCodeAreaFactory(postal_code="00900") + postal_code_area.set_current_language("fi") + postal_code_area.post_office = "HELSINKI" + postal_code_area.save() + + address = AddressFactory(postal_code_area=postal_code_area) + AddressFactory() # Different postal code area + + url = reverse("address:address-list") + response = api_client.get(url, {"postoffice": "HELSINKI"}) + + assert response.status_code == 200 + assert response.data["count"] == 1 + assert response.data["results"][0]["number"] == address.number + + +@mark.django_db +def test_filter_addresses_by_post_office_swedish(api_client: APIClient): + """Test filtering addresses by Swedish post office name.""" + postal_code_area = PostalCodeAreaFactory(postal_code="00900") + postal_code_area.set_current_language("sv") + postal_code_area.post_office = "HELSINGFORS" + postal_code_area.save() + + address = AddressFactory(postal_code_area=postal_code_area) + AddressFactory() # Different postal code area + + url = reverse("address:address-list") + response = api_client.get(url, {"postoffice": "HELSINGFORS"}) + + assert response.status_code == 200 + assert response.data["count"] == 1 + assert response.data["results"][0]["number"] == address.number + + +@mark.django_db +def test_filter_addresses_by_post_office_case_insensitive(api_client: APIClient): + """Test that post office filtering is case insensitive.""" + postal_code_area = PostalCodeAreaFactory(postal_code="00900") + postal_code_area.set_current_language("fi") + postal_code_area.post_office = "HELSINKI" + postal_code_area.save() + + AddressFactory(postal_code_area=postal_code_area) + + url = reverse("address:address-list") + response = api_client.get(url, {"postoffice": "helsinki"}) + + assert response.status_code == 200 + assert response.data["count"] == 1 + + +@mark.django_db +def test_filter_addresses_by_post_office_no_results(api_client: APIClient): + """Test filtering by non-existent post office returns no results.""" + AddressFactory() + + url = reverse("address:address-list") + response = api_client.get(url, {"postoffice": "NONEXISTENT"}) + + assert response.status_code == 200 + assert response.data["count"] == 0 + + +@mark.django_db +def test_filter_postal_code_areas_by_post_office(api_client: APIClient): + """Test filtering postal code areas by post office name.""" + area1 = PostalCodeAreaFactory(postal_code="00900") + area1.set_current_language("fi") + area1.post_office = "HELSINKI" + area1.save() + + area2 = PostalCodeAreaFactory(postal_code="02760") + area2.set_current_language("fi") + area2.post_office = "ESPOO" + area2.save() + + url = reverse("address:postalcodearea-list") + response = api_client.get(url, {"postoffice": "HELSINKI"}) + + assert response.status_code == 200 + assert response.data["count"] == 1 + assert response.data["results"][0]["postal_code"] == "00900" + + +@mark.django_db +def test_filter_addresses_combined_with_other_params(api_client: APIClient): + """Test filtering by post office combined with other parameters.""" + postal_code_area = PostalCodeAreaFactory(postal_code="00900") + postal_code_area.set_current_language("fi") + postal_code_area.post_office = "HELSINKI" + postal_code_area.save() + + AddressFactory(postal_code_area=postal_code_area, number="5") + AddressFactory(postal_code_area=postal_code_area, number="10") + + url = reverse("address:address-list") + response = api_client.get(url, {"postoffice": "HELSINKI", "streetnumber": "5"}) + + assert response.status_code == 200 + assert response.data["count"] == 1 + assert response.data["results"][0]["number"] == "5" diff --git a/address/tests/test_postal_code_area_api.py b/address/tests/test_postal_code_area_api.py new file mode 100644 index 0000000..ab5f71c --- /dev/null +++ b/address/tests/test_postal_code_area_api.py @@ -0,0 +1,47 @@ +""" +Tests for PostalCodeArea API with post_office field. +""" + +from django.urls import reverse +from pytest import mark +from rest_framework.test import APIClient + +from address.tests.factories import PostalCodeAreaFactory + + +@mark.django_db +def test_postal_code_area_api_includes_post_office_translations(api_client: APIClient): + """Test that the API returns post_office in all languages.""" + area = PostalCodeAreaFactory(postal_code="00900") + + area.set_current_language("fi") + area.name = "Helsinki Keskusta" + area.post_office = "HELSINKI" + + area.set_current_language("sv") + area.name = "Helsingfors Centrum" + area.post_office = "HELSINGFORS" + + area.set_current_language("en") + area.name = "Helsinki Centre" + area.post_office = "HELSINKI" + + area.save() + + url = reverse("address:postalcodearea-detail", kwargs={"pk": area.pk}) + response = api_client.get(url) + + assert response.status_code == 200 + data = response.json() + + assert data["postal_code"] == "00900" + + assert "post_office" in data + assert data["post_office"]["fi"] == "HELSINKI" + assert data["post_office"]["sv"] == "HELSINGFORS" + assert data["post_office"]["en"] == "HELSINKI" + + assert "name" in data + assert data["name"]["fi"] == "Helsinki Keskusta" + assert data["name"]["sv"] == "Helsingfors Centrum" + assert data["name"]["en"] == "Helsinki Centre" diff --git a/scripts/import-post-office-data.sh b/scripts/import-post-office-data.sh new file mode 100755 index 0000000..277fd27 --- /dev/null +++ b/scripts/import-post-office-data.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +set -e + +echo "Importing Posti post office data" + +# Posti's postal code file URL +# The URL follows the format: https://www.posti.fi/webpcode/PCF_YYYYMMDD.zip +# where YYYYMMDD is the date. You can find the latest file at: +# https://www.posti.fi/webpcode/ + +# Get today's date in YYYYMMDD format +DATE=$(date +%Y%m%d) +DATA_URL="https://www.posti.fi/webpcode/PCF_${DATE}.zip" + +DATA_DIR=/tmp/posti + +# Download the source data +mkdir -p $DATA_DIR +echo "Downloading postal code data from Posti..." +curl --proto "=https" --tlsv1.2 -sSf -L -o $DATA_DIR/postal_codes.zip "$DATA_URL" + +# Run the management command +SCRIPT_DIR="$(cd "$(dirname "$0")"; pwd)" +python "$SCRIPT_DIR/../manage.py" import_post_offices "$DATA_DIR/postal_codes.zip" + +echo "Post office data import complete!" From cf5f33b9c83319af491704cb8916a2c200bc93ae Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Fri, 20 Feb 2026 08:57:05 +0200 Subject: [PATCH 12/18] feat: add import-municipalities-data.sh script - Create convenience script for municipality data import - Add file existence validation and clear error messages - Include usage instructions with NLS download link - Require shapefile path as command-line argument - Set executable permissions and verify with shellcheck Refs: RATYK-160 --- .../commands/import_municipalities.py | 3 +- scripts/import-municipalities-data.sh | 36 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100755 scripts/import-municipalities-data.sh diff --git a/address/management/commands/import_municipalities.py b/address/management/commands/import_municipalities.py index 186c278..61e1cec 100644 --- a/address/management/commands/import_municipalities.py +++ b/address/management/commands/import_municipalities.py @@ -1,6 +1,7 @@ """ This management command imports municipalities from NLS data: -https://www.maanmittauslaitos.fi/en/maps-and-spatial-data/datasets-and-interfaces/product-descriptions/division-administrative-areas-vector +https://www.maanmittauslaitos.fi/en/maps-and-spatial-data/datasets-and-interfaces/ +product-descriptions/division-administrative-areas-vector Data requires manual downloading and unzipping. The shapefiles can be given to the management command as arguments: diff --git a/scripts/import-municipalities-data.sh b/scripts/import-municipalities-data.sh new file mode 100755 index 0000000..b6f8271 --- /dev/null +++ b/scripts/import-municipalities-data.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +set -e + +if [ $# -eq 0 ] +then + echo "Municipality shapefile path required." + echo "" + echo "Usage: $0 " + echo "" + echo "Example:" + echo " $0 /tmp/nls/SuomenKuntajako_2026_10k.shp" + echo "" + echo "Download municipality data from:" + echo "https://www.maanmittauslaitos.fi/en/maps-and-spatial-data/datasets-and-interfaces/" + echo "product-descriptions/division-administrative-areas-vector" + echo "" + echo "The data must be manually downloaded from NLS following their" + echo "download process, then extracted from the ZIP file." + exit 0 +fi + +SHAPEFILE_PATH="$1" + +if [ ! -f "$SHAPEFILE_PATH" ] +then + echo "Error: Shapefile not found: $SHAPEFILE_PATH" >&2 + exit 1 +fi + +echo "Importing municipalities from $SHAPEFILE_PATH" + +SCRIPT_DIR="$(cd "$(dirname "$0")"; pwd)" +python "$SCRIPT_DIR/../manage.py" import_municipalities "$SHAPEFILE_PATH" + +echo "Municipality import complete!" From c9d4fbef560e425f98edf9456bbdc744113c464d Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Fri, 20 Feb 2026 09:07:25 +0200 Subject: [PATCH 13/18] docs: add comprehensive data import/re-import guide Refs: RATYK-160 --- README.md | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/README.md b/README.md index 6db9d73..e34599d 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,85 @@ Create the PostGIS extension if needed sudo -u postgres psql -c 'CREATE EXTENSION postgis;' +## Import or re-import data + +The project includes convenient shell scripts for importing geospatial +data from various sources. + +### Available import scripts + +* `scripts/import-municipalities-data.sh` - Import municipalities from NLS (requires manual download) +* `scripts/import-digiroad-data.sh` - Import address data from Digiroad / Finnish Transport Infrastructure Agency +* `scripts/import-paavo-data.sh` - Import postal code areas from Paavo / Statistics Finland +* `scripts/import-post-office-data.sh` - Import post office names from Posti +* `scripts/delete-address-data.sh` - Delete all address data (with confirmation) + +### First-time import + +**Important:** Municipality data must be imported first, before importing addresses. + +#### 1. Import municipalities (required, manual download) + +Municipality data must be manually downloaded from NLS: + +1. Visit [NLS Administrative Areas](https://www.maanmittauslaitos.fi/en/maps-and-spatial-data/datasets-and-interfaces/product-descriptions/division-administrative-areas-vector) +2. Download the dataset following NLS's download process +3. Extract the ZIP file to a directory (e.g., `/tmp/nls/`) +4. Run the import script: + + ./scripts/import-municipalities-data.sh /tmp/nls/SuomenKuntajako_2026_10k.shp + +#### 2. Import addresses and other data + +After municipalities are imported, import other data: + + # Import addresses (required, specify province) + ./scripts/import-digiroad-data.sh uusimaa + + # Import postal code areas (optional, specify province) + ./scripts/import-paavo-data.sh uusimaa + + # Import post office names (optional, downloads latest data) + ./scripts/import-post-office-data.sh + +Available provinces: `uusimaa` and `varsinais-suomi` + +### Re-importing data + +To re-import data (e.g., after updates): + + # Delete existing address data (prompts for confirmation) + ./scripts/delete-address-data.sh + + # Re-import municipalities if needed + ./scripts/import-municipalities-data.sh /path/to/SuomenKuntajako_2026_10k.shp + + # Re-import other data + ./scripts/import-digiroad-data.sh uusimaa + ./scripts/import-paavo-data.sh uusimaa + ./scripts/import-post-office-data.sh + +### Manual import using Django commands + +You can also use the Django management commands directly: + + # Import municipalities (required first, manual download needed) + python manage.py import_municipalities + + # Import addresses + python manage.py import_addresses + + # Import postal code areas + python manage.py import_postal_code_areas + + # Import post office names + python manage.py import_post_offices + # or download directly: + python manage.py import_post_offices --url + + # Delete all address data + python manage.py delete_address_data + ## Keeping Python requirements up to date 1. Add new packages to `requirements.in` or `requirements-dev.in` From c013bd9e3d298bb9f169f50653c2c373ffdccc0c Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Fri, 20 Feb 2026 12:06:36 +0200 Subject: [PATCH 14/18] ci: add Sonar exclusions Exclude migrations and pipelines from the Sonar analysis. Refs: RATYK-160 --- sonar-project.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/sonar-project.properties b/sonar-project.properties index 8829b88..d75a6b2 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -4,3 +4,4 @@ sonar.projectName=geo-search sonar.python.version=3.12 sonar.python.coverage.reportPaths=coverage.xml sonar.sourceEncoding=UTF-8 +sonar.exclusions=**/migrations/*,pipelines/* From e6b01f94adcff1a460b73f33e718db9794efae22 Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Fri, 20 Feb 2026 12:31:33 +0200 Subject: [PATCH 15/18] feat: remove province from postal code area import Province is only needed in the shell-script, therefore remove it from the management command. Refs: RATYK-163 --- address/management/commands/import_postal_code_areas.py | 6 +++--- address/services/postal_code_area_import.py | 6 ++---- address/tests/test_import_postal_codes_command.py | 2 +- scripts/import-paavo-data.sh | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/address/management/commands/import_postal_code_areas.py b/address/management/commands/import_postal_code_areas.py index 0d9713b..1d30c08 100644 --- a/address/management/commands/import_postal_code_areas.py +++ b/address/management/commands/import_postal_code_areas.py @@ -19,18 +19,18 @@ class Command(BaseCommand): help = "Imports postal code areas from the given Paavo shapefiles." def add_arguments(self, parser) -> None: - parser.add_argument("province") parser.add_argument("files", nargs="+", type=Path) def handle(self, *args, **options) -> None: start_time = time() - importer = PostalCodeAreaImporter(options["province"]) paths = options["files"] num_addresses_updated = 0 for path in paths: self.stdout.write(f"Reading data from {path}.") for layer in DataSource(path, encoding="latin-1"): - num_addresses_updated += importer.import_postal_code_areas(layer) + num_addresses_updated += ( + PostalCodeAreaImporter().import_postal_code_areas(layer) + ) self.stdout.write( self.style.SUCCESS( f"{num_addresses_updated} addresses updated " diff --git a/address/services/postal_code_area_import.py b/address/services/postal_code_area_import.py index 2072090..28d3a8a 100644 --- a/address/services/postal_code_area_import.py +++ b/address/services/postal_code_area_import.py @@ -14,10 +14,8 @@ class PostalCodeAreaImporter: - def __init__(self, province: str = None): - self.province = province - - def import_postal_code_areas(self, features: Iterable[Feature]) -> int: + @staticmethod + def import_postal_code_areas(features: Iterable[Feature]) -> int: """ Go through the given postal code area features and create/update PostalCodeArea objects for each area. diff --git a/address/tests/test_import_postal_codes_command.py b/address/tests/test_import_postal_codes_command.py index 552c500..3ced8c7 100644 --- a/address/tests/test_import_postal_codes_command.py +++ b/address/tests/test_import_postal_codes_command.py @@ -13,7 +13,7 @@ def test_import_postal_codes_updates_postal_codes_from_shapefile(paavo_shapefile location=Point(x=24.9428, y=60.1666, srid=settings.PROJECTION_SRID), municipality=municipality, ) - call_command("import_postal_code_areas", None, [paavo_shapefile]) + call_command("import_postal_code_areas", [paavo_shapefile]) address.refresh_from_db() assert address.postal_code_area.postal_code == "00100" assert address.postal_code_area.name == "Helsinki Keskusta - Etu-Töölö" diff --git a/scripts/import-paavo-data.sh b/scripts/import-paavo-data.sh index 61fa313..6b0dc1d 100755 --- a/scripts/import-paavo-data.sh +++ b/scripts/import-paavo-data.sh @@ -41,4 +41,4 @@ unzip $DATA_DIR/data.zip -d $EXTRACTED_DIR # Run the management command with all the shapefiles as arguments SCRIPT_DIR="$(cd "$(dirname "$0")"; pwd)" -python "$SCRIPT_DIR/../manage.py" import_postal_code_areas "$1" "$(find $EXTRACTED_DIR -type f -name "*.shp")" +python "$SCRIPT_DIR/../manage.py" import_postal_code_areas "$(find $EXTRACTED_DIR -type f -name "*.shp")" From 1e9f57063c14c43f5a227ecfde1175303fad16cb Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Fri, 20 Feb 2026 13:16:51 +0200 Subject: [PATCH 16/18] fix: improve MunicipalityImporter validation Refs: RATYK-163 --- address/services/municipality_import.py | 58 +++++++++++++++++++++---- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/address/services/municipality_import.py b/address/services/municipality_import.py index 1c00b05..f2ab528 100644 --- a/address/services/municipality_import.py +++ b/address/services/municipality_import.py @@ -14,7 +14,8 @@ class MunicipalityImporter: - def import_municipalities(self, features: Iterable[Feature]) -> int: + @staticmethod + def import_municipalities(features: Iterable[Feature]) -> int: """ Import municipalities from given features to db. Also update address municipality according to municipality area. @@ -24,16 +25,12 @@ def import_municipalities(self, features: Iterable[Feature]) -> int: # Update municipality for all addresses within each postal code area for feature in features: - geometry = feature.geom.geos - code = int(value_or_empty(feature, "NATCODE")) - name_fi = value_or_empty(feature, "NAMEFIN") - name_sv = value_or_empty(feature, "NAMESWE") or name_fi - if not name_fi and not name_sv: - logger.warning(f"Municipality with code {code} has no name, skipping") + validated_data = MunicipalityImporter._extract_and_validate_fields(feature) + if not validated_data: continue - if not name_fi: - name_fi = name_sv + code, name_fi, name_sv = validated_data + geometry = feature.geom.geos municipality = create_municipality( code=code, @@ -61,3 +58,46 @@ def import_municipalities(self, features: Iterable[Feature]) -> int: total_addresses_updated += num_addresses_updated return total_addresses_updated + + @staticmethod + def _extract_and_validate_fields( + feature: Feature, + ) -> tuple[int, str, str] | None: + """Extract and validate required municipality fields from feature. + + Returns: + Tuple of (code, name_fi, name_sv) if all validations pass, + None if any validation fails. + + Required fields: + - NATCODE: Municipality code (must be valid integer) + - NAMEFIN or NAMESWE: Municipality name in Finnish or Swedish + """ + nat_code = value_or_empty(feature, "NATCODE") + code = None + + if not nat_code: + logger.warning("Municipality feature missing NATCODE, skipping") + else: + try: + code = int(nat_code) + except ValueError: + logger.warning(f"Invalid NATCODE value '{nat_code}', skipping") + + if code is None: + return None + + name_fi = value_or_empty(feature, "NAMEFIN") + name_sv = value_or_empty(feature, "NAMESWE") + + if not name_fi and not name_sv: + logger.warning(f"Municipality with code {nat_code} has no name, skipping") + return None + + # Apply fallbacks + if not name_sv: + name_sv = name_fi + if not name_fi: + name_fi = name_sv + + return code, name_fi, name_sv From 6669aed34fdd871d84a39c1ace40210e7735bc7f Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Fri, 20 Feb 2026 13:30:01 +0200 Subject: [PATCH 17/18] refactor: use dicts in test assertions Use dictionary comparisons instead of individual field assertion for cleaner and more maintainable code. Refs: RATYK-163 --- .../tests/test_import_post_offices_command.py | 76 +++++++++++-------- address/tests/test_postal_code_area_api.py | 20 ++--- 2 files changed, 55 insertions(+), 41 deletions(-) diff --git a/address/tests/test_import_post_offices_command.py b/address/tests/test_import_post_offices_command.py index 609f48a..95f62ce 100644 --- a/address/tests/test_import_post_offices_command.py +++ b/address/tests/test_import_post_offices_command.py @@ -12,6 +12,15 @@ from address.tests.factories import PostalCodeAreaFactory +def get_post_office_translations(area: PostalCodeArea) -> dict[str, str]: + """Get post_office translations as a dict with language codes as keys.""" + translations = {} + for lang in ["fi", "sv", "en"]: + area.set_current_language(lang) + translations[lang] = area.post_office + return translations + + def create_test_zip(zip_path: Path, postal_data: list[tuple[str, str, str]]) -> None: """ Create a test ZIP file with Posti fixed-width format DAT file. @@ -63,28 +72,25 @@ def test_import_post_offices_from_zip_file(tmp_path): call_command("import_post_offices", str(zip_file)) area1.refresh_from_db() - area1.set_current_language("fi") - assert area1.post_office == "HELSINKI" - area1.set_current_language("sv") - assert area1.post_office == "HELSINGFORS" - area1.set_current_language("en") - assert area1.post_office == "HELSINKI" + assert get_post_office_translations(area1) == { + "fi": "HELSINKI", + "sv": "HELSINGFORS", + "en": "HELSINKI", + } area2.refresh_from_db() - area2.set_current_language("fi") - assert area2.post_office == "ESPOO" - area2.set_current_language("sv") - assert area2.post_office == "ESBO" - area2.set_current_language("en") - assert area2.post_office == "ESPOO" + assert get_post_office_translations(area2) == { + "fi": "ESPOO", + "sv": "ESBO", + "en": "ESPOO", + } area3.refresh_from_db() - area3.set_current_language("fi") - assert area3.post_office == "ASKOLA" - area3.set_current_language("sv") - assert area3.post_office == "ASKOLA" - area3.set_current_language("en") - assert area3.post_office == "ASKOLA" + assert get_post_office_translations(area3) == { + "fi": "ASKOLA", + "sv": "ASKOLA", + "en": "ASKOLA", + } @mark.django_db @@ -97,10 +103,11 @@ def test_import_post_offices_uses_finnish_fallback_for_swedish(tmp_path): call_command("import_post_offices", str(zip_file)) area.refresh_from_db() - area.set_current_language("fi") - assert area.post_office == "HELSINKI" - area.set_current_language("sv") - assert area.post_office == "HELSINKI" + assert get_post_office_translations(area) == { + "fi": "HELSINKI", + "sv": "HELSINKI", # Should use Finnish as fallback + "en": "HELSINKI", + } @mark.django_db @@ -113,10 +120,11 @@ def test_import_post_offices_uses_swedish_fallback_for_finnish(tmp_path): call_command("import_post_offices", str(zip_file)) area.refresh_from_db() - area.set_current_language("fi") - assert area.post_office == "HELSINGFORS" - area.set_current_language("sv") - assert area.post_office == "HELSINGFORS" + assert get_post_office_translations(area) == { + "fi": "HELSINGFORS", # Should use Swedish as fallback + "sv": "HELSINGFORS", + "en": "HELSINGFORS", + } @mark.django_db @@ -135,8 +143,11 @@ def test_import_post_offices_skips_missing_postal_codes(tmp_path): call_command("import_post_offices", str(zip_file)) area.refresh_from_db() - area.set_current_language("fi") - assert area.post_office == "HELSINKI" + assert get_post_office_translations(area) == { + "fi": "HELSINKI", + "sv": "HELSINGFORS", + "en": "HELSINKI", + } assert not PostalCodeArea.objects.filter(postal_code="99999").exists() @@ -154,7 +165,8 @@ def test_import_post_offices_handles_whitespace(tmp_path): call_command("import_post_offices", str(zip_file)) area.refresh_from_db() - area.set_current_language("fi") - assert area.post_office == "HELSINKI" - area.set_current_language("sv") - assert area.post_office == "HELSINGFORS" + assert get_post_office_translations(area) == { + "fi": "HELSINKI", + "sv": "HELSINGFORS", + "en": "HELSINKI", + } diff --git a/address/tests/test_postal_code_area_api.py b/address/tests/test_postal_code_area_api.py index ab5f71c..10330d3 100644 --- a/address/tests/test_postal_code_area_api.py +++ b/address/tests/test_postal_code_area_api.py @@ -36,12 +36,14 @@ def test_postal_code_area_api_includes_post_office_translations(api_client: APIC assert data["postal_code"] == "00900" - assert "post_office" in data - assert data["post_office"]["fi"] == "HELSINKI" - assert data["post_office"]["sv"] == "HELSINGFORS" - assert data["post_office"]["en"] == "HELSINKI" - - assert "name" in data - assert data["name"]["fi"] == "Helsinki Keskusta" - assert data["name"]["sv"] == "Helsingfors Centrum" - assert data["name"]["en"] == "Helsinki Centre" + assert data["post_office"] == { + "fi": "HELSINKI", + "sv": "HELSINGFORS", + "en": "HELSINKI", + } + + assert data["name"] == { + "fi": "Helsinki Keskusta", + "sv": "Helsingfors Centrum", + "en": "Helsinki Centre", + } From 06c16d9cfb0f5cd363fed72373afec70d21127a9 Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Fri, 20 Feb 2026 13:36:09 +0200 Subject: [PATCH 18/18] fix: validate municipality_code in address import Validate that municipality code exists in the province's municipality list. Refs: RATYK-163 --- address/services/address_import.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/address/services/address_import.py b/address/services/address_import.py index a703736..bea2592 100644 --- a/address/services/address_import.py +++ b/address/services/address_import.py @@ -74,6 +74,12 @@ def _build_addresses_from_feature(self, feature: Feature) -> list[Address]: # Create the municipality and street if they don't exist yet municipality_code = int(value_or_empty(feature, "KUNTAKOODI")) + + # Validate that municipality code exists in the province's municipality list + if municipality_code not in MUNICIPALITIES[self.province]: + # Skip this feature - municipality code not defined for this province + return [] + municipality_fi, municipality_sv = MUNICIPALITIES[self.province][ municipality_code ]