From dfccdb99e83fb56aa8d7a33432de26db1f6d0e01 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Mon, 1 Sep 2025 09:31:16 +0530 Subject: [PATCH 01/34] [Automated] Update the native jar versions --- ballerina/Ballerina.toml | 14 +++++++------- ballerina/CompilerPlugin.toml | 2 +- ballerina/Dependencies.toml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index 36598e94..8159e2d3 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -1,7 +1,7 @@ [package] org = "ballerina" name = "log" -version = "2.12.0" +version = "2.14.0" authors = ["Ballerina"] keywords = ["level", "format"] repository = "https://github.com/ballerina-platform/module-ballerina-log" @@ -15,18 +15,18 @@ graalvmCompatible = true [[platform.java21.dependency]] groupId = "io.ballerina.stdlib" artifactId = "log-native" -version = "2.12.0" -path = "../native/build/libs/log-native-2.12.0.jar" +version = "2.14.0" +path = "../native/build/libs/log-native-2.14.0-SNAPSHOT.jar" [[platform.java21.dependency]] groupId = "io.ballerina.stdlib" artifactId = "log-compiler-plugin" -version = "2.12.0" -path = "../compiler-plugin/build/libs/log-compiler-plugin-2.12.0.jar" +version = "2.14.0" +path = "../compiler-plugin/build/libs/log-compiler-plugin-2.14.0-SNAPSHOT.jar" [[platform.java21.dependency]] groupId = "io.ballerina.stdlib" artifactId = "log-test-utils" -version = "2.12.0" -path = "../test-utils/build/libs/log-test-utils-2.12.0.jar" +version = "2.14.0" +path = "../test-utils/build/libs/log-test-utils-2.14.0-SNAPSHOT.jar" scope = "testOnly" diff --git a/ballerina/CompilerPlugin.toml b/ballerina/CompilerPlugin.toml index 9c99a28c..100bfa67 100644 --- a/ballerina/CompilerPlugin.toml +++ b/ballerina/CompilerPlugin.toml @@ -3,4 +3,4 @@ id = "log-compiler-plugin" class = "io.ballerina.stdlib.log.compiler.LogCompilerPlugin" [[dependency]] -path = "../compiler-plugin/build/libs/log-compiler-plugin-2.12.0.jar" +path = "../compiler-plugin/build/libs/log-compiler-plugin-2.14.0-SNAPSHOT.jar" diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index 44c6a884..040bfb8a 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -76,7 +76,7 @@ modules = [ [[package]] org = "ballerina" name = "log" -version = "2.12.0" +version = "2.14.0" dependencies = [ {org = "ballerina", name = "io"}, {org = "ballerina", name = "jballerina.java"}, From 3fc78ff4a8115df71f07bf4c18a515a417161574 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Mon, 1 Sep 2025 18:50:58 +0530 Subject: [PATCH 02/34] Add initial implementation --- ballerina/sensitive_data_masking.bal | 52 ++++++ gradle.properties | 2 +- .../java/io/ballerina/stdlib/log/Utils.java | 170 ++++++++++++++++++ 3 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 ballerina/sensitive_data_masking.bal diff --git a/ballerina/sensitive_data_masking.bal b/ballerina/sensitive_data_masking.bal new file mode 100644 index 00000000..8763a1a5 --- /dev/null +++ b/ballerina/sensitive_data_masking.bal @@ -0,0 +1,52 @@ +// Copyright (c) 2025 WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/jballerina.java; + +# Exclude the field from log output +public const EXCLUDE = "EXCLUDE"; + +# Replacement function type for sensitive data masking +public type ReplacementFunction isolated function (string input) returns string; + +# Replacement strategy for sensitive data +# +# + replacement - The replacement value. This can be a string which will be used to replace the +# entire value, or a function that takes the original value and returns a masked version. +public type Replacement record {| + string|ReplacementFunction replacement; +|}; + +# Masking strategy for sensitive data +public type MaskingStrategy EXCLUDE|Replacement; + +# Represents sensitive data with a masking strategy +# +# + strategy - The masking strategy to apply (default: EXCLUDE) +public type SensitiveDataConfig record {| + MaskingStrategy strategy = EXCLUDE; +|}; + +# Marks a record field or type as sensitive, excluding it from log output +# +# + strategy - The masking strategy to apply (default: EXCLUDE) +public annotation SensitiveDataConfig SensitiveData on record field; + +configurable boolean enableSensitiveDataMasking = true; + +public function getMaskedString(anydata data) returns string = @java:Method { + 'class: "io.ballerina.stdlib.log.Utils" +} external; diff --git a/gradle.properties b/gradle.properties index dca1030f..676d4715 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ org.gradle.caching=true group=io.ballerina.stdlib -version=2.12.1-SNAPSHOT +version=2.14.0-SNAPSHOT ballerinaLangVersion=2201.12.0 checkstylePluginVersion=10.12.0 diff --git a/native/src/main/java/io/ballerina/stdlib/log/Utils.java b/native/src/main/java/io/ballerina/stdlib/log/Utils.java index fe600dec..f2f99d74 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/Utils.java +++ b/native/src/main/java/io/ballerina/stdlib/log/Utils.java @@ -18,12 +18,30 @@ package io.ballerina.stdlib.log; +import io.ballerina.runtime.api.Environment; +import io.ballerina.runtime.api.Runtime; +import io.ballerina.runtime.api.creators.ErrorCreator; +import io.ballerina.runtime.api.types.Field; +import io.ballerina.runtime.api.types.RecordType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.types.TypeTags; import io.ballerina.runtime.api.utils.IdentifierUtils; import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.utils.TypeUtils; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BFunctionPointer; +import io.ballerina.runtime.api.values.BMap; import io.ballerina.runtime.api.values.BString; +import io.ballerina.runtime.api.values.BTable; import java.text.SimpleDateFormat; +import java.util.Collection; import java.util.Date; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; /** * Native function implementations of the log-api module. @@ -61,4 +79,156 @@ public static BString getCurrentTime() { new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX") .format(new Date())); } + + public static BString getMaskedString(Environment env, Object value) { + Set visitedValues = new HashSet<>(); + return StringUtils.fromString(getMaskedStringInternal(env.getRuntime(), value, visitedValues)); + } + + static String getMaskedStringInternal(Runtime runtime, Object value, Set visitedValues) { + if (isBasicType(value)) { + return StringUtils.getStringValue(value); + } + if (!visitedValues.add(value)) { + throw ErrorCreator.createError(StringUtils.fromString("Cyclic value reference detected in the record")); + } + Type type = TypeUtils.getType(value); + if (value instanceof BMap mapValue) { + RecordType recType = (RecordType) type; + Map fields = recType.getFields(); + StringBuilder maskedString = new StringBuilder("{"); + Map fieldAnnotations = extractFieldAnnotations(recType); + for (Map.Entry entry : fields.entrySet()) { + String fieldName = entry.getKey(); + Optional fieldStringValue; + Optional annotation = getLogSensitiveDataAnnotation(fieldAnnotations, fieldName); + Object fieldValue = mapValue.get(StringUtils.fromString(fieldName)); + if (fieldValue == null) { + continue; + } + if (annotation.isPresent()) { + fieldStringValue = getStringValue(annotation.get(), fieldValue, runtime); + } else { + fieldStringValue = Optional.of(getMaskedStringInternal(runtime, fieldValue, visitedValues)); + } + fieldStringValue.ifPresent(s -> { + maskedString.append("\"").append(fieldName).append("\"") + .append(":"); + if (annotation.isPresent() || fieldValue instanceof BString) { + maskedString.append("\"").append(s).append("\""); + } else { + maskedString.append(s); + } + maskedString.append(","); + }); + } + if (maskedString.length() > 1) { + maskedString.setLength(maskedString.length() - 1); + } + maskedString.append("}"); + return maskedString.toString(); + } + if (value instanceof BTable tableValue) { + StringBuilder tableString = new StringBuilder("["); + for (Object row : tableValue.values()) { + String rowString = getMaskedStringInternal(runtime, row, visitedValues); + if (row instanceof BString) { + tableString.append("\"").append(rowString).append("\""); + } else { + tableString.append(rowString); + } + tableString.append(","); + } + if (tableString.length() > 1) { + tableString.setLength(tableString.length() - 1); + } + tableString.append("]"); + visitedValues.remove(value); + return tableString.toString(); + } + if (value instanceof BArray listValue) { + StringBuilder arrayString = new StringBuilder("["); + long length = listValue.getLength(); + for (long i = 0; i < length; i++) { + Object element = listValue.get(i); + String elementString = getMaskedStringInternal(runtime, element, visitedValues); + if (element instanceof BString) { + arrayString.append("\"").append(elementString).append("\""); + } else { + arrayString.append(elementString); + } + arrayString.append(","); + } + if (arrayString.length() > 1) { + arrayString.setLength(arrayString.length() - 1); + } + arrayString.append("]"); + visitedValues.remove(value); + return arrayString.toString(); + } + visitedValues.remove(value); + return StringUtils.getStringValue(value); + } + + static boolean isBasicType(Object value) { + return value == null || TypeUtils.getType(value).getTag() <= 7; + } + + static Optional getLogSensitiveDataAnnotation(Map fieldAnnotations, String fieldName) { + if (!fieldAnnotations.containsKey(fieldName)) { + return Optional.empty(); + } + + BMap fieldAnnotationMap = fieldAnnotations.get(fieldName); + Object[] keys = fieldAnnotationMap.getKeys(); + Object targetKey = null; + for (Object key : keys) { + if (key instanceof BString bStringKey && bStringKey.getValue().startsWith("ballerina/log") && + bStringKey.getValue().endsWith(":SensitiveData")) { + targetKey = key; + break; + } + } + if (targetKey != null) { + Object annotation = fieldAnnotationMap.get(targetKey); + if (annotation instanceof BMap) { + return Optional.of((BMap) annotation); + } + } + return Optional.empty(); + } + + static Map extractFieldAnnotations(RecordType recordType) { + BMap annotations = recordType.getAnnotations(); + if (annotations == null) { + return Map.of(); + } + return annotations.entrySet().stream() + .filter(entry -> entry.getKey().getValue().startsWith("$field$.")) + .filter(entry -> entry.getValue() instanceof BMap) + .collect(Collectors.toMap( + entry -> entry.getKey().getValue().substring(8), + entry -> (BMap) entry.getValue() + )); + } + + static Optional getStringValue(BMap annotation, Object realValue, Runtime runtime) { + Object strategy = annotation.get(StringUtils.fromString("strategy")); + if (strategy instanceof BString excluded && excluded.getValue().equals("EXCLUDE")) { + return Optional.empty(); + } + if (strategy instanceof BMap replacementMap) { + Object replacement = replacementMap.get(StringUtils.fromString("replacement")); + if (replacement instanceof BString replacementStr) { + return Optional.of(replacementStr.getValue()); + } + if (replacement instanceof BFunctionPointer replacer) { + Object replacementString = replacer.call(runtime, StringUtils.fromString(StringUtils.getStringValue(realValue))); + if (replacementString instanceof BString replacementStrVal) { + return Optional.of(replacementStrVal.getValue()); + } + } + } + return Optional.of(StringUtils.getStringValue(realValue)); + } } From 87787b60d5f9269e0825f5cc0b296b6533b9b364 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Tue, 2 Sep 2025 08:44:45 +0530 Subject: [PATCH 03/34] Refactor with perf improvements --- .../java/io/ballerina/stdlib/log/Utils.java | 300 ++++++++++++------ 1 file changed, 198 insertions(+), 102 deletions(-) diff --git a/native/src/main/java/io/ballerina/stdlib/log/Utils.java b/native/src/main/java/io/ballerina/stdlib/log/Utils.java index f2f99d74..94f43d8f 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/Utils.java +++ b/native/src/main/java/io/ballerina/stdlib/log/Utils.java @@ -37,10 +37,9 @@ import java.text.SimpleDateFormat; import java.util.Collection; import java.util.Date; -import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; /** @@ -50,6 +49,21 @@ */ public class Utils { + // Cache frequently used BString constants to avoid repeated allocations + private static final BString STRATEGY_KEY = StringUtils.fromString("strategy"); + private static final BString REPLACEMENT_KEY = StringUtils.fromString("replacement"); + private static final BString EXCLUDE_VALUE = StringUtils.fromString("EXCLUDE"); + private static final String FIELD_PREFIX = "$field$."; + private static final String LOG_ANNOTATION_PREFIX = "ballerina/log"; + private static final String SENSITIVE_DATA_SUFFIX = ":SensitiveData"; + + // Cache error message to avoid repeated BString creation + private static final BString CYCLIC_REFERENCE_ERROR = StringUtils.fromString("Cyclic value reference detected in the record"); + + // Cache for SimpleDateFormat to avoid creating new instances (thread-safe) + private static final ThreadLocal DATE_FORMAT = + ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")); + private Utils() { } @@ -75,150 +89,232 @@ public static BString getModuleNameExtern() { * @return current local time in RFC3339 format */ public static BString getCurrentTime() { - return StringUtils.fromString( - new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX") - .format(new Date())); + return StringUtils.fromString(DATE_FORMAT.get().format(new Date())); } public static BString getMaskedString(Environment env, Object value) { - Set visitedValues = new HashSet<>(); + // Use IdentityHashMap for much better memory efficiency + // Only stores identity-based references, not hash-based ones + IdentityHashMap visitedValues = new IdentityHashMap<>(); return StringUtils.fromString(getMaskedStringInternal(env.getRuntime(), value, visitedValues)); } - static String getMaskedStringInternal(Runtime runtime, Object value, Set visitedValues) { + static String getMaskedStringInternal(Runtime runtime, Object value, IdentityHashMap visitedValues) { if (isBasicType(value)) { return StringUtils.getStringValue(value); } - if (!visitedValues.add(value)) { - throw ErrorCreator.createError(StringUtils.fromString("Cyclic value reference detected in the record")); + + // Use identity-based checking instead of hash-based + if (visitedValues.put(value, Boolean.TRUE) != null) { + throw ErrorCreator.createError(CYCLIC_REFERENCE_ERROR); + } + + try { + return processValue(runtime, value, visitedValues); + } finally { + visitedValues.remove(value); } + } + + private static String processValue(Runtime runtime, Object value, IdentityHashMap visitedValues) { Type type = TypeUtils.getType(value); - if (value instanceof BMap mapValue) { - RecordType recType = (RecordType) type; - Map fields = recType.getFields(); - StringBuilder maskedString = new StringBuilder("{"); - Map fieldAnnotations = extractFieldAnnotations(recType); - for (Map.Entry entry : fields.entrySet()) { - String fieldName = entry.getKey(); - Optional fieldStringValue; - Optional annotation = getLogSensitiveDataAnnotation(fieldAnnotations, fieldName); - Object fieldValue = mapValue.get(StringUtils.fromString(fieldName)); - if (fieldValue == null) { - continue; - } - if (annotation.isPresent()) { - fieldStringValue = getStringValue(annotation.get(), fieldValue, runtime); - } else { - fieldStringValue = Optional.of(getMaskedStringInternal(runtime, fieldValue, visitedValues)); - } - fieldStringValue.ifPresent(s -> { - maskedString.append("\"").append(fieldName).append("\"") - .append(":"); - if (annotation.isPresent() || fieldValue instanceof BString) { - maskedString.append("\"").append(s).append("\""); - } else { - maskedString.append(s); - } - maskedString.append(","); - }); + + // Use switch-like pattern for better performance + return switch (value) { + case BMap mapValue -> processMapValue(runtime, mapValue, type, visitedValues); + case BTable tableValue -> processTableValue(runtime, tableValue, visitedValues); + case BArray listValue -> processArrayValue(runtime, listValue, visitedValues); + default -> StringUtils.getStringValue(value); + }; + } + + private static String processMapValue(Runtime runtime, BMap mapValue, Type valueType, IdentityHashMap visitedValues) { + if (valueType.getTag() != TypeTags.RECORD_TYPE_TAG) { + // For non-record maps, use default string representation + return StringUtils.getStringValue(mapValue); + } + + RecordType recType = (RecordType) valueType; + Map fields = recType.getFields(); + if (fields.isEmpty()) { + return "{}"; + } + + // More conservative and safer capacity estimation + // Use adaptive sizing based on actual field count with reasonable bounds + int fieldCount = fields.size(); + int baseCapacity = fieldCount <= 5 ? 64 : + fieldCount <= 20 ? 256 : + Math.min(fieldCount * 15, 2048); // Cap at 2KB for very large objects + + StringBuilder maskedString = new StringBuilder(baseCapacity); + maskedString.append('{'); + + Map> fieldAnnotations = extractFieldAnnotations(recType); + boolean first = true; + + for (Map.Entry entry : fields.entrySet()) { + String fieldName = entry.getKey(); + Object fieldValue = mapValue.get(StringUtils.fromString(fieldName)); + + if (fieldValue == null) { + continue; } - if (maskedString.length() > 1) { - maskedString.setLength(maskedString.length() - 1); + + Optional> annotation = getLogSensitiveDataAnnotation(fieldAnnotations, fieldName); + Optional fieldStringValue; + + if (annotation.isPresent()) { + fieldStringValue = getStringValue(annotation.get(), fieldValue, runtime); + } else { + fieldStringValue = Optional.of(getMaskedStringInternal(runtime, fieldValue, visitedValues)); } - maskedString.append("}"); - return maskedString.toString(); - } - if (value instanceof BTable tableValue) { - StringBuilder tableString = new StringBuilder("["); - for (Object row : tableValue.values()) { - String rowString = getMaskedStringInternal(runtime, row, visitedValues); - if (row instanceof BString) { - tableString.append("\"").append(rowString).append("\""); - } else { - tableString.append(rowString); + + if (fieldStringValue.isPresent()) { + if (!first) { + maskedString.append(','); } - tableString.append(","); - } - if (tableString.length() > 1) { - tableString.setLength(tableString.length() - 1); + appendFieldToJson(maskedString, fieldName, fieldStringValue.get(), + annotation.isPresent(), fieldValue); + first = false; } - tableString.append("]"); - visitedValues.remove(value); - return tableString.toString(); - } - if (value instanceof BArray listValue) { - StringBuilder arrayString = new StringBuilder("["); - long length = listValue.getLength(); - for (long i = 0; i < length; i++) { - Object element = listValue.get(i); - String elementString = getMaskedStringInternal(runtime, element, visitedValues); - if (element instanceof BString) { - arrayString.append("\"").append(elementString).append("\""); - } else { - arrayString.append(elementString); - } - arrayString.append(","); + } + + maskedString.append('}'); + return maskedString.toString(); + } + + private static void appendFieldToJson(StringBuilder sb, String fieldName, String value, + boolean hasAnnotation, Object fieldValue) { + sb.append('"').append(fieldName).append("\":"); + if (hasAnnotation || fieldValue instanceof BString) { + sb.append('"').append(value).append('"'); + } else { + sb.append(value); + } + } + + private static String processTableValue(Runtime runtime, BTable tableValue, IdentityHashMap visitedValues) { + Collection values = tableValue.values(); + if (values.isEmpty()) { + return "[]"; + } + + // Safer capacity estimation with bounds checking + int valueCount = values.size(); + int baseCapacity = valueCount <= 10 ? 128 : + valueCount <= 100 ? 512 : + Math.min(valueCount * 20, 4096); // Cap at 4KB + + StringBuilder tableString = new StringBuilder(baseCapacity); + tableString.append('['); + + boolean first = true; + for (Object row : values) { + if (!first) { + tableString.append(','); } - if (arrayString.length() > 1) { - arrayString.setLength(arrayString.length() - 1); + appendValueToArray(tableString, getMaskedStringInternal(runtime, row, visitedValues), row); + first = false; + } + + tableString.append(']'); + return tableString.toString(); + } + + private static String processArrayValue(Runtime runtime, BArray listValue, IdentityHashMap visitedValues) { + long length = listValue.getLength(); + if (length == 0) { + return "[]"; + } + + // Safe capacity calculation with overflow protection + int safeLength = length > Integer.MAX_VALUE / 20 ? Integer.MAX_VALUE / 20 : (int) length; + int baseCapacity = safeLength <= 20 ? 96 : + safeLength <= 200 ? 384 : + Math.min(safeLength * 12, 3072); // Cap at 3KB + + StringBuilder arrayString = new StringBuilder(baseCapacity); + arrayString.append('['); + + for (long i = 0; i < length; i++) { + if (i > 0) { + arrayString.append(','); } - arrayString.append("]"); - visitedValues.remove(value); - return arrayString.toString(); + Object element = listValue.get(i); + String elementString = getMaskedStringInternal(runtime, element, visitedValues); + appendValueToArray(arrayString, elementString, element); + } + + arrayString.append(']'); + return arrayString.toString(); + } + + private static void appendValueToArray(StringBuilder sb, String value, Object originalValue) { + if (originalValue instanceof BString) { + sb.append('"').append(value).append('"'); + } else { + sb.append(value); } - visitedValues.remove(value); - return StringUtils.getStringValue(value); } static boolean isBasicType(Object value) { - return value == null || TypeUtils.getType(value).getTag() <= 7; + return value == null || TypeUtils.getType(value).getTag() <= TypeTags.BOOLEAN_TAG; } - static Optional getLogSensitiveDataAnnotation(Map fieldAnnotations, String fieldName) { - if (!fieldAnnotations.containsKey(fieldName)) { + static Optional> getLogSensitiveDataAnnotation(Map> fieldAnnotations, String fieldName) { + BMap fieldAnnotationMap = fieldAnnotations.get(fieldName); + if (fieldAnnotationMap == null) { return Optional.empty(); } - BMap fieldAnnotationMap = fieldAnnotations.get(fieldName); + // Cache the keys array to avoid repeated calls Object[] keys = fieldAnnotationMap.getKeys(); - Object targetKey = null; + + // Optimized search - most annotation maps are small, so linear search is efficient for (Object key : keys) { - if (key instanceof BString bStringKey && bStringKey.getValue().startsWith("ballerina/log") && - bStringKey.getValue().endsWith(":SensitiveData")) { - targetKey = key; - break; - } - } - if (targetKey != null) { - Object annotation = fieldAnnotationMap.get(targetKey); - if (annotation instanceof BMap) { - return Optional.of((BMap) annotation); + if (key instanceof BString bStringKey) { + String keyValue = bStringKey.getValue(); + // Use more efficient string matching - check suffix first (likely to fail faster) + if (keyValue.endsWith(SENSITIVE_DATA_SUFFIX) && keyValue.startsWith(LOG_ANNOTATION_PREFIX)) { + Object annotation = fieldAnnotationMap.get(key); + if (annotation instanceof BMap bMapAnnotation) { + return Optional.of(bMapAnnotation); + } + // Found the target annotation type, no need to continue + break; + } } } return Optional.empty(); } - static Map extractFieldAnnotations(RecordType recordType) { + static Map> extractFieldAnnotations(RecordType recordType) { BMap annotations = recordType.getAnnotations(); if (annotations == null) { return Map.of(); } + + // Use more efficient stream processing return annotations.entrySet().stream() - .filter(entry -> entry.getKey().getValue().startsWith("$field$.")) - .filter(entry -> entry.getValue() instanceof BMap) + .filter(entry -> { + String keyValue = entry.getKey().getValue(); + return keyValue.startsWith(FIELD_PREFIX) && entry.getValue() instanceof BMap; + }) .collect(Collectors.toMap( - entry -> entry.getKey().getValue().substring(8), - entry -> (BMap) entry.getValue() + entry -> entry.getKey().getValue().substring(FIELD_PREFIX.length()), + entry -> (BMap) entry.getValue(), + (existing, replacement) -> existing // Handle potential duplicates )); } - static Optional getStringValue(BMap annotation, Object realValue, Runtime runtime) { - Object strategy = annotation.get(StringUtils.fromString("strategy")); - if (strategy instanceof BString excluded && excluded.getValue().equals("EXCLUDE")) { + static Optional getStringValue(BMap annotation, Object realValue, Runtime runtime) { + Object strategy = annotation.get(STRATEGY_KEY); + if (strategy instanceof BString strategyStr && EXCLUDE_VALUE.getValue().equals(strategyStr.getValue())) { return Optional.empty(); } - if (strategy instanceof BMap replacementMap) { - Object replacement = replacementMap.get(StringUtils.fromString("replacement")); + if (strategy instanceof BMap replacementMap) { + Object replacement = replacementMap.get(REPLACEMENT_KEY); if (replacement instanceof BString replacementStr) { return Optional.of(replacementStr.getValue()); } From 8ca08e715546e0f0107df398a2b075dc10358348 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Tue, 2 Sep 2025 10:43:45 +0530 Subject: [PATCH 04/34] Refactor by adding a builder --- .../stdlib/log/MaskedStringBuilder.java | 438 ++++++++++++++++++ .../java/io/ballerina/stdlib/log/Utils.java | 262 +---------- 2 files changed, 441 insertions(+), 259 deletions(-) create mode 100644 native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java diff --git a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java new file mode 100644 index 00000000..b98f0733 --- /dev/null +++ b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java @@ -0,0 +1,438 @@ +/* + * Copyright (c) 2025, WSO2 LLC. (http://www.wso2.org). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.stdlib.log; + +import io.ballerina.runtime.api.Runtime; +import io.ballerina.runtime.api.creators.ErrorCreator; +import io.ballerina.runtime.api.types.Field; +import io.ballerina.runtime.api.types.RecordType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.types.TypeTags; +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.utils.TypeUtils; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BFunctionPointer; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; +import io.ballerina.runtime.api.values.BTable; + +import java.util.Collection; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * High-performance builder for creating masked string representations of Ballerina values. + * Implements Closeable for proper resource management and memory efficiency. + * + * @since 2.14.0 + */ +public class MaskedStringBuilder implements AutoCloseable { + + private static final BString STRATEGY_KEY = StringUtils.fromString("strategy"); + private static final BString REPLACEMENT_KEY = StringUtils.fromString("replacement"); + private static final BString EXCLUDE_VALUE = StringUtils.fromString("EXCLUDE"); + private static final String FIELD_PREFIX = "$field$."; + private static final String LOG_ANNOTATION_PREFIX = "ballerina/log"; + private static final String SENSITIVE_DATA_SUFFIX = ":SensitiveData"; + + private static final BString CYCLIC_REFERENCE_ERROR = StringUtils.fromString("Cyclic value reference detected in the record"); + + private final Runtime runtime; + private final IdentityHashMap visitedValues; + private StringBuilder stringBuilder; + private boolean closed = false; + + // Initial capacity configuration + private static final int DEFAULT_INITIAL_CAPACITY = 256; + private static final int MAX_REUSABLE_CAPACITY = 8192; // 8KB threshold for reuse + + public MaskedStringBuilder(Runtime runtime) { + this.runtime = runtime; + this.visitedValues = new IdentityHashMap<>(); + this.stringBuilder = new StringBuilder(DEFAULT_INITIAL_CAPACITY); + } + + public MaskedStringBuilder(Runtime runtime, int initialCapacity) { + this.runtime = runtime; + this.visitedValues = new IdentityHashMap<>(); + this.stringBuilder = new StringBuilder(Math.max(initialCapacity, DEFAULT_INITIAL_CAPACITY)); + } + + /** + * Build a masked string representation of the given value. + * + * @param value the value to mask + * @return the masked string representation + */ + public String build(Object value) { + if (closed) { + throw new IllegalStateException("MaskedStringBuilder has been closed"); + } + + try { + visitedValues.clear(); + stringBuilder.setLength(0); + + String result = buildInternal(value); + + // If the builder grew too large, replace it with a smaller one for future use + if (stringBuilder.capacity() > MAX_REUSABLE_CAPACITY) { + stringBuilder = new StringBuilder(DEFAULT_INITIAL_CAPACITY); + } + + return result; + } finally { + visitedValues.clear(); + } + } + + private String buildInternal(Object value) { + if (isBasicType(value)) { + return StringUtils.getStringValue(value); + } + + // Use identity-based checking for cycle detection + if (visitedValues.put(value, Boolean.TRUE) != null) { + throw ErrorCreator.createError(CYCLIC_REFERENCE_ERROR); + } + + try { + return processValue(value); + } finally { + visitedValues.remove(value); + } + } + + private String processValue(Object value) { + Type type = TypeUtils.getType(value); + + return switch (value) { + // Processing only the structured types, since the basic types does not contain the + // inherent type information unless they are part of a structured type. + case BMap mapValue -> processMapValue(mapValue, type); + case BTable tableValue -> processTableValue(tableValue); + case BArray listValue -> processArrayValue(listValue); + default -> StringUtils.getStringValue(value); + }; + } + + private String processMapValue(BMap mapValue, Type valueType) { + if (valueType.getTag() != TypeTags.RECORD_TYPE_TAG) { + return StringUtils.getStringValue(mapValue); + } + + RecordType recType = (RecordType) valueType; + Map fields = recType.getFields(); + if (fields.isEmpty()) { + return "{}"; + } + + int startPos = stringBuilder.length(); + stringBuilder.append('{'); + + Map> fieldAnnotations = extractFieldAnnotations(recType); + boolean first = true; + + for (Map.Entry entry : fields.entrySet()) { + String fieldName = entry.getKey(); + Object fieldValue = mapValue.get(StringUtils.fromString(fieldName)); + + if (fieldValue == null) { + continue; + } + + Optional> annotation = getLogSensitiveDataAnnotation(fieldAnnotations, fieldName); + Optional fieldStringValue; + + if (annotation.isPresent()) { + fieldStringValue = getStringValue(annotation.get(), fieldValue, runtime); + } else { + fieldStringValue = Optional.of(buildInternal(fieldValue)); + } + + if (fieldStringValue.isPresent()) { + if (!first) { + stringBuilder.append(','); + } + appendFieldToJson(fieldName, fieldStringValue.get(), annotation.isPresent(), fieldValue); + first = false; + } + } + + stringBuilder.append('}'); + + String result = stringBuilder.substring(startPos); + stringBuilder.setLength(startPos); + return result; + } + + private void appendFieldToJson(String fieldName, String value, boolean hasAnnotation, Object fieldValue) { + stringBuilder.append('"').append(escapeJsonString(fieldName)).append("\":"); + if (hasAnnotation || fieldValue instanceof BString) { + stringBuilder.append('"').append(escapeJsonString(value)).append('"'); + } else { + stringBuilder.append(value); + } + } + + private String processTableValue(BTable tableValue) { + Collection values = tableValue.values(); + if (values.isEmpty()) { + return "[]"; + } + + int startPos = stringBuilder.length(); + stringBuilder.append('['); + + boolean first = true; + for (Object row : values) { + if (!first) { + stringBuilder.append(','); + } + String elementString = buildInternal(row); + appendValueToArray(elementString, row); + first = false; + } + + stringBuilder.append(']'); + + String result = stringBuilder.substring(startPos); + stringBuilder.setLength(startPos); + return result; + } + + private String processArrayValue(BArray listValue) { + long length = listValue.getLength(); + if (length == 0) { + return "[]"; + } + + int startPos = stringBuilder.length(); + stringBuilder.append('['); + + // Using traditional for loop instead of for-each loop since BArray giving + // this error: Cannot read the array length because "" is null + for (long i = 0; i < length; i++) { + if (i > 0) { + stringBuilder.append(','); + } + Object element = listValue.get(i); + String elementString = buildInternal(element); + appendValueToArray(elementString, element); + } + + stringBuilder.append(']'); + + String result = stringBuilder.substring(startPos); + stringBuilder.setLength(startPos); + return result; + } + + private void appendValueToArray(String value, Object originalValue) { + if (originalValue instanceof BString) { + stringBuilder.append('"').append(escapeJsonString(value)).append('"'); + } else { + stringBuilder.append(value); + } + } + + /** + * Escape characters in a string for safe JSON representation. + * Handles quotes, backslashes, and control characters. + * + * @param input the input string to escape + * @return the escaped string + */ + private static String escapeJsonString(String input) { + if (input == null) { + return "null"; + } + + if (!needsEscaping(input)) { + return input; + } + + StringBuilder escaped = new StringBuilder(input.length() + 16); + + for (int i = 0; i < input.length(); i++) { + char c = input.charAt(i); + switch (c) { + case '"' -> escaped.append("\\\""); + case '\\' -> escaped.append("\\\\"); + case '\b' -> escaped.append("\\b"); + case '\f' -> escaped.append("\\f"); + case '\n' -> escaped.append("\\n"); + case '\r' -> escaped.append("\\r"); + case '\t' -> escaped.append("\\t"); + default -> { + if (c < 0x20 || c == 0x7F) { + escaped.append(String.format("\\u%04x", (int) c)); + } else { + escaped.append(c); + } + } + } + } + + return escaped.toString(); + } + + /** + * Quick check if a string needs JSON escaping. + * This avoids unnecessary StringBuilder allocation for clean strings. + */ + private static boolean needsEscaping(String input) { + for (int i = 0; i < input.length(); i++) { + char c = input.charAt(i); + if (c == '"' || c == '\\' || c < 0x20 || c == 0x7F) { + return true; + } + } + return false; + } + + private static boolean isBasicType(Object value) { + return value == null || TypeUtils.getType(value).getTag() <= TypeTags.BOOLEAN_TAG; + } + + /** + * Get the current capacity of the internal StringBuilder. + * Useful for monitoring memory usage. + * + * @return current capacity + */ + public int getCapacity() { + return stringBuilder.capacity(); + } + + /** + * Reset the builder state for reuse while keeping the allocated memory. + * This is more efficient than creating a new builder instance. + */ + public void reset() { + if (closed) { + throw new IllegalStateException("MaskedStringBuilder has been closed"); + } + visitedValues.clear(); + stringBuilder.setLength(0); + } + + /** + * Check if the builder has been closed. + * + * @return true if closed, false otherwise + */ + public boolean isClosed() { + return closed; + } + + @Override + public void close() { + if (!closed) { + visitedValues.clear(); + stringBuilder = null; + closed = true; + } + } + + /** + * Create a new MaskedStringBuilder instance with default settings. + * + * @param runtime the Ballerina runtime + * @return a new MaskedStringBuilder instance + */ + public static MaskedStringBuilder create(Runtime runtime) { + return new MaskedStringBuilder(runtime); + } + + /** + * Create a new MaskedStringBuilder instance with specified initial capacity. + * + * @param runtime the Ballerina runtime + * @param initialCapacity the initial capacity for the internal StringBuilder + * @return a new MaskedStringBuilder instance + */ + public static MaskedStringBuilder create(Runtime runtime, int initialCapacity) { + return new MaskedStringBuilder(runtime, initialCapacity); + } + + static Optional> getLogSensitiveDataAnnotation(Map> fieldAnnotations, String fieldName) { + BMap fieldAnnotationMap = fieldAnnotations.get(fieldName); + if (fieldAnnotationMap == null) { + return Optional.empty(); + } + + Object[] keys = fieldAnnotationMap.getKeys(); + + for (Object key : keys) { + if (key instanceof BString bStringKey) { + String keyValue = bStringKey.getValue(); + if (keyValue.endsWith(SENSITIVE_DATA_SUFFIX) && keyValue.startsWith(LOG_ANNOTATION_PREFIX)) { + Object annotation = fieldAnnotationMap.get(key); + if (annotation instanceof BMap bMapAnnotation) { + return Optional.of(bMapAnnotation); + } + // Found the target annotation type, no need to continue + break; + } + } + } + return Optional.empty(); + } + + static Map> extractFieldAnnotations(RecordType recordType) { + BMap annotations = recordType.getAnnotations(); + if (annotations == null) { + return Map.of(); + } + + return annotations.entrySet().stream() + .filter(entry -> { + String keyValue = entry.getKey().getValue(); + return keyValue.startsWith(FIELD_PREFIX) && entry.getValue() instanceof BMap; + }) + .collect(Collectors.toMap( + entry -> entry.getKey().getValue().substring(FIELD_PREFIX.length()), + entry -> (BMap) entry.getValue(), + (existing, replacement) -> existing + )); + } + + static Optional getStringValue(BMap annotation, Object realValue, Runtime runtime) { + Object strategy = annotation.get(STRATEGY_KEY); + if (strategy instanceof BString strategyStr && EXCLUDE_VALUE.getValue().equals(strategyStr.getValue())) { + return Optional.empty(); + } + if (strategy instanceof BMap replacementMap) { + Object replacement = replacementMap.get(REPLACEMENT_KEY); + if (replacement instanceof BString replacementStr) { + return Optional.of(replacementStr.getValue()); + } + if (replacement instanceof BFunctionPointer replacer) { + Object replacementString = replacer.call(runtime, StringUtils.fromString(StringUtils.getStringValue(realValue))); + if (replacementString instanceof BString replacementStrVal) { + return Optional.of(replacementStrVal.getValue()); + } + } + } + return Optional.of(StringUtils.getStringValue(realValue)); + } +} diff --git a/native/src/main/java/io/ballerina/stdlib/log/Utils.java b/native/src/main/java/io/ballerina/stdlib/log/Utils.java index 94f43d8f..04f63821 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/Utils.java +++ b/native/src/main/java/io/ballerina/stdlib/log/Utils.java @@ -19,28 +19,12 @@ package io.ballerina.stdlib.log; import io.ballerina.runtime.api.Environment; -import io.ballerina.runtime.api.Runtime; -import io.ballerina.runtime.api.creators.ErrorCreator; -import io.ballerina.runtime.api.types.Field; -import io.ballerina.runtime.api.types.RecordType; -import io.ballerina.runtime.api.types.Type; -import io.ballerina.runtime.api.types.TypeTags; import io.ballerina.runtime.api.utils.IdentifierUtils; import io.ballerina.runtime.api.utils.StringUtils; -import io.ballerina.runtime.api.utils.TypeUtils; -import io.ballerina.runtime.api.values.BArray; -import io.ballerina.runtime.api.values.BFunctionPointer; -import io.ballerina.runtime.api.values.BMap; import io.ballerina.runtime.api.values.BString; -import io.ballerina.runtime.api.values.BTable; import java.text.SimpleDateFormat; -import java.util.Collection; import java.util.Date; -import java.util.IdentityHashMap; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; /** * Native function implementations of the log-api module. @@ -49,17 +33,6 @@ */ public class Utils { - // Cache frequently used BString constants to avoid repeated allocations - private static final BString STRATEGY_KEY = StringUtils.fromString("strategy"); - private static final BString REPLACEMENT_KEY = StringUtils.fromString("replacement"); - private static final BString EXCLUDE_VALUE = StringUtils.fromString("EXCLUDE"); - private static final String FIELD_PREFIX = "$field$."; - private static final String LOG_ANNOTATION_PREFIX = "ballerina/log"; - private static final String SENSITIVE_DATA_SUFFIX = ":SensitiveData"; - - // Cache error message to avoid repeated BString creation - private static final BString CYCLIC_REFERENCE_ERROR = StringUtils.fromString("Cyclic value reference detected in the record"); - // Cache for SimpleDateFormat to avoid creating new instances (thread-safe) private static final ThreadLocal DATE_FORMAT = ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")); @@ -93,238 +66,9 @@ public static BString getCurrentTime() { } public static BString getMaskedString(Environment env, Object value) { - // Use IdentityHashMap for much better memory efficiency - // Only stores identity-based references, not hash-based ones - IdentityHashMap visitedValues = new IdentityHashMap<>(); - return StringUtils.fromString(getMaskedStringInternal(env.getRuntime(), value, visitedValues)); - } - - static String getMaskedStringInternal(Runtime runtime, Object value, IdentityHashMap visitedValues) { - if (isBasicType(value)) { - return StringUtils.getStringValue(value); - } - - // Use identity-based checking instead of hash-based - if (visitedValues.put(value, Boolean.TRUE) != null) { - throw ErrorCreator.createError(CYCLIC_REFERENCE_ERROR); - } - - try { - return processValue(runtime, value, visitedValues); - } finally { - visitedValues.remove(value); - } - } - - private static String processValue(Runtime runtime, Object value, IdentityHashMap visitedValues) { - Type type = TypeUtils.getType(value); - - // Use switch-like pattern for better performance - return switch (value) { - case BMap mapValue -> processMapValue(runtime, mapValue, type, visitedValues); - case BTable tableValue -> processTableValue(runtime, tableValue, visitedValues); - case BArray listValue -> processArrayValue(runtime, listValue, visitedValues); - default -> StringUtils.getStringValue(value); - }; - } - - private static String processMapValue(Runtime runtime, BMap mapValue, Type valueType, IdentityHashMap visitedValues) { - if (valueType.getTag() != TypeTags.RECORD_TYPE_TAG) { - // For non-record maps, use default string representation - return StringUtils.getStringValue(mapValue); - } - - RecordType recType = (RecordType) valueType; - Map fields = recType.getFields(); - if (fields.isEmpty()) { - return "{}"; - } - - // More conservative and safer capacity estimation - // Use adaptive sizing based on actual field count with reasonable bounds - int fieldCount = fields.size(); - int baseCapacity = fieldCount <= 5 ? 64 : - fieldCount <= 20 ? 256 : - Math.min(fieldCount * 15, 2048); // Cap at 2KB for very large objects - - StringBuilder maskedString = new StringBuilder(baseCapacity); - maskedString.append('{'); - - Map> fieldAnnotations = extractFieldAnnotations(recType); - boolean first = true; - - for (Map.Entry entry : fields.entrySet()) { - String fieldName = entry.getKey(); - Object fieldValue = mapValue.get(StringUtils.fromString(fieldName)); - - if (fieldValue == null) { - continue; - } - - Optional> annotation = getLogSensitiveDataAnnotation(fieldAnnotations, fieldName); - Optional fieldStringValue; - - if (annotation.isPresent()) { - fieldStringValue = getStringValue(annotation.get(), fieldValue, runtime); - } else { - fieldStringValue = Optional.of(getMaskedStringInternal(runtime, fieldValue, visitedValues)); - } - - if (fieldStringValue.isPresent()) { - if (!first) { - maskedString.append(','); - } - appendFieldToJson(maskedString, fieldName, fieldStringValue.get(), - annotation.isPresent(), fieldValue); - first = false; - } - } - - maskedString.append('}'); - return maskedString.toString(); - } - - private static void appendFieldToJson(StringBuilder sb, String fieldName, String value, - boolean hasAnnotation, Object fieldValue) { - sb.append('"').append(fieldName).append("\":"); - if (hasAnnotation || fieldValue instanceof BString) { - sb.append('"').append(value).append('"'); - } else { - sb.append(value); - } - } - - private static String processTableValue(Runtime runtime, BTable tableValue, IdentityHashMap visitedValues) { - Collection values = tableValue.values(); - if (values.isEmpty()) { - return "[]"; - } - - // Safer capacity estimation with bounds checking - int valueCount = values.size(); - int baseCapacity = valueCount <= 10 ? 128 : - valueCount <= 100 ? 512 : - Math.min(valueCount * 20, 4096); // Cap at 4KB - - StringBuilder tableString = new StringBuilder(baseCapacity); - tableString.append('['); - - boolean first = true; - for (Object row : values) { - if (!first) { - tableString.append(','); - } - appendValueToArray(tableString, getMaskedStringInternal(runtime, row, visitedValues), row); - first = false; - } - - tableString.append(']'); - return tableString.toString(); - } - - private static String processArrayValue(Runtime runtime, BArray listValue, IdentityHashMap visitedValues) { - long length = listValue.getLength(); - if (length == 0) { - return "[]"; - } - - // Safe capacity calculation with overflow protection - int safeLength = length > Integer.MAX_VALUE / 20 ? Integer.MAX_VALUE / 20 : (int) length; - int baseCapacity = safeLength <= 20 ? 96 : - safeLength <= 200 ? 384 : - Math.min(safeLength * 12, 3072); // Cap at 3KB - - StringBuilder arrayString = new StringBuilder(baseCapacity); - arrayString.append('['); - - for (long i = 0; i < length; i++) { - if (i > 0) { - arrayString.append(','); - } - Object element = listValue.get(i); - String elementString = getMaskedStringInternal(runtime, element, visitedValues); - appendValueToArray(arrayString, elementString, element); - } - - arrayString.append(']'); - return arrayString.toString(); - } - - private static void appendValueToArray(StringBuilder sb, String value, Object originalValue) { - if (originalValue instanceof BString) { - sb.append('"').append(value).append('"'); - } else { - sb.append(value); - } - } - - static boolean isBasicType(Object value) { - return value == null || TypeUtils.getType(value).getTag() <= TypeTags.BOOLEAN_TAG; - } - - static Optional> getLogSensitiveDataAnnotation(Map> fieldAnnotations, String fieldName) { - BMap fieldAnnotationMap = fieldAnnotations.get(fieldName); - if (fieldAnnotationMap == null) { - return Optional.empty(); - } - - // Cache the keys array to avoid repeated calls - Object[] keys = fieldAnnotationMap.getKeys(); - - // Optimized search - most annotation maps are small, so linear search is efficient - for (Object key : keys) { - if (key instanceof BString bStringKey) { - String keyValue = bStringKey.getValue(); - // Use more efficient string matching - check suffix first (likely to fail faster) - if (keyValue.endsWith(SENSITIVE_DATA_SUFFIX) && keyValue.startsWith(LOG_ANNOTATION_PREFIX)) { - Object annotation = fieldAnnotationMap.get(key); - if (annotation instanceof BMap bMapAnnotation) { - return Optional.of(bMapAnnotation); - } - // Found the target annotation type, no need to continue - break; - } - } - } - return Optional.empty(); - } - - static Map> extractFieldAnnotations(RecordType recordType) { - BMap annotations = recordType.getAnnotations(); - if (annotations == null) { - return Map.of(); - } - - // Use more efficient stream processing - return annotations.entrySet().stream() - .filter(entry -> { - String keyValue = entry.getKey().getValue(); - return keyValue.startsWith(FIELD_PREFIX) && entry.getValue() instanceof BMap; - }) - .collect(Collectors.toMap( - entry -> entry.getKey().getValue().substring(FIELD_PREFIX.length()), - entry -> (BMap) entry.getValue(), - (existing, replacement) -> existing // Handle potential duplicates - )); - } - - static Optional getStringValue(BMap annotation, Object realValue, Runtime runtime) { - Object strategy = annotation.get(STRATEGY_KEY); - if (strategy instanceof BString strategyStr && EXCLUDE_VALUE.getValue().equals(strategyStr.getValue())) { - return Optional.empty(); - } - if (strategy instanceof BMap replacementMap) { - Object replacement = replacementMap.get(REPLACEMENT_KEY); - if (replacement instanceof BString replacementStr) { - return Optional.of(replacementStr.getValue()); - } - if (replacement instanceof BFunctionPointer replacer) { - Object replacementString = replacer.call(runtime, StringUtils.fromString(StringUtils.getStringValue(realValue))); - if (replacementString instanceof BString replacementStrVal) { - return Optional.of(replacementStrVal.getValue()); - } - } + // Use try-with-resources for automatic cleanup + try (MaskedStringBuilder builder = MaskedStringBuilder.create(env.getRuntime())) { + return StringUtils.fromString(builder.build(value)); } - return Optional.of(StringUtils.getStringValue(realValue)); } } From 1b60ee21929e361bc7a4a2db61c66f50c3fd7900 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Tue, 2 Sep 2025 10:48:00 +0530 Subject: [PATCH 05/34] Fix checkstyle issues --- .../stdlib/log/MaskedStringBuilder.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java index b98f0733..957c4e54 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java +++ b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java @@ -53,7 +53,10 @@ public class MaskedStringBuilder implements AutoCloseable { private static final String LOG_ANNOTATION_PREFIX = "ballerina/log"; private static final String SENSITIVE_DATA_SUFFIX = ":SensitiveData"; - private static final BString CYCLIC_REFERENCE_ERROR = StringUtils.fromString("Cyclic value reference detected in the record"); + private static final BString CYCLIC_REFERENCE_ERROR = StringUtils.fromString("Cyclic value reference detected " + + "in the record"); + public static final BString MASKED_STRING_BUILDER_HAS_BEEN_CLOSED = StringUtils.fromString("MaskedStringBuilder" + + " has been closed"); private final Runtime runtime; private final IdentityHashMap visitedValues; @@ -84,7 +87,7 @@ public MaskedStringBuilder(Runtime runtime, int initialCapacity) { */ public String build(Object value) { if (closed) { - throw new IllegalStateException("MaskedStringBuilder has been closed"); + throw ErrorCreator.createError(MASKED_STRING_BUILDER_HAS_BEEN_CLOSED); } try { @@ -111,6 +114,7 @@ private String buildInternal(Object value) { // Use identity-based checking for cycle detection if (visitedValues.put(value, Boolean.TRUE) != null) { + // Panics on cyclic value references throw ErrorCreator.createError(CYCLIC_REFERENCE_ERROR); } @@ -329,7 +333,7 @@ public int getCapacity() { */ public void reset() { if (closed) { - throw new IllegalStateException("MaskedStringBuilder has been closed"); + throw ErrorCreator.createError(MASKED_STRING_BUILDER_HAS_BEEN_CLOSED); } visitedValues.clear(); stringBuilder.setLength(0); @@ -374,7 +378,8 @@ public static MaskedStringBuilder create(Runtime runtime, int initialCapacity) { return new MaskedStringBuilder(runtime, initialCapacity); } - static Optional> getLogSensitiveDataAnnotation(Map> fieldAnnotations, String fieldName) { + static Optional> getLogSensitiveDataAnnotation(Map> fieldAnnotations, + String fieldName) { BMap fieldAnnotationMap = fieldAnnotations.get(fieldName); if (fieldAnnotationMap == null) { return Optional.empty(); @@ -427,7 +432,8 @@ static Optional getStringValue(BMap annotation, Object realValue, return Optional.of(replacementStr.getValue()); } if (replacement instanceof BFunctionPointer replacer) { - Object replacementString = replacer.call(runtime, StringUtils.fromString(StringUtils.getStringValue(realValue))); + Object replacementString = replacer.call(runtime, + StringUtils.fromString(StringUtils.getStringValue(realValue))); if (replacementString instanceof BString replacementStrVal) { return Optional.of(replacementStrVal.getValue()); } From 0180d5321308a994d04e89e87397c6f8f1d0fd74 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Wed, 3 Sep 2025 08:09:18 +0530 Subject: [PATCH 06/34] Add sensitive data masking support for root logger --- ballerina/natives.bal | 7 ++++--- ballerina/root_logger.bal | 4 +++- ballerina/sensitive_data_masking.bal | 8 ++++++-- .../io/ballerina/stdlib/log/MaskedStringBuilder.java | 11 ++++++++++- .../src/main/java/io/ballerina/stdlib/log/Utils.java | 2 +- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/ballerina/natives.bal b/ballerina/natives.bal index e17c9165..7aa475d7 100644 --- a/ballerina/natives.bal +++ b/ballerina/natives.bal @@ -176,8 +176,8 @@ public isolated function processTemplate(PrintableRawTemplate template) returns string insertionStr = insertion is PrintableRawTemplate ? processTemplate(insertion) : insertion is Valuer ? - insertion().toString() : - insertion.toString(); + (enableSensitiveDataMasking ? toMaskedString(insertion()) : insertion().toString()) : + (enableSensitiveDataMasking ? toMaskedString(insertion) : insertion.toString()); result += insertionStr + templateStrings[i]; } return result; @@ -367,7 +367,8 @@ isolated function printLogFmt(LogRecord logRecord) returns string { value = v.toBalString(); } _ => { - value = v is string ? string `${escape(v.toString())}` : v.toString(); + value = v is string ? string `${escape(v.toString())}` : + (enableSensitiveDataMasking ? toMaskedString(v) : v.toString()); } } if message == "" { diff --git a/ballerina/root_logger.bal b/ballerina/root_logger.bal index 1ac6a692..3756c356 100644 --- a/ballerina/root_logger.bal +++ b/ballerina/root_logger.bal @@ -143,7 +143,9 @@ isolated class RootLogger { logRecord[k] = v is Valuer ? v() : v is PrintableRawTemplate ? processMessage(v) : v; } - string logOutput = self.format == JSON_FORMAT ? logRecord.toJsonString() : printLogFmt(logRecord); + string logOutput = self.format == JSON_FORMAT ? + (enableSensitiveDataMasking ? toMaskedString(logRecord) : logRecord.toJsonString()) : + printLogFmt(logRecord); lock { if outputFilePath is string { diff --git a/ballerina/sensitive_data_masking.bal b/ballerina/sensitive_data_masking.bal index 8763a1a5..27e55f6f 100644 --- a/ballerina/sensitive_data_masking.bal +++ b/ballerina/sensitive_data_masking.bal @@ -45,8 +45,12 @@ public type SensitiveDataConfig record {| # + strategy - The masking strategy to apply (default: EXCLUDE) public annotation SensitiveDataConfig SensitiveData on record field; -configurable boolean enableSensitiveDataMasking = true; +configurable boolean enableSensitiveDataMasking = false; -public function getMaskedString(anydata data) returns string = @java:Method { +# Returns a masked string representation of the given data based on the sensitive data masking configuration. +# +# + data - The data to be masked +# + return - The masked string representation of the data +public isolated function toMaskedString(anydata data) returns string = @java:Method { 'class: "io.ballerina.stdlib.log.Utils" } external; diff --git a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java index 957c4e54..da719a3c 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java +++ b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java @@ -130,7 +130,7 @@ private String processValue(Object value) { return switch (value) { // Processing only the structured types, since the basic types does not contain the - // inherent type information unless they are part of a structured type. + // inherent type information. case BMap mapValue -> processMapValue(mapValue, type); case BTable tableValue -> processTableValue(tableValue); case BArray listValue -> processArrayValue(listValue); @@ -157,9 +157,18 @@ private String processMapValue(BMap mapValue, Type valueType) { for (Map.Entry entry : fields.entrySet()) { String fieldName = entry.getKey(); + BString fieldNameKey = StringUtils.fromString(fieldName); Object fieldValue = mapValue.get(StringUtils.fromString(fieldName)); if (fieldValue == null) { + // For optional fields with default value as null, the map will contain the key + if (mapValue.containsKey(fieldNameKey)) { + // Add the field with null value + if (!first) { + stringBuilder.append(','); + } + appendFieldToJson(fieldName, "null", false, null); + } continue; } diff --git a/native/src/main/java/io/ballerina/stdlib/log/Utils.java b/native/src/main/java/io/ballerina/stdlib/log/Utils.java index cd54f393..dc69d43f 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/Utils.java +++ b/native/src/main/java/io/ballerina/stdlib/log/Utils.java @@ -100,7 +100,7 @@ public static BString getCurrentTime() { .format(new Date())); } - public static BString getMaskedString(Environment env, Object value) { + public static BString toMaskedString(Environment env, Object value) { // Use try-with-resources for automatic cleanup try (MaskedStringBuilder builder = MaskedStringBuilder.create(env.getRuntime())) { return StringUtils.fromString(builder.build(value)); From 784083884723b6fe5866312df12da5cc7a48d679 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Wed, 3 Sep 2025 08:39:07 +0530 Subject: [PATCH 07/34] Add annotation caching support --- .../stdlib/log/MaskedStringBuilder.java | 175 ++++++++++++------ 1 file changed, 123 insertions(+), 52 deletions(-) diff --git a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java index da719a3c..d7b7d28a 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java +++ b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java @@ -36,11 +36,12 @@ import java.util.IdentityHashMap; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; /** * High-performance builder for creating masked string representations of Ballerina values. - * Implements Closeable for proper resource management and memory efficiency. + * Implements AutoCloseable for proper resource management and memory efficiency. * * @since 2.14.0 */ @@ -58,25 +59,41 @@ public class MaskedStringBuilder implements AutoCloseable { public static final BString MASKED_STRING_BUILDER_HAS_BEEN_CLOSED = StringUtils.fromString("MaskedStringBuilder" + " has been closed"); + // Cache for field annotations to avoid repeated extraction + private static final Map>> ANNOTATION_CACHE = new ConcurrentHashMap<>(); + private static final int MAX_CACHE_SIZE = 1000; + + private static final char[] QUOTE_ESCAPE = {'\\', '"'}; + private static final char[] BACKSLASH_ESCAPE = {'\\', '\\'}; + private static final char[] NEWLINE_ESCAPE = {'\\', 'n'}; + private static final char[] TAB_ESCAPE = {'\\', 't'}; + private static final char[] CARRIAGE_RETURN_ESCAPE = {'\\', 'r'}; + private static final char[] BACKSPACE_ESCAPE = {'\\', 'b'}; + private static final char[] FORM_FEED_ESCAPE = {'\\', 'f'}; + private final Runtime runtime; private final IdentityHashMap visitedValues; private StringBuilder stringBuilder; + private StringBuilder escapeBuffer; private boolean closed = false; // Initial capacity configuration private static final int DEFAULT_INITIAL_CAPACITY = 256; - private static final int MAX_REUSABLE_CAPACITY = 8192; // 8KB threshold for reuse + private static final int MAX_REUSABLE_CAPACITY = 8192; + private static final int ESCAPE_BUFFER_SIZE = 64; public MaskedStringBuilder(Runtime runtime) { this.runtime = runtime; this.visitedValues = new IdentityHashMap<>(); this.stringBuilder = new StringBuilder(DEFAULT_INITIAL_CAPACITY); + this.escapeBuffer = new StringBuilder(ESCAPE_BUFFER_SIZE); } public MaskedStringBuilder(Runtime runtime, int initialCapacity) { this.runtime = runtime; this.visitedValues = new IdentityHashMap<>(); this.stringBuilder = new StringBuilder(Math.max(initialCapacity, DEFAULT_INITIAL_CAPACITY)); + this.escapeBuffer = new StringBuilder(ESCAPE_BUFFER_SIZE); } /** @@ -101,6 +118,11 @@ public String build(Object value) { stringBuilder = new StringBuilder(DEFAULT_INITIAL_CAPACITY); } + // Reset escape buffer if it grew too large + if (escapeBuffer.capacity() > ESCAPE_BUFFER_SIZE * 4) { + escapeBuffer = new StringBuilder(ESCAPE_BUFFER_SIZE); + } + return result; } finally { visitedValues.clear(); @@ -152,13 +174,14 @@ private String processMapValue(BMap mapValue, Type valueType) { int startPos = stringBuilder.length(); stringBuilder.append('{'); - Map> fieldAnnotations = extractFieldAnnotations(recType); + // Use cached field annotations for better performance + Map> fieldAnnotations = getCachedFieldAnnotations(recType); boolean first = true; for (Map.Entry entry : fields.entrySet()) { String fieldName = entry.getKey(); BString fieldNameKey = StringUtils.fromString(fieldName); - Object fieldValue = mapValue.get(StringUtils.fromString(fieldName)); + Object fieldValue = mapValue.get(fieldNameKey); if (fieldValue == null) { // For optional fields with default value as null, the map will contain the key @@ -167,7 +190,8 @@ private String processMapValue(BMap mapValue, Type valueType) { if (!first) { stringBuilder.append(','); } - appendFieldToJson(fieldName, "null", false, null); + appendFieldToJsonOptimized(fieldName, "null", false, null); + first = false; } continue; } @@ -185,7 +209,7 @@ private String processMapValue(BMap mapValue, Type valueType) { if (!first) { stringBuilder.append(','); } - appendFieldToJson(fieldName, fieldStringValue.get(), annotation.isPresent(), fieldValue); + appendFieldToJsonOptimized(fieldName, fieldStringValue.get(), annotation.isPresent(), fieldValue); first = false; } } @@ -197,15 +221,81 @@ private String processMapValue(BMap mapValue, Type valueType) { return result; } - private void appendFieldToJson(String fieldName, String value, boolean hasAnnotation, Object fieldValue) { - stringBuilder.append('"').append(escapeJsonString(fieldName)).append("\":"); + /** + * Optimized version of appendFieldToJson that writes directly to StringBuilder + * without creating intermediate String objects for better performance. + */ + private void appendFieldToJsonOptimized(String fieldName, String value, boolean hasAnnotation, Object fieldValue) { + stringBuilder.append('"'); + appendEscapedStringOptimized(fieldName); + stringBuilder.append("\":"); if (hasAnnotation || fieldValue instanceof BString) { - stringBuilder.append('"').append(escapeJsonString(value)).append('"'); + stringBuilder.append('"'); + appendEscapedStringOptimized(value); + stringBuilder.append('"'); } else { stringBuilder.append(value); } } + /** + * Optimized method to append escaped string directly to the main StringBuilder. + * This avoids creating intermediate String objects for better performance. + */ + private void appendEscapedStringOptimized(String input) { + if (input == null) { + stringBuilder.append("null"); + return; + } + + if (!needsEscaping(input)) { + stringBuilder.append(input); + return; + } + + for (int i = 0; i < input.length(); i++) { + char c = input.charAt(i); + switch (c) { + case '"' -> stringBuilder.append(QUOTE_ESCAPE); + case '\\' -> stringBuilder.append(BACKSLASH_ESCAPE); + case '\b' -> stringBuilder.append(BACKSPACE_ESCAPE); + case '\f' -> stringBuilder.append(FORM_FEED_ESCAPE); + case '\n' -> stringBuilder.append(NEWLINE_ESCAPE); + case '\r' -> stringBuilder.append(CARRIAGE_RETURN_ESCAPE); + case '\t' -> stringBuilder.append(TAB_ESCAPE); + default -> { + if (c < 0x20 || c == 0x7F) { + stringBuilder.append("\\u"); + stringBuilder.append(String.format("%04x", (int) c)); + } else { + stringBuilder.append(c); + } + } + } + } + } + + /** + * Get cached field annotations for better performance. + * Implements simple cache eviction when cache grows too large. + */ + private Map> getCachedFieldAnnotations(RecordType recordType) { + Map> cached = ANNOTATION_CACHE.get(recordType); + if (cached != null) { + return cached; + } + + // Implement simple cache size management + if (ANNOTATION_CACHE.size() >= MAX_CACHE_SIZE) { + // Clear half the cache when it gets too large (simple eviction strategy) + ANNOTATION_CACHE.entrySet().removeIf(entry -> System.identityHashCode(entry.getKey()) % 2 == 0); + } + + Map> annotations = extractFieldAnnotations(recordType); + ANNOTATION_CACHE.put(recordType, annotations); + return annotations; + } + private String processTableValue(BTable tableValue) { Collection values = tableValue.values(); if (values.isEmpty()) { @@ -261,51 +351,19 @@ private String processArrayValue(BArray listValue) { private void appendValueToArray(String value, Object originalValue) { if (originalValue instanceof BString) { - stringBuilder.append('"').append(escapeJsonString(value)).append('"'); + stringBuilder.append('"'); + appendEscapedStringOptimized(value); + stringBuilder.append('"'); } else { stringBuilder.append(value); } } /** - * Escape characters in a string for safe JSON representation. - * Handles quotes, backslashes, and control characters. - * - * @param input the input string to escape - * @return the escaped string + * Check if a value is a basic type that doesn't need complex processing. */ - private static String escapeJsonString(String input) { - if (input == null) { - return "null"; - } - - if (!needsEscaping(input)) { - return input; - } - - StringBuilder escaped = new StringBuilder(input.length() + 16); - - for (int i = 0; i < input.length(); i++) { - char c = input.charAt(i); - switch (c) { - case '"' -> escaped.append("\\\""); - case '\\' -> escaped.append("\\\\"); - case '\b' -> escaped.append("\\b"); - case '\f' -> escaped.append("\\f"); - case '\n' -> escaped.append("\\n"); - case '\r' -> escaped.append("\\r"); - case '\t' -> escaped.append("\\t"); - default -> { - if (c < 0x20 || c == 0x7F) { - escaped.append(String.format("\\u%04x", (int) c)); - } else { - escaped.append(c); - } - } - } - } - - return escaped.toString(); + private static boolean isBasicType(Object value) { + return value == null || TypeUtils.getType(value).getTag() <= TypeTags.BOOLEAN_TAG; } /** @@ -315,17 +373,13 @@ private static String escapeJsonString(String input) { private static boolean needsEscaping(String input) { for (int i = 0; i < input.length(); i++) { char c = input.charAt(i); - if (c == '"' || c == '\\' || c < 0x20 || c == 0x7F) { + if (c == '"' || c == '\\' || (c & 0xFFE0) == 0 || c == 0x7F) { return true; } } return false; } - private static boolean isBasicType(Object value) { - return value == null || TypeUtils.getType(value).getTag() <= TypeTags.BOOLEAN_TAG; - } - /** * Get the current capacity of the internal StringBuilder. * Useful for monitoring memory usage. @@ -346,6 +400,7 @@ public void reset() { } visitedValues.clear(); stringBuilder.setLength(0); + escapeBuffer.setLength(0); } /** @@ -362,10 +417,26 @@ public void close() { if (!closed) { visitedValues.clear(); stringBuilder = null; + escapeBuffer = null; closed = true; } } + /** + * Clear the annotation cache to free memory. + * Should be called periodically in long-running applications. + */ + public static void clearAnnotationCache() { + ANNOTATION_CACHE.clear(); + } + + /** + * Get the size of the annotation cache for monitoring purposes. + */ + public static int getAnnotationCacheSize() { + return ANNOTATION_CACHE.size(); + } + /** * Create a new MaskedStringBuilder instance with default settings. * From 34904990165adac01fc96520b046e68127d80513 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Wed, 17 Sep 2025 07:57:19 +0530 Subject: [PATCH 08/34] Fix xml toString --- .../main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java index d7b7d28a..a20ff17e 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java +++ b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java @@ -31,6 +31,7 @@ import io.ballerina.runtime.api.values.BMap; import io.ballerina.runtime.api.values.BString; import io.ballerina.runtime.api.values.BTable; +import io.ballerina.runtime.api.values.BXml; import java.util.Collection; import java.util.IdentityHashMap; @@ -156,6 +157,7 @@ private String processValue(Object value) { case BMap mapValue -> processMapValue(mapValue, type); case BTable tableValue -> processTableValue(tableValue); case BArray listValue -> processArrayValue(listValue); + case BXml xmlValue -> String.format("\"%s\"", StringUtils.getStringValue(xmlValue)); default -> StringUtils.getStringValue(value); }; } From e6e3aa4175f0ccf8a3eae3ceb529dd537ef26d38 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Wed, 17 Sep 2025 11:34:36 +0530 Subject: [PATCH 09/34] Support undefined fields in the record to string --- .../stdlib/log/MaskedStringBuilder.java | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java index a20ff17e..659ce7f9 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java +++ b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java @@ -131,6 +131,9 @@ public String build(Object value) { } private String buildInternal(Object value) { + if (value == null) { + return "null"; + } if (isBasicType(value)) { return StringUtils.getStringValue(value); } @@ -157,7 +160,6 @@ private String processValue(Object value) { case BMap mapValue -> processMapValue(mapValue, type); case BTable tableValue -> processTableValue(tableValue); case BArray listValue -> processArrayValue(listValue); - case BXml xmlValue -> String.format("\"%s\"", StringUtils.getStringValue(xmlValue)); default -> StringUtils.getStringValue(value); }; } @@ -170,7 +172,7 @@ private String processMapValue(BMap mapValue, Type valueType) { RecordType recType = (RecordType) valueType; Map fields = recType.getFields(); if (fields.isEmpty()) { - return "{}"; + return StringUtils.getStringValue(mapValue); } int startPos = stringBuilder.length(); @@ -180,38 +182,36 @@ private String processMapValue(BMap mapValue, Type valueType) { Map> fieldAnnotations = getCachedFieldAnnotations(recType); boolean first = true; - for (Map.Entry entry : fields.entrySet()) { - String fieldName = entry.getKey(); - BString fieldNameKey = StringUtils.fromString(fieldName); - Object fieldValue = mapValue.get(fieldNameKey); + for (Object key : mapValue.getKeys()) { + if (!(key instanceof BString keyStr)) { + continue; + } + Object fieldValue = mapValue.get(key); + String fieldName = keyStr.getValue(); + if (fields.containsKey(fieldName)) { + Optional> annotation = getLogSensitiveDataAnnotation(fieldAnnotations, fieldName); + Optional fieldStringValue; + + if (annotation.isPresent()) { + fieldStringValue = getStringValue(annotation.get(), fieldValue, runtime); + } else { + fieldStringValue = Optional.of(buildInternal(fieldValue)); + } - if (fieldValue == null) { - // For optional fields with default value as null, the map will contain the key - if (mapValue.containsKey(fieldNameKey)) { - // Add the field with null value + if (fieldStringValue.isPresent()) { if (!first) { stringBuilder.append(','); } - appendFieldToJsonOptimized(fieldName, "null", false, null); + appendFieldToJsonOptimized(fieldName, fieldStringValue.get(), annotation.isPresent(), fieldValue); first = false; } - continue; - } - - Optional> annotation = getLogSensitiveDataAnnotation(fieldAnnotations, fieldName); - Optional fieldStringValue; - - if (annotation.isPresent()) { - fieldStringValue = getStringValue(annotation.get(), fieldValue, runtime); } else { - fieldStringValue = Optional.of(buildInternal(fieldValue)); - } - - if (fieldStringValue.isPresent()) { + // Handle dynamic fields not defined in the record type + String fieldStringValue = buildInternal(fieldValue); if (!first) { stringBuilder.append(','); } - appendFieldToJsonOptimized(fieldName, fieldStringValue.get(), annotation.isPresent(), fieldValue); + appendFieldToJsonOptimized(fieldName, fieldStringValue, false, fieldValue); first = false; } } @@ -231,7 +231,7 @@ private void appendFieldToJsonOptimized(String fieldName, String value, boolean stringBuilder.append('"'); appendEscapedStringOptimized(fieldName); stringBuilder.append("\":"); - if (hasAnnotation || fieldValue instanceof BString) { + if (hasAnnotation || fieldValue instanceof BString || fieldValue instanceof BXml) { stringBuilder.append('"'); appendEscapedStringOptimized(value); stringBuilder.append('"'); From 1ad62cf638bfd79f4906d8aa1b47ed7ad3b90c98 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Wed, 17 Sep 2025 11:37:17 +0530 Subject: [PATCH 10/34] Add tests for masked string function --- ballerina/tests/masked_string_test.bal | 329 +++++++++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100644 ballerina/tests/masked_string_test.bal diff --git a/ballerina/tests/masked_string_test.bal b/ballerina/tests/masked_string_test.bal new file mode 100644 index 00000000..e37d1130 --- /dev/null +++ b/ballerina/tests/masked_string_test.bal @@ -0,0 +1,329 @@ +// Copyright (c) 2025 WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/test; + +isolated function maskStringPartially(string input) returns string { + int len = input.length(); + if len <= 6 { + return "******"; + } + string maskedString = input.substring(0, 3); + foreach int i in 3 ... len - 4 { + maskedString += "*"; + } + maskedString += input.substring(len - 3); + return maskedString; +}; + +type User record {| + string name; + @SensitiveData + string ssn; + @SensitiveData {strategy: {replacement: "*****"}} + string password; + @SensitiveData {strategy: {replacement: maskStringPartially}} + string mail; + @SensitiveData {strategy: EXCLUDE} + string creditCard; +|}; + +@test:Config { + groups: ["maskedString"] +} +function testMaskedString() { + User user = { + name: "John Doe", + ssn: "123-45-6789", + password: "password123", + mail: "john.doe@example.com", + creditCard: "4111-1111-1111-1111" + }; + string maskedUserStr = toMaskedString(user); + string expectedStr = string `{"name":"John Doe","password":"*****","mail":"joh**************com"}`; + test:assertEquals(maskedUserStr, expectedStr); +} + +type RecordWithAnydataValues record {| + string str; + int|float num; + boolean bool; + map jsonMap; + table> tableData; + anydata[] arr; + xml xmlRaw; + xml:Text xmlText; + [int, float, string] tuple; +|}; + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithAnydataValues() { + RecordWithAnydataValues anydataRec = { + str: "Test String", + num: 123.45, + bool: true, + jsonMap: {key1: "value1", key2: 2}, + tableData: table [ + {col1: "row1col1", col2: "row1col2"}, + {col1: "row2col1", col2: "row2col2"} + ], + arr: ["elem1", 2, {key: "value"}], + xmlRaw: xml `UserAdminReminderDon't forget the meeting!`, + xmlText: xml `Just some text`, + tuple: [1, 2.5, "three"] + }; + string maskedAnydataRecStr = toMaskedString(anydataRec); + string expectedStr = string `{"str":"Test String","num":123.45,"bool":true,"jsonMap":{"key1":"value1","key2":2},"tableData":[{"col1":"row1col1","col2":"row1col2"},{"col1":"row2col1","col2":"row2col2"}],"arr":["elem1",2,{"key":"value"}],"xmlRaw":"UserAdminReminderDon't forget the meeting!","xmlText":"Just some text","tuple":[1,2.5,"three"]}`; + test:assertEquals(maskedAnydataRecStr, expectedStr); +} + +type OpenAnydataRecord record { + string name; + @SensitiveData + anydata sensitiveField; +}; + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithOpenAnydataRecord() { + OpenAnydataRecord fieldRec = { + name: "Field Record", + sensitiveField: "Sensitive Data", + "extraField": "extraValue" + }; + + OpenAnydataRecord openRec = { + name: "Open Record", + sensitiveField: {key1: "value1", key2: 2, key3: true}, + "extraField": "extraValue", + "extraMapField": {mapKey: "mapValue"}, + "extraArrayField": [1, "two", 3.0], + "extraRecordField": fieldRec + }; + string maskedOpenRecStr = toMaskedString(openRec); + string expectedStr = string `{"name":"Open Record","extraField":"extraValue","extraMapField":{"mapKey":"mapValue"},"extraArrayField":[1,"two",3.0],"extraRecordField":{"name":"Field Record","extraField":"extraValue"}}`; + test:assertEquals(maskedOpenRecStr, expectedStr); +} + +type Record1 record {| + string field1; + @SensitiveData + Record2 field2; + Record2 field3; +|}; + +type Record2 record {| + string subField1; + @SensitiveData {strategy: {replacement: "###"}} + string subField2; + @SensitiveData {strategy: {replacement: maskStringPartially}} + string subField3; + @SensitiveData {strategy: EXCLUDE} + string subField4; +|}; + +type Record3 record {| + string info; + @SensitiveData + string details; +|}; + +type NestedRecord record {| + string name; + @SensitiveData + Record1 details1; + Record1 details2; + Record3[] records; +|}; + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithNestedRecords() { + NestedRecord nestedRec = { + name: "Nested Record", + details1: { + field1: "Field1 Value", + field2: { + subField1: "SubField1 Value", + subField2: "SubField2 Value", + subField3: "SubField3 Value", + subField4: "SubField4 Value" + }, + field3: { + subField1: "SubField1 Value", + subField2: "SubField2 Value", + subField3: "SubField3 Value", + subField4: "SubField4 Value" + } + }, + details2: { + field1: "Field1 Value", + field2: { + subField1: "SubField1 Value", + subField2: "SubField2 Value", + subField3: "SubField3 Value", + subField4: "SubField4 Value" + }, + field3: { + subField1: "SubField1 Value", + subField2: "SubField2 Value", + subField3: "SubField3 Value", + subField4: "SubField4 Value" + } + }, + records: [ + {info: "Record1 Info", details: "Record1 Details"}, + {info: "Record2 Info", details: "Record2 Details"} + ] + }; + string maskedNestedRecStr = toMaskedString(nestedRec); + string expectedStr = string `{"name":"Nested Record","details2":{"field1":"Field1 Value","field3":{"subField1":"SubField1 Value","subField2":"###","subField3":"Sub*********lue"}},"records":[{"info":"Record1 Info"},{"info":"Record2 Info"}]}`; + test:assertEquals(maskedNestedRecStr, expectedStr); +} + +type NilableSensitiveFieldRecord record {| + string name; + @SensitiveData + string? sensitiveField; + int? id; +|}; + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithNilableSensitiveField() { + NilableSensitiveFieldRecord recWithNil = { + name: "Nilable Record", + sensitiveField: (), + id: null + }; + string maskedRecWithNilStr = toMaskedString(recWithNil); + string expectedStr = string `{"name":"Nilable Record","id":null}`; + test:assertEquals(maskedRecWithNilStr, expectedStr); +} + +type OptionalSensitiveFieldRecord record {| + string name; + @SensitiveData + string sensitiveField?; + int id?; +|}; + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithOptionalSensitiveField() { + OptionalSensitiveFieldRecord recWithOptional = { + name: "Optional Record", + id: 101 + }; + string maskedRecWithOptionalStr = toMaskedString(recWithOptional); + string expectedStr = string `{"name":"Optional Record","id":101}`; + test:assertEquals(maskedRecWithOptionalStr, expectedStr); + + recWithOptional.sensitiveField = "Sensitive Data"; + string maskedRecWithOptionalSetStr = toMaskedString(recWithOptional); + test:assertEquals(maskedRecWithOptionalSetStr, expectedStr); + + recWithOptional.id = (); + string maskedRecWithOptionalSetNilStr = toMaskedString(recWithOptional); + string expectedStrWithoutId = string `{"name":"Optional Record"}`; + test:assertEquals(maskedRecWithOptionalSetNilStr, expectedStrWithoutId); +} + +type NeverSensitiveFieldRecord record {| + string name; + @SensitiveData + never sensitiveField?; +|}; + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithNeverSensitiveField() { + NeverSensitiveFieldRecord rec = { + name: "Never Record", + sensitiveField: () + }; + string maskedRecStr = toMaskedString(rec); + string expectedStr = string `{"name":"Never Record"}`; + test:assertEquals(maskedRecStr, expectedStr); +} + +type RecordWithRestField record {| + string name; + @SensitiveData + string sensitiveField; + string...; +|}; + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithRestField() { + RecordWithRestField rec = { + name: "Rest Field Record", + sensitiveField: "Sensitive Data", + "extraField1": "extraValue1", + "extraField2": "extraValue2" + }; + string maskedRecStr = toMaskedString(rec); + string expectedStr = string `{"name":"Rest Field Record","extraField1":"extraValue1","extraField2":"extraValue2"}`; + test:assertEquals(maskedRecStr, expectedStr); +} + +type CyclicRecord record {| + string name; + CyclicRecord child?; +|}; + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithCyclicRecord() { + CyclicRecord rec = { + name: "name" + }; + rec.child = rec; + string|error maskedRecStr = trap toMaskedString(rec); + if maskedRecStr is string { + test:assertFail("Expected an error due to cyclic value reference, but got a string"); + } + test:assertEquals(maskedRecStr.message(), "Cyclic value reference detected in the record"); +} + +type RecordWithCyclicSensitiveField record {| + string name; + @SensitiveData + RecordWithCyclicSensitiveField child?; +|}; + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithCyclicSensitiveField() { + RecordWithCyclicSensitiveField rec = { + name: "name" + }; + rec.child = rec; + string maskedRecStr = toMaskedString(rec); + string expectedStr = string `{"name":"name"}`; + test:assertEquals(maskedRecStr, expectedStr); +} From 2f2e3cb0495d731f75da2811f8fe15ea759a9219 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Wed, 17 Sep 2025 12:10:40 +0530 Subject: [PATCH 11/34] Refactor class to reduce cognitive complexity --- .../stdlib/log/MaskedStringBuilder.java | 70 +++++++++++-------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java index 659ce7f9..e747ceaf 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java +++ b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java @@ -175,11 +175,26 @@ private String processMapValue(BMap mapValue, Type valueType) { return StringUtils.getStringValue(mapValue); } + return processRecordValue(mapValue, recType, fields); + } + + private String processRecordValue(BMap mapValue, RecordType recType, Map fields) { int startPos = stringBuilder.length(); stringBuilder.append('{'); // Use cached field annotations for better performance Map> fieldAnnotations = getCachedFieldAnnotations(recType); + addRecordFields(mapValue, fields, fieldAnnotations); + + stringBuilder.append('}'); + + String result = stringBuilder.substring(startPos); + stringBuilder.setLength(startPos); + return result; + } + + private void addRecordFields(BMap mapValue, Map fields, + Map> fieldAnnotations) { boolean first = true; for (Object key : mapValue.getKeys()) { @@ -188,39 +203,36 @@ private String processMapValue(BMap mapValue, Type valueType) { } Object fieldValue = mapValue.get(key); String fieldName = keyStr.getValue(); - if (fields.containsKey(fieldName)) { - Optional> annotation = getLogSensitiveDataAnnotation(fieldAnnotations, fieldName); - Optional fieldStringValue; - - if (annotation.isPresent()) { - fieldStringValue = getStringValue(annotation.get(), fieldValue, runtime); - } else { - fieldStringValue = Optional.of(buildInternal(fieldValue)); - } + first = fields.containsKey(fieldName) ? + addDefinedFieldValue(fieldAnnotations, fieldName, fieldValue, first) : + addDynamicFieldValue(fieldValue, first, fieldName); + } + } - if (fieldStringValue.isPresent()) { - if (!first) { - stringBuilder.append(','); - } - appendFieldToJsonOptimized(fieldName, fieldStringValue.get(), annotation.isPresent(), fieldValue); - first = false; - } - } else { - // Handle dynamic fields not defined in the record type - String fieldStringValue = buildInternal(fieldValue); - if (!first) { - stringBuilder.append(','); - } - appendFieldToJsonOptimized(fieldName, fieldStringValue, false, fieldValue); - first = false; - } + private boolean addDynamicFieldValue(Object fieldValue, boolean first, String fieldName) { + String fieldStringValue = buildInternal(fieldValue); + if (!first) { + stringBuilder.append(','); } + appendFieldToJsonOptimized(fieldName, fieldStringValue, false, fieldValue); + return false; + } - stringBuilder.append('}'); + private boolean addDefinedFieldValue(Map> fieldAnnotations, String fieldName, Object fieldValue, + boolean first) { + Optional> annotation = getLogSensitiveDataAnnotation(fieldAnnotations, fieldName); + Optional fieldStringValue = annotation + .map(fieldAnnotation -> getStringValue(fieldAnnotation, fieldValue, runtime)) + .orElseGet(() -> Optional.of(buildInternal(fieldValue))); - String result = stringBuilder.substring(startPos); - stringBuilder.setLength(startPos); - return result; + if (fieldStringValue.isPresent()) { + if (!first) { + stringBuilder.append(','); + } + appendFieldToJsonOptimized(fieldName, fieldStringValue.get(), annotation.isPresent(), fieldValue); + first = false; + } + return first; } /** From 7dbb203fb3e0750676f3eb4f0125e68a607d9b62 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Wed, 17 Sep 2025 13:47:04 +0530 Subject: [PATCH 12/34] Add support for map value --- ballerina/tests/masked_string_test.bal | 31 +++++++++++++++++++ .../stdlib/log/MaskedStringBuilder.java | 26 +++++++--------- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/ballerina/tests/masked_string_test.bal b/ballerina/tests/masked_string_test.bal index e37d1130..0367feea 100644 --- a/ballerina/tests/masked_string_test.bal +++ b/ballerina/tests/masked_string_test.bal @@ -327,3 +327,34 @@ function testMaskedStringWithCyclicSensitiveField() { string expectedStr = string `{"name":"name"}`; test:assertEquals(maskedRecStr, expectedStr); } + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithMap() { + map jsonMap = { + key1: "value1", + key2: 2, + key3: true, + key4: {nestedKey: "nestedValue"}, + key5: [1, "two", 3.0] + }; + string maskedMapStr = toMaskedString(jsonMap); + string expectedStr = string `{"key1":"value1","key2":2,"key3":true,"key4":{"nestedKey":"nestedValue"},"key5":[1,"two",3.0]}`; + test:assertEquals(maskedMapStr, expectedStr); + + User user = { + name: "John Doe", + ssn: "123-45-6789", + password: "password123", + mail: "john.doe@example.com", + creditCard: "4111-1111-1111-1111" + }; + map mapWithSensitiveData = { + normalKey: "normalValue", + sensitiveKey: user + }; + string maskedMapWithSensitiveDataStr = toMaskedString(mapWithSensitiveData); + string expectedMapWithSensitiveDataStr = string `{"normalKey":"normalValue","sensitiveKey":{"name":"John Doe","password":"*****","mail":"joh**************com"}}`; + test:assertEquals(maskedMapWithSensitiveDataStr, expectedMapWithSensitiveDataStr); +} diff --git a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java index e747ceaf..c9b852cd 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java +++ b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java @@ -165,27 +165,25 @@ private String processValue(Object value) { } private String processMapValue(BMap mapValue, Type valueType) { - if (valueType.getTag() != TypeTags.RECORD_TYPE_TAG) { - return StringUtils.getStringValue(mapValue); + Map fields = Map.of(); + Map> fieldAnnotations = Map.of(); + + if (valueType.getTag() == TypeTags.RECORD_TYPE_TAG) { + RecordType recType = (RecordType) valueType; + fields = recType.getFields(); + // Use cached field annotations for better performance + fieldAnnotations = getCachedFieldAnnotations(recType); } - RecordType recType = (RecordType) valueType; - Map fields = recType.getFields(); - if (fields.isEmpty()) { - return StringUtils.getStringValue(mapValue); - } - - return processRecordValue(mapValue, recType, fields); + return processRecordValue(mapValue, fieldAnnotations, fields); } - private String processRecordValue(BMap mapValue, RecordType recType, Map fields) { + private String processRecordValue(BMap mapValue, Map> fieldAnnotations, + Map fields) { int startPos = stringBuilder.length(); - stringBuilder.append('{'); - // Use cached field annotations for better performance - Map> fieldAnnotations = getCachedFieldAnnotations(recType); + stringBuilder.append('{'); addRecordFields(mapValue, fields, fieldAnnotations); - stringBuilder.append('}'); String result = stringBuilder.substring(startPos); From 78298b21a505c7c2909a9e7ccb63c8c288858ed3 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Fri, 19 Sep 2025 08:59:51 +0530 Subject: [PATCH 13/34] Fix issues with field names with special characters --- ballerina/tests/masked_string_test.bal | 72 +++++++++++++++++++ .../stdlib/log/MaskedStringBuilder.java | 5 +- 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/ballerina/tests/masked_string_test.bal b/ballerina/tests/masked_string_test.bal index 0367feea..9ee61402 100644 --- a/ballerina/tests/masked_string_test.bal +++ b/ballerina/tests/masked_string_test.bal @@ -29,6 +29,11 @@ isolated function maskStringPartially(string input) returns string { return maskedString; }; +function checkJsonParsing(string maskedStr) { + map|error parsedJson = maskedStr.fromJsonStringWithType(); + test:assertTrue(parsedJson is map); +} + type User record {| string name; @SensitiveData @@ -90,6 +95,7 @@ function testMaskedStringWithAnydataValues() { string maskedAnydataRecStr = toMaskedString(anydataRec); string expectedStr = string `{"str":"Test String","num":123.45,"bool":true,"jsonMap":{"key1":"value1","key2":2},"tableData":[{"col1":"row1col1","col2":"row1col2"},{"col1":"row2col1","col2":"row2col2"}],"arr":["elem1",2,{"key":"value"}],"xmlRaw":"UserAdminReminderDon't forget the meeting!","xmlText":"Just some text","tuple":[1,2.5,"three"]}`; test:assertEquals(maskedAnydataRecStr, expectedStr); + checkJsonParsing(maskedAnydataRecStr); } type OpenAnydataRecord record { @@ -119,6 +125,7 @@ function testMaskedStringWithOpenAnydataRecord() { string maskedOpenRecStr = toMaskedString(openRec); string expectedStr = string `{"name":"Open Record","extraField":"extraValue","extraMapField":{"mapKey":"mapValue"},"extraArrayField":[1,"two",3.0],"extraRecordField":{"name":"Field Record","extraField":"extraValue"}}`; test:assertEquals(maskedOpenRecStr, expectedStr); + checkJsonParsing(maskedOpenRecStr); } type Record1 record {| @@ -196,6 +203,7 @@ function testMaskedStringWithNestedRecords() { string maskedNestedRecStr = toMaskedString(nestedRec); string expectedStr = string `{"name":"Nested Record","details2":{"field1":"Field1 Value","field3":{"subField1":"SubField1 Value","subField2":"###","subField3":"Sub*********lue"}},"records":[{"info":"Record1 Info"},{"info":"Record2 Info"}]}`; test:assertEquals(maskedNestedRecStr, expectedStr); + checkJsonParsing(maskedNestedRecStr); } type NilableSensitiveFieldRecord record {| @@ -217,6 +225,7 @@ function testMaskedStringWithNilableSensitiveField() { string maskedRecWithNilStr = toMaskedString(recWithNil); string expectedStr = string `{"name":"Nilable Record","id":null}`; test:assertEquals(maskedRecWithNilStr, expectedStr); + checkJsonParsing(maskedRecWithNilStr); } type OptionalSensitiveFieldRecord record {| @@ -237,15 +246,18 @@ function testMaskedStringWithOptionalSensitiveField() { string maskedRecWithOptionalStr = toMaskedString(recWithOptional); string expectedStr = string `{"name":"Optional Record","id":101}`; test:assertEquals(maskedRecWithOptionalStr, expectedStr); + checkJsonParsing(maskedRecWithOptionalStr); recWithOptional.sensitiveField = "Sensitive Data"; string maskedRecWithOptionalSetStr = toMaskedString(recWithOptional); test:assertEquals(maskedRecWithOptionalSetStr, expectedStr); + checkJsonParsing(maskedRecWithOptionalSetStr); recWithOptional.id = (); string maskedRecWithOptionalSetNilStr = toMaskedString(recWithOptional); string expectedStrWithoutId = string `{"name":"Optional Record"}`; test:assertEquals(maskedRecWithOptionalSetNilStr, expectedStrWithoutId); + checkJsonParsing(maskedRecWithOptionalSetNilStr); } type NeverSensitiveFieldRecord record {| @@ -265,6 +277,7 @@ function testMaskedStringWithNeverSensitiveField() { string maskedRecStr = toMaskedString(rec); string expectedStr = string `{"name":"Never Record"}`; test:assertEquals(maskedRecStr, expectedStr); + checkJsonParsing(maskedRecStr); } type RecordWithRestField record {| @@ -287,6 +300,7 @@ function testMaskedStringWithRestField() { string maskedRecStr = toMaskedString(rec); string expectedStr = string `{"name":"Rest Field Record","extraField1":"extraValue1","extraField2":"extraValue2"}`; test:assertEquals(maskedRecStr, expectedStr); + checkJsonParsing(maskedRecStr); } type CyclicRecord record {| @@ -326,6 +340,7 @@ function testMaskedStringWithCyclicSensitiveField() { string maskedRecStr = toMaskedString(rec); string expectedStr = string `{"name":"name"}`; test:assertEquals(maskedRecStr, expectedStr); + checkJsonParsing(maskedRecStr); } @test:Config { @@ -342,6 +357,7 @@ function testMaskedStringWithMap() { string maskedMapStr = toMaskedString(jsonMap); string expectedStr = string `{"key1":"value1","key2":2,"key3":true,"key4":{"nestedKey":"nestedValue"},"key5":[1,"two",3.0]}`; test:assertEquals(maskedMapStr, expectedStr); + checkJsonParsing(maskedMapStr); User user = { name: "John Doe", @@ -357,4 +373,60 @@ function testMaskedStringWithMap() { string maskedMapWithSensitiveDataStr = toMaskedString(mapWithSensitiveData); string expectedMapWithSensitiveDataStr = string `{"normalKey":"normalValue","sensitiveKey":{"name":"John Doe","password":"*****","mail":"joh**************com"}}`; test:assertEquals(maskedMapWithSensitiveDataStr, expectedMapWithSensitiveDataStr); + checkJsonParsing(maskedMapWithSensitiveDataStr); +} + +type SpecialCharFieldsRec record {| + string field_with_underscores; + string FieldWithCamelCase; + string field\-With\$pecialChar\!; + string 'type; + string 'value\\\-Field; +|}; + +type SpecialCharSensitiveFieldsRec record {| + @SensitiveData {strategy: {replacement: "*****"}} + string field_with_underscores; + @SensitiveData {strategy: {replacement: "#####"}} + string FieldWithCamelCase; + @SensitiveData {strategy: {replacement: "1!1!1!"}} + string field\-With\$pecialChar\!; + @SensitiveData {strategy: {replacement: "[REDACTED]"}} + string 'type; + @SensitiveData {strategy: {replacement: "~~~~~~"}} + string 'value\\\-Field; +|}; + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithSpecialCharFieldsAndSpecialCharValues() { + SpecialCharFieldsRec rec = { + field_with_underscores: "\"value1\",\"value2\"", + FieldWithCamelCase: "value2", + field\-With\$pecialChar\!: "value3 & 'value4' ", + 'type: "exampleType\n\t", + 'value\\\-Field: "value" + }; + string maskedRecStr = toMaskedString(rec); + string expectedStr = string `{"field_with_underscores":"\"value1\",\"value2\"","FieldWithCamelCase":"value2","field-With$pecialChar!":"value3 & 'value4' ","type":"exampleType\n\t","value\\-Field":"value"}`; + test:assertEquals(maskedRecStr, expectedStr); + checkJsonParsing(maskedRecStr); +} + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithSpecialCharFields() { + SpecialCharSensitiveFieldsRec rec = { + field_with_underscores: "\"value1\",\"value2\"", + FieldWithCamelCase: "value2", + field\-With\$pecialChar\!: "value3 & 'value4' ", + 'type: "exampleType\n\t", + 'value\\\-Field: "value" + }; + string maskedRecStr = toMaskedString(rec); + string expectedStr = string `{"field_with_underscores":"*****","FieldWithCamelCase":"#####","field-With$pecialChar!":"1!1!1!","type":"[REDACTED]","value\\-Field":"~~~~~~"}`; + test:assertEquals(maskedRecStr, expectedStr); + checkJsonParsing(maskedRecStr); } diff --git a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java index c9b852cd..fcbe9be9 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java +++ b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java @@ -24,6 +24,7 @@ import io.ballerina.runtime.api.types.RecordType; import io.ballerina.runtime.api.types.Type; import io.ballerina.runtime.api.types.TypeTags; +import io.ballerina.runtime.api.utils.IdentifierUtils; import io.ballerina.runtime.api.utils.StringUtils; import io.ballerina.runtime.api.utils.TypeUtils; import io.ballerina.runtime.api.values.BArray; @@ -472,7 +473,9 @@ public static MaskedStringBuilder create(Runtime runtime, int initialCapacity) { static Optional> getLogSensitiveDataAnnotation(Map> fieldAnnotations, String fieldName) { - BMap fieldAnnotationMap = fieldAnnotations.get(fieldName); + // In the value map keys are unescaped, but the annotation keys are escaped + // Moreover runtime does not provide a way to unescape the annotation keys, so we need to escape the field name + BMap fieldAnnotationMap = fieldAnnotations.get(IdentifierUtils.escapeSpecialCharacters(fieldName)); if (fieldAnnotationMap == null) { return Optional.empty(); } From 466d4b8fc7a60fce1615ff9fdfdfb89fdd20ea42 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Thu, 2 Oct 2025 11:41:44 +0530 Subject: [PATCH 14/34] Add support to enable sensitive data masking via configuration --- ballerina/natives.bal | 6 +++--- ballerina/root_logger.bal | 15 +++++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/ballerina/natives.bal b/ballerina/natives.bal index 7aa475d7..7cac6c59 100644 --- a/ballerina/natives.bal +++ b/ballerina/natives.bal @@ -349,7 +349,7 @@ isolated function fileWrite(string logOutput) { } } -isolated function printLogFmt(LogRecord logRecord) returns string { +isolated function printLogFmt(LogRecord logRecord, boolean enableSensitiveDataMasking = false) returns string { string message = ""; foreach [string, anydata] [k, v] in logRecord.entries() { string value; @@ -367,8 +367,8 @@ isolated function printLogFmt(LogRecord logRecord) returns string { value = v.toBalString(); } _ => { - value = v is string ? string `${escape(v.toString())}` : - (enableSensitiveDataMasking ? toMaskedString(v) : v.toString()); + string strValue = enableSensitiveDataMasking ? toMaskedString(v) : v.toString(); + value = v is string ? string `${escape(strValue)}` : strValue; } } if message == "" { diff --git a/ballerina/root_logger.bal b/ballerina/root_logger.bal index 3756c356..54e377e0 100644 --- a/ballerina/root_logger.bal +++ b/ballerina/root_logger.bal @@ -27,6 +27,8 @@ public type Config record {| readonly & OutputDestination[] destinations = destinations; # Additional key-value pairs to include in the log messages. Default is the key-values configured in the module level readonly & AnydataKeyValues keyValues = {...keyValues}; + # Enable sensitive data masking. Default is the module level configuration + boolean enableSensitiveDataMasking = enableSensitiveDataMasking; |}; type ConfigInternal record {| @@ -34,6 +36,7 @@ type ConfigInternal record {| Level level = level; readonly & OutputDestination[] destinations = destinations; readonly & KeyValues keyValues = {...keyValues}; + boolean enableSensitiveDataMasking = enableSensitiveDataMasking; |}; final RootLogger rootLogger; @@ -59,7 +62,8 @@ public isolated function fromConfig(*Config config) returns Logger|Error { format: config.format, level: config.level, destinations: config.destinations, - keyValues: newKeyValues.cloneReadOnly() + keyValues: newKeyValues.cloneReadOnly(), + enableSensitiveDataMasking: config.enableSensitiveDataMasking }; return new RootLogger(newConfig); } @@ -71,12 +75,14 @@ isolated class RootLogger { private final Level level; private final readonly & OutputDestination[] destinations; private final readonly & KeyValues keyValues; + private final boolean enableSensitiveDataMasking; public isolated function init(Config|ConfigInternal config = {}) { self.format = config.format; self.level = config.level; self.destinations = config.destinations; self.keyValues = config.keyValues; + self.enableSensitiveDataMasking = config.enableSensitiveDataMasking; } public isolated function printDebug(string|PrintableRawTemplate msg, error? 'error, error:StackFrame[]? stackTrace, *KeyValues keyValues) { @@ -108,7 +114,8 @@ isolated class RootLogger { format: self.format, level: self.level, destinations: self.destinations, - keyValues: newKeyValues.cloneReadOnly() + keyValues: newKeyValues.cloneReadOnly(), + enableSensitiveDataMasking: self.enableSensitiveDataMasking }; return new RootLogger(config); } @@ -144,8 +151,8 @@ isolated class RootLogger { } string logOutput = self.format == JSON_FORMAT ? - (enableSensitiveDataMasking ? toMaskedString(logRecord) : logRecord.toJsonString()) : - printLogFmt(logRecord); + (self.enableSensitiveDataMasking ? toMaskedString(logRecord) : logRecord.toJsonString()) : + printLogFmt(logRecord, self.enableSensitiveDataMasking); lock { if outputFilePath is string { From 868fca57c11946f8f6431191fc1536f6a3af1430 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Thu, 2 Oct 2025 11:42:32 +0530 Subject: [PATCH 15/34] Add tests for masked logging --- ballerina/tests/log_masking.bal | 76 +++++++++++++++++++++++++++++++++ ballerina/tests/logger_test.bal | 3 ++ 2 files changed, 79 insertions(+) create mode 100644 ballerina/tests/log_masking.bal diff --git a/ballerina/tests/log_masking.bal b/ballerina/tests/log_masking.bal new file mode 100644 index 00000000..2744342e --- /dev/null +++ b/ballerina/tests/log_masking.bal @@ -0,0 +1,76 @@ +// Copyright (c) 2025 WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/io; +import ballerina/test; + +final Logger maskerJsonLogger = check fromConfig(enableSensitiveDataMasking = true); +final Logger maskerLogger = check fromConfig(enableSensitiveDataMasking = true, format = LOGFMT); + +string[] maskedLogs = []; + +function addMaskedLogs(io:FileOutputStream fileOutputStream, io:Printable... values) { + var firstValue = values[0]; + if firstValue is string { + maskedLogs.push(firstValue); + } +} + +@test:Config { + groups: ["logMasking"] +} +function testLogMasking() returns error? { + test:when(mock_fprintln).call("addMaskedLogs"); + User user = { + name: "John Doe", + ssn: "123-45-6789", + password: "password123", + mail: "john.doe@example.com", + creditCard: "4111-1111-1111-1111" + }; + maskerJsonLogger.printInfo("user logged in", user = user); + string expectedLog = string `"message":"user logged in","user":{"name":"John Doe","password":"*****","mail":"joh**************com"},"env":"test"`; + test:assertEquals(maskedLogs.length(), 1); + test:assertTrue(maskedLogs[0].includes(expectedLog)); + maskedLogs.removeAll(); + + maskerLogger.printInfo("user logged in", user = user); + expectedLog= string `message="user logged in" user={"name":"John Doe","password":"*****","mail":"joh**************com"} env="test"`; + test:assertEquals(maskedLogs.length(), 1); + test:assertTrue(maskedLogs[0].includes(expectedLog)); + maskedLogs.removeAll(); + + map userTmp = user; + maskerJsonLogger.printInfo("user logged in", user = userTmp); + expectedLog = string `"message":"user logged in","user":{"name":"John Doe","password":"*****","mail":"joh**************com"},"env":"test"`; + test:assertEquals(maskedLogs.length(), 1); + test:assertTrue(maskedLogs[0].includes(expectedLog)); + maskedLogs.removeAll(); + + userTmp = check user.cloneWithType(); + maskerJsonLogger.printInfo("user logged in", user = userTmp); + expectedLog = string `"message":"user logged in","user":{"name":"John Doe","ssn":"123-45-6789","password":"password123","mail":"john.doe@example.com","creditCard":"4111-1111-1111-1111"},"env":"test"`; + test:assertEquals(maskedLogs.length(), 1); + test:assertTrue(maskedLogs[0].includes(expectedLog)); + maskedLogs.removeAll(); + + user = check userTmp.cloneWithType(); + maskerLogger.printInfo("user logged in", user = user); + expectedLog = string `message="user logged in" user={"name":"John Doe","password":"*****","mail":"joh**************com"} env="test"`; + test:assertEquals(maskedLogs.length(), 1); + test:assertTrue(maskedLogs[0].includes(expectedLog)); + maskedLogs.removeAll(); +} diff --git a/ballerina/tests/logger_test.bal b/ballerina/tests/logger_test.bal index b3bef0ce..8599910d 100644 --- a/ballerina/tests/logger_test.bal +++ b/ballerina/tests/logger_test.bal @@ -22,6 +22,9 @@ configurable Config loggerConfig2 = {}; type Context record {| string id; + // By default root logger is configured not to mask sensitive data + // So this is added as a negative test case + @SensitiveData string msg; |}; From e04e2a2ad0d24cd237ab6bf01c71bcc2d71c6fd8 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Thu, 2 Oct 2025 13:20:46 +0530 Subject: [PATCH 16/34] Enhance type processing to handle intersection and reference types --- .../stdlib/log/MaskedStringBuilder.java | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java index fcbe9be9..70fbcc52 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java +++ b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java @@ -21,7 +21,9 @@ import io.ballerina.runtime.api.Runtime; import io.ballerina.runtime.api.creators.ErrorCreator; import io.ballerina.runtime.api.types.Field; +import io.ballerina.runtime.api.types.IntersectionType; import io.ballerina.runtime.api.types.RecordType; +import io.ballerina.runtime.api.types.ReferenceType; import io.ballerina.runtime.api.types.Type; import io.ballerina.runtime.api.types.TypeTags; import io.ballerina.runtime.api.utils.IdentifierUtils; @@ -36,6 +38,7 @@ import java.util.Collection; import java.util.IdentityHashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -153,7 +156,8 @@ private String buildInternal(Object value) { } private String processValue(Object value) { - Type type = TypeUtils.getType(value); + // Getting implied type to handle intersection types with readonly + Type type = getEffectiveType(TypeUtils.getType(value)); return switch (value) { // Processing only the structured types, since the basic types does not contain the @@ -165,6 +169,33 @@ private String processValue(Object value) { }; } + private Type getEffectiveType(Type type) { + // For intersection types, get the first constituent type that is not readonly + if (type.getTag() == TypeTags.INTERSECTION_TAG) { + List constituentTypes = ((IntersectionType) type).getConstituentTypes(); + if (constituentTypes.size() == 2) { + type = constituentTypes.get(0).getTag() == TypeTags.READONLY_TAG ? constituentTypes.get(1) : + constituentTypes.get(0); + return getEffectiveType(type); + } + } + + // Record types can be intersection types, so unwrap them to get the actual record type + if (type.getTag() == TypeTags.RECORD_TYPE_TAG) { + Optional intersectionType = ((RecordType) type).getIntersectionType(); + if (intersectionType.isPresent()) { + return getEffectiveType(intersectionType.get()); + } + } + + // Unwrap reference types to get the actual referred type + if (type.getTag() == TypeTags.TYPE_REFERENCED_TYPE_TAG) { + type = ((ReferenceType) type).getReferredType(); + return getEffectiveType(type); + } + return type; + } + private String processMapValue(BMap mapValue, Type valueType) { Map fields = Map.of(); Map> fieldAnnotations = Map.of(); From 1ad022a3ee47c37e465bb6cef8c605516842b83d Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Thu, 2 Oct 2025 13:21:18 +0530 Subject: [PATCH 17/34] Add support for sensitive data masking in templates and value functions --- ballerina/natives.bal | 8 ++++---- ballerina/root_logger.bal | 8 +++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ballerina/natives.bal b/ballerina/natives.bal index 7cac6c59..566367f3 100644 --- a/ballerina/natives.bal +++ b/ballerina/natives.bal @@ -166,7 +166,7 @@ public enum FileWriteOption { # # + template - The raw template to be processed # + return - The processed string -public isolated function processTemplate(PrintableRawTemplate template) returns string { +public isolated function processTemplate(PrintableRawTemplate template, boolean enableSensitiveDataMasking = false) returns string { string[] templateStrings = template.strings; Value[] insertions = template.insertions; string result = templateStrings[0]; @@ -174,7 +174,7 @@ public isolated function processTemplate(PrintableRawTemplate template) returns foreach int i in 1 ..< templateStrings.length() { Value insertion = insertions[i - 1]; string insertionStr = insertion is PrintableRawTemplate ? - processTemplate(insertion) : + processTemplate(insertion, enableSensitiveDataMasking) : insertion is Valuer ? (enableSensitiveDataMasking ? toMaskedString(insertion()) : insertion().toString()) : (enableSensitiveDataMasking ? toMaskedString(insertion) : insertion.toString()); @@ -183,8 +183,8 @@ public isolated function processTemplate(PrintableRawTemplate template) returns return result; } -isolated function processMessage(string|PrintableRawTemplate msg) returns string => - msg !is string ? processTemplate(msg) : msg; +isolated function processMessage(string|PrintableRawTemplate msg, boolean enableSensitiveDataMasking) returns string => + msg !is string ? processTemplate(msg, enableSensitiveDataMasking) : msg; # Prints debug logs. # ```ballerina diff --git a/ballerina/root_logger.bal b/ballerina/root_logger.bal index 54e377e0..9fced72e 100644 --- a/ballerina/root_logger.bal +++ b/ballerina/root_logger.bal @@ -128,7 +128,7 @@ isolated class RootLogger { time: getCurrentTime(), level: logLevel, module: moduleName, - message: processMessage(msg) + message: processMessage(msg, self.enableSensitiveDataMasking) }; if err is error { logRecord.'error = getFullErrorDetails(err); @@ -138,7 +138,8 @@ isolated class RootLogger { select element.toString(); } foreach [string, Value] [k, v] in keyValues.entries() { - logRecord[k] = v is Valuer ? v() : v is PrintableRawTemplate ? processMessage(v) : v; + logRecord[k] = v is Valuer ? v() : + (v is PrintableRawTemplate ? processMessage(v, self.enableSensitiveDataMasking) : v); } if observe:isTracingEnabled() { map spanContext = observe:getSpanContext(); @@ -147,7 +148,8 @@ isolated class RootLogger { } } foreach [string, Value] [k, v] in self.keyValues.entries() { - logRecord[k] = v is Valuer ? v() : v is PrintableRawTemplate ? processMessage(v) : v; + logRecord[k] = v is Valuer ? v() : + (v is PrintableRawTemplate ? processMessage(v, self.enableSensitiveDataMasking) : v); } string logOutput = self.format == JSON_FORMAT ? From b6cb5275bc9e216283da867408f240929975f907 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Thu, 2 Oct 2025 13:21:46 +0530 Subject: [PATCH 18/34] Add tests for readonly types --- ballerina/tests/log_masking.bal | 30 ++++++++----- ballerina/tests/masked_string_test.bal | 61 ++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/ballerina/tests/log_masking.bal b/ballerina/tests/log_masking.bal index 2744342e..bd03912d 100644 --- a/ballerina/tests/log_masking.bal +++ b/ballerina/tests/log_masking.bal @@ -29,18 +29,21 @@ function addMaskedLogs(io:FileOutputStream fileOutputStream, io:Printable... val } } +final readonly & User user = { + name: "John Doe", + ssn: "123-45-6789", + password: "password123", + mail: "john.doe@example.com", + creditCard: "4111-1111-1111-1111" +}; + +isolated function getUser() returns User => user; + @test:Config { groups: ["logMasking"] } function testLogMasking() returns error? { test:when(mock_fprintln).call("addMaskedLogs"); - User user = { - name: "John Doe", - ssn: "123-45-6789", - password: "password123", - mail: "john.doe@example.com", - creditCard: "4111-1111-1111-1111" - }; maskerJsonLogger.printInfo("user logged in", user = user); string expectedLog = string `"message":"user logged in","user":{"name":"John Doe","password":"*****","mail":"joh**************com"},"env":"test"`; test:assertEquals(maskedLogs.length(), 1); @@ -48,7 +51,7 @@ function testLogMasking() returns error? { maskedLogs.removeAll(); maskerLogger.printInfo("user logged in", user = user); - expectedLog= string `message="user logged in" user={"name":"John Doe","password":"*****","mail":"joh**************com"} env="test"`; + expectedLog = string `message="user logged in" user={"name":"John Doe","password":"*****","mail":"joh**************com"} env="test"`; test:assertEquals(maskedLogs.length(), 1); test:assertTrue(maskedLogs[0].includes(expectedLog)); maskedLogs.removeAll(); @@ -67,9 +70,14 @@ function testLogMasking() returns error? { test:assertTrue(maskedLogs[0].includes(expectedLog)); maskedLogs.removeAll(); - user = check userTmp.cloneWithType(); - maskerLogger.printInfo("user logged in", user = user); - expectedLog = string `message="user logged in" user={"name":"John Doe","password":"*****","mail":"joh**************com"} env="test"`; + maskerLogger.printDebug(`user login event. user details: ${user}`); + expectedLog = string `message="user login event. user details: {\"name\":\"John Doe\",\"password\":\"*****\",\"mail\":\"joh**************com\"}" env="test"`; + test:assertEquals(maskedLogs.length(), 1); + test:assertTrue(maskedLogs[0].includes(expectedLog)); + maskedLogs.removeAll(); + + maskerLogger.printWarn("user login attempt failed", user = getUser); + expectedLog = string `message="user login attempt failed" user={"name":"John Doe","password":"*****","mail":"joh**************com"} env="test"`; test:assertEquals(maskedLogs.length(), 1); test:assertTrue(maskedLogs[0].includes(expectedLog)); maskedLogs.removeAll(); diff --git a/ballerina/tests/masked_string_test.bal b/ballerina/tests/masked_string_test.bal index 9ee61402..db92ccc5 100644 --- a/ballerina/tests/masked_string_test.bal +++ b/ballerina/tests/masked_string_test.bal @@ -430,3 +430,64 @@ function testMaskedStringWithSpecialCharFields() { test:assertEquals(maskedRecStr, expectedStr); checkJsonParsing(maskedRecStr); } + +type ReadonlyUser1 readonly & record {| + string name; + @SensitiveData + string ssn; + @SensitiveData {strategy: {replacement: "*****"}} + string password; + @SensitiveData {strategy: {replacement: maskStringPartially}} + string mail; + @SensitiveData {strategy: EXCLUDE} + string creditCard; +|}; + +type ReadonlyUser2 readonly & User; + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithReadonlyRecords() returns error? { + User user = { + name: "John Doe", + ssn: "123-45-6789", + password: "password123", + mail: "john.doe@example.com", + creditCard: "4111-1111-1111-1111" + }; + + ReadonlyUser1 readonlyUser1 = {...user}; + ReadonlyUser2 readonlyUser2 = {...user}; + readonly & User readonlyUser3 = {...user}; + ReadonlyUser1 readonlyUser4 = check user.cloneWithType(); + ReadonlyUser2 readonlyUser5 = check user.cloneWithType(); + readonly & User readonlyUser6 = check user.cloneWithType(); + readonly & User readonlyUser7 = user.cloneReadOnly(); + + string maskedReadonlyUser1Str = toMaskedString(readonlyUser1); + string maskedReadonlyUser2Str = toMaskedString(readonlyUser2); + string maskedReadonlyUser3Str = toMaskedString(readonlyUser3); + string maskedReadonlyUser4Str = toMaskedString(readonlyUser4); + string maskedReadonlyUser5Str = toMaskedString(readonlyUser5); + string maskedReadonlyUser6Str = toMaskedString(readonlyUser6); + string maskedReadonlyUser7Str = toMaskedString(readonlyUser7); + + string expectedStr = string `{"name":"John Doe","password":"*****","mail":"joh**************com"}`; + + test:assertEquals(maskedReadonlyUser1Str, expectedStr); + test:assertEquals(maskedReadonlyUser2Str, expectedStr); + test:assertEquals(maskedReadonlyUser3Str, expectedStr); + test:assertEquals(maskedReadonlyUser4Str, expectedStr); + test:assertEquals(maskedReadonlyUser5Str, expectedStr); + test:assertEquals(maskedReadonlyUser6Str, expectedStr); + test:assertEquals(maskedReadonlyUser7Str, expectedStr); + + checkJsonParsing(maskedReadonlyUser1Str); + checkJsonParsing(maskedReadonlyUser2Str); + checkJsonParsing(maskedReadonlyUser3Str); + checkJsonParsing(maskedReadonlyUser4Str); + checkJsonParsing(maskedReadonlyUser5Str); + checkJsonParsing(maskedReadonlyUser6Str); + checkJsonParsing(maskedReadonlyUser7Str); +} From de5cdb1d374d1e085d85cb8003e8e9bbaf436d56 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Thu, 2 Oct 2025 13:45:36 +0530 Subject: [PATCH 19/34] Add an integration test --- integration-tests/build.gradle | 3 + .../samples/masked-logger/Ballerina.toml | 4 ++ .../samples/masked-logger/Config.toml | 3 + .../resources/samples/masked-logger/main.bal | 58 +++++++++++++++++++ .../tests/test_logger_masking.bal | 39 +++++++++++++ 5 files changed, 107 insertions(+) create mode 100644 integration-tests/tests/resources/samples/masked-logger/Ballerina.toml create mode 100644 integration-tests/tests/resources/samples/masked-logger/Config.toml create mode 100644 integration-tests/tests/resources/samples/masked-logger/main.bal create mode 100644 integration-tests/tests/test_logger_masking.bal diff --git a/integration-tests/build.gradle b/integration-tests/build.gradle index b918811e..92aa4d91 100644 --- a/integration-tests/build.gradle +++ b/integration-tests/build.gradle @@ -117,6 +117,9 @@ task copyTestResources(type: Copy) { into("logger-from-config") { from "tests/resources/samples/logger/logger-from-config" } + into("masked-logger") { + from "tests/resources/samples/masked-logger" + } } task copyTestOutputResources(type: Copy) { diff --git a/integration-tests/tests/resources/samples/masked-logger/Ballerina.toml b/integration-tests/tests/resources/samples/masked-logger/Ballerina.toml new file mode 100644 index 00000000..dc338c58 --- /dev/null +++ b/integration-tests/tests/resources/samples/masked-logger/Ballerina.toml @@ -0,0 +1,4 @@ +[package] +org = "wso2" +name = "masked_logger" +version = "1.0.0" diff --git a/integration-tests/tests/resources/samples/masked-logger/Config.toml b/integration-tests/tests/resources/samples/masked-logger/Config.toml new file mode 100644 index 00000000..1e78c430 --- /dev/null +++ b/integration-tests/tests/resources/samples/masked-logger/Config.toml @@ -0,0 +1,3 @@ +[ballerina.log] +enableSensitiveDataMasking = true +level = "DEBUG" \ No newline at end of file diff --git a/integration-tests/tests/resources/samples/masked-logger/main.bal b/integration-tests/tests/resources/samples/masked-logger/main.bal new file mode 100644 index 00000000..b93250e9 --- /dev/null +++ b/integration-tests/tests/resources/samples/masked-logger/main.bal @@ -0,0 +1,58 @@ +// Copyright (c) 2025 WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/log; + +type User record {| + string name; + @log:SensitiveData + string ssn; + @log:SensitiveData {strategy: {replacement: "*****"}} + string password; + @log:SensitiveData {strategy: {replacement: maskStringPartially}} + string mail; + @log:SensitiveData {strategy: log:EXCLUDE} + string creditCard; +|}; + +isolated function maskStringPartially(string input) returns string { + int len = input.length(); + if len <= 6 { + return "******"; + } + string maskedString = input.substring(0, 3); + foreach int i in 3 ... len - 4 { + maskedString += "*"; + } + maskedString += input.substring(len - 3); + return maskedString; +}; + +final readonly & User user = { + name: "John Doe", + ssn: "123-45-6789", + password: "P@ssw0rd!", + mail: "john.doe@example.com", + creditCard: "4111-1111-1111-1111" +}; + +isolated function getUser() returns User => user; + +public function main() { + log:printInfo("user logged in", userDetails = user); + log:printDebug(`user details: ${user}`); + log:printError("error occurred", userDetails = getUser); +} diff --git a/integration-tests/tests/test_logger_masking.bal b/integration-tests/tests/test_logger_masking.bal new file mode 100644 index 00000000..3b459d2d --- /dev/null +++ b/integration-tests/tests/test_logger_masking.bal @@ -0,0 +1,39 @@ +// Copyright (c) 2025 WSO2 LLC. (https://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/test; +import ballerina/io; + +const MASKED_LOGGER_CONFIG_FILE = "tests/resources/samples/masked-logger/Config.toml"; + +@test:Config { + groups: ["maskedLogger"] +} +function testMaskedLogger() returns error? { + Process|error execResult = exec(bal_exec_path, {BAL_CONFIG_FILES: MASKED_LOGGER_CONFIG_FILE}, (), "run", string `${temp_dir_path}/masked-logger`); + Process result = check execResult; + int _ = check result.waitForExit(); + int _ = check result.exitCode(); + io:ReadableByteChannel readableResult = result.stderr(); + io:ReadableCharacterChannel sc = new (readableResult, UTF_8); + string outText = check sc.read(100000); + string[] logLines = re `\n`.split(outText.trim()); + test:assertEquals(logLines.length(), 8, INCORRECT_NUMBER_OF_LINES); + test:assertTrue(logLines[5].includes(string `level=INFO module=wso2/masked_logger message="user logged in" userDetails={"name":"John Doe","password":"*****","mail":"joh**************com"}`)); + test:assertTrue(logLines[6].includes(string `level=DEBUG module=wso2/masked_logger message="user details: {\"name\":\"John Doe\",\"password\":\"*****\",\"mail\":\"joh**************com\"}"`)); + test:assertTrue(logLines[7].includes(string `level=ERROR module=wso2/masked_logger message="error occurred" userDetails={"name":"John Doe","password":"*****","mail":"joh**************com"}`)); + check sc.close(); +} \ No newline at end of file From 60a4cf4da52901cb92616c8a92db64bf25fece5d Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Thu, 2 Oct 2025 14:59:38 +0530 Subject: [PATCH 20/34] Update changelog --- changelog.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/changelog.md b/changelog.md index 5c6a58fd..6c61536d 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,12 @@ # Change Log This file contains all the notable changes done to the Ballerina TCP package through the releases. +## [Unreleased] + +### Added + +- [Introduce sensitive data masking support](https://github.com/ballerina-platform/ballerina-library/issues/8211) + ## [2.13.0]- 2025-08-28 ### Added From 11fdebda94eb8cfdbf593df3c2b1cd2947d47a77 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Thu, 2 Oct 2025 15:51:59 +0530 Subject: [PATCH 21/34] Update spec --- docs/spec/spec.md | 159 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 158 insertions(+), 1 deletion(-) diff --git a/docs/spec/spec.md b/docs/spec/spec.md index ab629c75..881aceca 100644 --- a/docs/spec/spec.md +++ b/docs/spec/spec.md @@ -3,7 +3,7 @@ _Authors_: @daneshk @MadhukaHarith92 @TharmiganK _Reviewers_: @daneshk @ThisaruGuruge _Created_: 2021/11/15 -_Updated_: 2025/08/20 +_Updated_: 2025/10/02 _Edition_: Swan Lake ## Introduction @@ -31,6 +31,10 @@ The conforming implementation of the specification is released and included in t * 4.3. [Child logger](#43-child-logger) * 4.3.1. [Loggers with additional context](#431-loggers-with-additional-context) * 4.3.2. [Loggers with unique logging configuration](#432-loggers-with-unique-logging-configuration) +5. [Sensitive data masking](#5-sensitive-data-masking) + * 5.1. [Sensitive data annotation](#51-sensitive-data-annotation) + * 5.2. [Masked string function](#52-masked-string-function) + * 5.3. [Configure sensitive data masking](#53-configure-sensitive-data-masking) ## 1. Overview @@ -279,6 +283,8 @@ public type Config record {| readonly & OutputDestination[] destinations = destinations; # Additional key-value pairs to include in the log messages. Default is the key-values configured in the module level readonly & AnydataKeyValues keyValues = {...keyValues}; + # Enable sensitive data masking in the logs. Default is false + boolean enableSensitiveDataMasking = false; |}; ``` @@ -294,3 +300,154 @@ log:Config auditLogConfig = { log:Logger auditLogger = log:fromConfig(auditLogConfig); auditLogger.printInfo("Hello World from the audit logger!"); ``` + +## 5. Sensitive data masking + +The Ballerina log module provides the capability to mask sensitive data in log messages. This is crucial for maintaining data privacy and security, especially when dealing with personally identifiable information (PII) or other sensitive data. + +### 5.1. Sensitive data annotation + +The `@log:SensitiveData` annotation can be used to mark fields in a record as sensitive. When such fields are logged, their values will be excluded or masked to prevent exposure of sensitive information. + +```ballerina +import ballerina/log; + +type User record { + string id; + @log:SensitiveData + string password; + string name; +}; + +public function main() { + User user = {id: "U001", password: "mypassword", name: "John Doe"}; + log:printInfo("user details", user = user); +} +``` + +Output: + +```log +time=2025-08-20T09:15:30.123+05:30 level=INFO module="" message="user details" user={"id":"U001","name":"John Doe"} +``` + +By default, the `@log:SensitiveData` annotation will exclude the sensitive field from the log output when sensitive data masking is enabled. + +Additionally, the masking strategy can be configured using the `strategy` field of the annotation. The available strategies are: +1. `EXCLUDE`: Excludes the field from the log output (default behavior). +2. `Replacement`: Replaces the field value with a specified replacement string or a function that generates a masked version of the value. + +Example: + +```ballerina +import ballerina/log; + +isolated function maskString(string input) returns string { + if input.length() <= 2 { + return "****"; + } + return input.substring(0, 1) + "****" + input.substring(input.length() - 1); +} + +type User record { + string id; + @log:SensitiveData { + strategy: { + replacement: "****" + } + } + string password; + @log:SensitiveData { + strategy: { + replacement: maskString + } + } + string ssn; + string name; +}; + +public function main() { + User user = {id: "U001", password: "mypassword", ssn: "123-45-6789", name: "John Doe"}; + log:printInfo("user details", user = user); +} +``` + +Output: + +```log +time=2025-08-20T09:20:45.456+05:30 level=INFO module="" message="user details" user={"id":"U001","password":"****","ssn":"1****9","name":"John Doe"} +``` + +### 5.2. Masked string function + +The `log:toMaskedString()` function can be used to obtain the masked version of a value. This is useful when developers want to implement custom loggers and need to mask sensitive data. + +```ballerina +import ballerina/log; +import ballerina/io; + +type User record { + string id; + @log:SensitiveData + string password; + string name; +}; + +public function main() { + User user = {id: "U001", password: "mypassword", name: "John Doe"}; + string maskedUser = log:toMaskedString(user); + io:println(maskedUser); +} +``` + +Output: + +```log +{"id":"U001","name":"John Doe"} +``` + +> **Note:** The masking is based on the type of the value. Since, Ballerina is a structurally typed language, same value can be assigned to different typed variables. So the masking is based on the actual value type which is determined at the value creation time. +> Example: +> ```ballerina +> type User record { +> string id; +> @log:SensitiveData +> string password; +> string name; +> }; +> +> type Student record { +> string id; +> string password; // Not marked as sensitive +> string name; +> }; +> +> public function main() returns error? { +> User user = {id: "U001", password: "mypassword", name: "John Doe"}; +> // password will be masked +> string maskedUser = log:toMaskedString(user); +> +> Student student = user; // Allowed since both have the same structure +> // password will be masked since the type at value creation is User +> string maskedStudent = log:toMaskedString(student); +> +> student = {id: "S001", password: "studentpass", name: "Jane Doe"}; +> user = student; // Allowed since both have the same structure +> // password will not be masked since the type at value creation is Student +> maskedStudent = log:toMaskedString(user); +> +> // Explicity creating a value with type +> user = check student.cloneWithType(); +> // password will be masked since the type at value creation is User +> maskedUser = log:toMaskedString(user); +> } +> ``` + +### 5.3. Configure sensitive data masking + +By default, sensitive data masking is disabled. It can be enabled via a configurable variable in the `Config.toml` file. + +```toml +[ballerina.log] +enableSensitiveDataMasking = true +``` From 7e27003157a202bb83669238792d9ecb53a85206 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Fri, 3 Oct 2025 08:40:08 +0530 Subject: [PATCH 22/34] Optimize Unicode escaping by using a pre-computed hex lookup table --- .../java/io/ballerina/stdlib/log/MaskedStringBuilder.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java index 70fbcc52..6b59cd4b 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java +++ b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java @@ -68,6 +68,9 @@ public class MaskedStringBuilder implements AutoCloseable { private static final Map>> ANNOTATION_CACHE = new ConcurrentHashMap<>(); private static final int MAX_CACHE_SIZE = 1000; + // Pre-computed hex lookup table for efficient Unicode escaping + private static final char[] HEX_CHARS = "0123456789abcdef".toCharArray(); + private static final char[] QUOTE_ESCAPE = {'\\', '"'}; private static final char[] BACKSLASH_ESCAPE = {'\\', '\\'}; private static final char[] NEWLINE_ESCAPE = {'\\', 'n'}; @@ -309,8 +312,9 @@ private void appendEscapedStringOptimized(String input) { case '\t' -> stringBuilder.append(TAB_ESCAPE); default -> { if (c < 0x20 || c == 0x7F) { - stringBuilder.append("\\u"); - stringBuilder.append(String.format("%04x", (int) c)); + stringBuilder.append("\\u00"); + stringBuilder.append(HEX_CHARS[(c >>> 4) & 0xF]); + stringBuilder.append(HEX_CHARS[c & 0xF]); } else { stringBuilder.append(c); } From 1dca78248131f5ea9dd73d77f88b4455431c4bae Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Fri, 3 Oct 2025 08:58:05 +0530 Subject: [PATCH 23/34] Add tests for masking structurally similar records and basic types --- ballerina/tests/log_masking.bal | 6 +++ ballerina/tests/masked_string_test.bal | 70 ++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/ballerina/tests/log_masking.bal b/ballerina/tests/log_masking.bal index bd03912d..e58b5fe3 100644 --- a/ballerina/tests/log_masking.bal +++ b/ballerina/tests/log_masking.bal @@ -81,4 +81,10 @@ function testLogMasking() returns error? { test:assertEquals(maskedLogs.length(), 1); test:assertTrue(maskedLogs[0].includes(expectedLog)); maskedLogs.removeAll(); + + maskerLogger.printInfo("basic types", str = "my string", num = 12345, flag = true, val = (), xmlVal = xml `bar`); + expectedLog = string `message="basic types" str="my string" num=12345 flag=true val=null xmlVal=bar env="test"`; + test:assertEquals(maskedLogs.length(), 1); + test:assertTrue(maskedLogs[0].includes(expectedLog)); + maskedLogs.removeAll(); } diff --git a/ballerina/tests/masked_string_test.bal b/ballerina/tests/masked_string_test.bal index db92ccc5..57bfbd09 100644 --- a/ballerina/tests/masked_string_test.bal +++ b/ballerina/tests/masked_string_test.bal @@ -491,3 +491,73 @@ function testMaskedStringWithReadonlyRecords() returns error? { checkJsonParsing(maskedReadonlyUser6Str); checkJsonParsing(maskedReadonlyUser7Str); } + +type StructurallySimilarUser record {| + string name; + string ssn; + string password; + string mail; + string creditCard; +|}; + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithStructurallySimilarRecord() returns error? { + User user = { + name: "John Doe", + ssn: "123-45-6789", + password: "password123", + mail: "john.doe@example.com", + creditCard: "4111-1111-1111-1111" + }; + + StructurallySimilarUser similarUser = user; + string maskedSimilarUserStr = toMaskedString(similarUser); + string expectedStr = string `{"name":"John Doe","password":"*****","mail":"joh**************com"}`; + test:assertEquals(maskedSimilarUserStr, expectedStr); + checkJsonParsing(maskedSimilarUserStr); + + similarUser = { + name: "John Doe", + ssn: "123-45-6789", + password: "password123", + mail: "john.doe@example.com", + creditCard: "4111-1111-1111-1111" + }; + maskedSimilarUserStr = toMaskedString(similarUser); + expectedStr = string `{"name":"John Doe","ssn":"123-45-6789","password":"password123","mail":"john.doe@example.com","creditCard":"4111-1111-1111-1111"}`; + test:assertEquals(maskedSimilarUserStr, expectedStr); + checkJsonParsing(maskedSimilarUserStr); + + user = similarUser; + string maskedUserStr = toMaskedString(user); + test:assertEquals(maskedUserStr, expectedStr); + checkJsonParsing(maskedUserStr); + + // Explicit type casting will not change the runtime type of the value for structural types + user = similarUser; + maskedUserStr = toMaskedString(user); + test:assertEquals(maskedUserStr, expectedStr); + checkJsonParsing(maskedUserStr); + + // Ensuretype will not change the runtime type of the value for structural types + user = check similarUser.ensureType(); + maskedUserStr = toMaskedString(user); + test:assertEquals(maskedUserStr, expectedStr); + checkJsonParsing(maskedUserStr); +} + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithBasicTypes() { + test:assertEquals(toMaskedString(()), "null"); + test:assertEquals(toMaskedString("Test String"), "Test String"); + test:assertEquals(toMaskedString(123), "123"); + test:assertEquals(toMaskedString(45.67), "45.67"); + test:assertEquals(toMaskedString(45.67d), "45.67"); + test:assertEquals(toMaskedString(true), "true"); + test:assertEquals(toMaskedString(xml `UserAdminReminderDon't forget the meeting!`), "UserAdminReminderDon't forget the meeting!"); + test:assertEquals(toMaskedString(xml `Just some text`), "Just some text"); +} From d19d7814e065c6dc69861a88294668fef77ad358 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Fri, 3 Oct 2025 09:03:15 +0530 Subject: [PATCH 24/34] Add test for masking special characters in strings --- ballerina/tests/masked_string_test.bal | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/ballerina/tests/masked_string_test.bal b/ballerina/tests/masked_string_test.bal index 57bfbd09..266b990c 100644 --- a/ballerina/tests/masked_string_test.bal +++ b/ballerina/tests/masked_string_test.bal @@ -536,7 +536,7 @@ function testMaskedStringWithStructurallySimilarRecord() returns error? { checkJsonParsing(maskedUserStr); // Explicit type casting will not change the runtime type of the value for structural types - user = similarUser; + user = similarUser; maskedUserStr = toMaskedString(user); test:assertEquals(maskedUserStr, expectedStr); checkJsonParsing(maskedUserStr); @@ -561,3 +561,20 @@ function testMaskedStringWithBasicTypes() { test:assertEquals(toMaskedString(xml `UserAdminReminderDon't forget the meeting!`), "UserAdminReminderDon't forget the meeting!"); test:assertEquals(toMaskedString(xml `Just some text`), "Just some text"); } + +@test:Config { + groups: ["maskedString"] +} +function testMaskedStringWithCharactersToBeEscaped() { + record {} specialCharMap = { + "quote": "\"DoubleQuote\"", + "backslash": "Back\\slash", + "newline": "New\nLine", + "tab": "Tab\tCharacter", + "carriageReturn": "Carriage\rReturn" + }; + string maskedMapStr = toMaskedString(specialCharMap); + string expectedStr = string `{"quote":"\"DoubleQuote\"","backslash":"Back\\slash","newline":"New\nLine","tab":"Tab\tCharacter","carriageReturn":"Carriage\rReturn"}`; + test:assertEquals(maskedMapStr, expectedStr); + checkJsonParsing(maskedMapStr); +} From 6516ecd3cee898965bbd77e620974872a926e551 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Fri, 3 Oct 2025 09:26:33 +0530 Subject: [PATCH 25/34] Update spec to clarify masking behavior and type extraction --- docs/spec/spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/spec/spec.md b/docs/spec/spec.md index 881aceca..5009a213 100644 --- a/docs/spec/spec.md +++ b/docs/spec/spec.md @@ -406,7 +406,7 @@ Output: {"id":"U001","name":"John Doe"} ``` -> **Note:** The masking is based on the type of the value. Since, Ballerina is a structurally typed language, same value can be assigned to different typed variables. So the masking is based on the actual value type which is determined at the value creation time. +> **Note:** The masking is based on the type of the value. Since, Ballerina is a structurally typed language, same value can be assigned to different typed variables. So the masking is based on the actual value type which is determined at the value creation time. The original type information can be extracted using the `typeof` operator. > Example: > ```ballerina > type User record { From 652f323f119f8c6fbc14476acf108db6ff02dd51 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Fri, 3 Oct 2025 09:42:20 +0530 Subject: [PATCH 26/34] Add tests for masking empty arrays, tables, and records --- ballerina/tests/masked_string_test.bal | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ballerina/tests/masked_string_test.bal b/ballerina/tests/masked_string_test.bal index 266b990c..0e79b8fb 100644 --- a/ballerina/tests/masked_string_test.bal +++ b/ballerina/tests/masked_string_test.bal @@ -560,6 +560,11 @@ function testMaskedStringWithBasicTypes() { test:assertEquals(toMaskedString(true), "true"); test:assertEquals(toMaskedString(xml `UserAdminReminderDon't forget the meeting!`), "UserAdminReminderDon't forget the meeting!"); test:assertEquals(toMaskedString(xml `Just some text`), "Just some text"); + test:assertEquals(toMaskedString([]), "[]"); + test:assertEquals(toMaskedString(table []), "[]"); + test:assertEquals(toMaskedString({"list": []}), string `{"list":[]}`); + record{} emptyRec = {}; + test:assertEquals(toMaskedString(emptyRec), "{}"); } @test:Config { From f93c4818ebdc76a7ebbe8aeda03aee093f5454f2 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Tue, 7 Oct 2025 09:46:28 +0530 Subject: [PATCH 27/34] Add documentation for sensitive data masking features --- README.md | 83 +++++++++++++++++++++++++++++++++++++++++++++ ballerina/README.md | 83 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) diff --git a/README.md b/README.md index 26ff0426..9857da0b 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,89 @@ The log module supports contextual logging, allowing you to create loggers with For more details and advanced usage, see the module specification and API documentation. +## Sensitive Data Masking + +The log module provides capabilities to mask sensitive data in log messages to maintain data privacy and security when dealing with personally identifiable information (PII) or other sensitive data. + +### Sensitive Data Annotation + +Use the `@log:SensitiveData` annotation to mark fields in records as sensitive. When such fields are logged, their values will be excluded or masked: + +```ballerina +import ballerina/log; + +type User record { + string id; + @log:SensitiveData + string password; + string name; +}; + +public function main() { + User user = {id: "U001", password: "mypassword", name: "John Doe"}; + log:printInfo("user details", user = user); +} +``` + +Output (with masking enabled): + +```log +time=2025-08-20T09:15:30.123+05:30 level=INFO module="" message="user details" user={"id":"U001","name":"John Doe"} +``` + +### Masking Strategies + +Configure masking strategies using the `strategy` field: + +```ballerina +type User record { + string id; + @log:SensitiveData { + strategy: { + replacement: "****" + } + } + string password; + @log:SensitiveData { + strategy: { + replacement: isolated function(string input) returns string { + return input.length() <= 2 ? "****" : input.substring(0, 1) + "****" + input.substring(input.length() - 1); + } + } + } + string ssn; + string name; +}; +``` + +### Masked String Function + +Use `log:toMaskedString()` to get the masked version of a value for custom logging implementations: + +```ballerina +User user = {id: "U001", password: "mypassword", name: "John Doe"}; +string maskedUser = log:toMaskedString(user); +io:println(maskedUser); // {"id":"U001","name":"John Doe"} +``` + +### Enable Sensitive Data Masking + +By default, sensitive data masking is disabled. Enable it in `Config.toml`: + +```toml +[ballerina.log] +enableSensitiveDataMasking = true +``` + +Or configure it per logger: + +```ballerina +log:Config secureConfig = { + enableSensitiveDataMasking: true +}; +log:Logger secureLogger = log:fromConfig(secureConfig); +``` + ## Build from the source ### Set up the prerequisites diff --git a/ballerina/README.md b/ballerina/README.md index 6f8edae0..50351753 100644 --- a/ballerina/README.md +++ b/ballerina/README.md @@ -113,3 +113,86 @@ The log module supports contextual logging, allowing you to create loggers with ``` For more details and advanced usage, see the module specification and API documentation. + +## Sensitive Data Masking + +The log module provides capabilities to mask sensitive data in log messages to maintain data privacy and security when dealing with personally identifiable information (PII) or other sensitive data. + +### Sensitive Data Annotation + +Use the `@log:SensitiveData` annotation to mark fields in records as sensitive. When such fields are logged, their values will be excluded or masked: + +```ballerina +import ballerina/log; + +type User record { + string id; + @log:SensitiveData + string password; + string name; +}; + +public function main() { + User user = {id: "U001", password: "mypassword", name: "John Doe"}; + log:printInfo("user details", user = user); +} +``` + +Output (with masking enabled): + +```log +time=2025-08-20T09:15:30.123+05:30 level=INFO module="" message="user details" user={"id":"U001","name":"John Doe"} +``` + +### Masking Strategies + +Configure masking strategies using the `strategy` field: + +```ballerina +type User record { + string id; + @log:SensitiveData { + strategy: { + replacement: "****" + } + } + string password; + @log:SensitiveData { + strategy: { + replacement: isolated function(string input) returns string { + return input.length() <= 2 ? "****" : input.substring(0, 1) + "****" + input.substring(input.length() - 1); + } + } + } + string ssn; + string name; +}; +``` + +### Masked String Function + +Use `log:toMaskedString()` to get the masked version of a value for custom logging implementations: + +```ballerina +User user = {id: "U001", password: "mypassword", name: "John Doe"}; +string maskedUser = log:toMaskedString(user); +io:println(maskedUser); // {"id":"U001","name":"John Doe"} +``` + +### Enable Sensitive Data Masking + +By default, sensitive data masking is disabled. Enable it in `Config.toml`: + +```toml +[ballerina.log] +enableSensitiveDataMasking = true +``` + +Or configure it per logger: + +```ballerina +log:Config secureConfig = { + enableSensitiveDataMasking: true +}; +log:Logger secureLogger = log:fromConfig(secureConfig); +``` From dfc98fa24a635714f6bf8082507c915d210ae997 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Tue, 7 Oct 2025 09:49:40 +0530 Subject: [PATCH 28/34] Enhance sensitive data masking documentation and functionality --- ballerina/natives.bal | 1 + ballerina/sensitive_data_masking.bal | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ballerina/natives.bal b/ballerina/natives.bal index 566367f3..d62a73e2 100644 --- a/ballerina/natives.bal +++ b/ballerina/natives.bal @@ -165,6 +165,7 @@ public enum FileWriteOption { # Process the raw template and return the processed string. # # + template - The raw template to be processed +# + enableSensitiveDataMasking - Flag to indicate if sensitive data masking is enabled # + return - The processed string public isolated function processTemplate(PrintableRawTemplate template, boolean enableSensitiveDataMasking = false) returns string { string[] templateStrings = template.strings; diff --git a/ballerina/sensitive_data_masking.bal b/ballerina/sensitive_data_masking.bal index 27e55f6f..0e444202 100644 --- a/ballerina/sensitive_data_masking.bal +++ b/ballerina/sensitive_data_masking.bal @@ -47,7 +47,8 @@ public annotation SensitiveDataConfig SensitiveData on record field; configurable boolean enableSensitiveDataMasking = false; -# Returns a masked string representation of the given data based on the sensitive data masking configuration. +# Returns a masked string representation of the given data based on the sensitive data masking annotation. +# This method panics if a cyclic value reference is encountered. # # + data - The data to be masked # + return - The masked string representation of the data From 7a8b9aa9d8665a698dec2b9f28d92a4301f31e8b Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Tue, 7 Oct 2025 11:28:55 +0530 Subject: [PATCH 29/34] Refactor sensitive data masking strategy to use a dedicated maskString function --- README.md | 13 ++++++++++--- ballerina/README.md | 13 ++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9857da0b..c92e9620 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,15 @@ time=2025-08-20T09:15:30.123+05:30 level=INFO module="" message="user details" u Configure masking strategies using the `strategy` field: ```ballerina +import ballerina/log; + +isolated function maskString(string input) returns string { + if input.length() <= 2 { + return "****"; + } + return input.substring(0, 1) + "****" + input.substring(input.length() - 1); +} + type User record { string id; @log:SensitiveData { @@ -172,9 +181,7 @@ type User record { string password; @log:SensitiveData { strategy: { - replacement: isolated function(string input) returns string { - return input.length() <= 2 ? "****" : input.substring(0, 1) + "****" + input.substring(input.length() - 1); - } + replacement: maskString } } string ssn; diff --git a/ballerina/README.md b/ballerina/README.md index 50351753..b954f739 100644 --- a/ballerina/README.md +++ b/ballerina/README.md @@ -149,6 +149,15 @@ time=2025-08-20T09:15:30.123+05:30 level=INFO module="" message="user details" u Configure masking strategies using the `strategy` field: ```ballerina +import ballerina/log; + +isolated function maskString(string input) returns string { + if input.length() <= 2 { + return "****"; + } + return input.substring(0, 1) + "****" + input.substring(input.length() - 1); +} + type User record { string id; @log:SensitiveData { @@ -159,9 +168,7 @@ type User record { string password; @log:SensitiveData { strategy: { - replacement: isolated function(string input) returns string { - return input.length() <= 2 ? "****" : input.substring(0, 1) + "****" + input.substring(input.length() - 1); - } + replacement: maskString } } string ssn; From cc0cc5dafe242cf532379a98664d2de8fe2f2f13 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Tue, 7 Oct 2025 13:32:07 +0530 Subject: [PATCH 30/34] Refactor sensitive data annotation from @SensitiveData to @Sensitive --- README.md | 8 +-- ballerina/README.md | 8 +-- ballerina/sensitive_data_masking.bal | 4 +- ballerina/tests/logger_test.bal | 2 +- ballerina/tests/masked_string_test.bal | 50 +++++++++---------- docs/spec/spec.md | 14 +++--- .../resources/samples/masked-logger/main.bal | 8 +-- .../stdlib/log/MaskedStringBuilder.java | 4 +- 8 files changed, 49 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index c92e9620..1beaa4e7 100644 --- a/README.md +++ b/README.md @@ -133,14 +133,14 @@ The log module provides capabilities to mask sensitive data in log messages to m ### Sensitive Data Annotation -Use the `@log:SensitiveData` annotation to mark fields in records as sensitive. When such fields are logged, their values will be excluded or masked: +Use the `@log:Sensitive` annotation to mark fields in records as sensitive. When such fields are logged, their values will be excluded or masked: ```ballerina import ballerina/log; type User record { string id; - @log:SensitiveData + @log:Sensitive string password; string name; }; @@ -173,13 +173,13 @@ isolated function maskString(string input) returns string { type User record { string id; - @log:SensitiveData { + @log:Sensitive { strategy: { replacement: "****" } } string password; - @log:SensitiveData { + @log:Sensitive { strategy: { replacement: maskString } diff --git a/ballerina/README.md b/ballerina/README.md index b954f739..7643ac42 100644 --- a/ballerina/README.md +++ b/ballerina/README.md @@ -120,14 +120,14 @@ The log module provides capabilities to mask sensitive data in log messages to m ### Sensitive Data Annotation -Use the `@log:SensitiveData` annotation to mark fields in records as sensitive. When such fields are logged, their values will be excluded or masked: +Use the `@log:Sensitive` annotation to mark fields in records as sensitive. When such fields are logged, their values will be excluded or masked: ```ballerina import ballerina/log; type User record { string id; - @log:SensitiveData + @log:Sensitive string password; string name; }; @@ -160,13 +160,13 @@ isolated function maskString(string input) returns string { type User record { string id; - @log:SensitiveData { + @log:Sensitive { strategy: { replacement: "****" } } string password; - @log:SensitiveData { + @log:Sensitive { strategy: { replacement: maskString } diff --git a/ballerina/sensitive_data_masking.bal b/ballerina/sensitive_data_masking.bal index 0e444202..3c47a4ab 100644 --- a/ballerina/sensitive_data_masking.bal +++ b/ballerina/sensitive_data_masking.bal @@ -36,14 +36,14 @@ public type MaskingStrategy EXCLUDE|Replacement; # Represents sensitive data with a masking strategy # # + strategy - The masking strategy to apply (default: EXCLUDE) -public type SensitiveDataConfig record {| +public type SensitiveConfig record {| MaskingStrategy strategy = EXCLUDE; |}; # Marks a record field or type as sensitive, excluding it from log output # # + strategy - The masking strategy to apply (default: EXCLUDE) -public annotation SensitiveDataConfig SensitiveData on record field; +public annotation SensitiveConfig Sensitive on record field; configurable boolean enableSensitiveDataMasking = false; diff --git a/ballerina/tests/logger_test.bal b/ballerina/tests/logger_test.bal index 8599910d..d7a99108 100644 --- a/ballerina/tests/logger_test.bal +++ b/ballerina/tests/logger_test.bal @@ -24,7 +24,7 @@ type Context record {| string id; // By default root logger is configured not to mask sensitive data // So this is added as a negative test case - @SensitiveData + @Sensitive string msg; |}; diff --git a/ballerina/tests/masked_string_test.bal b/ballerina/tests/masked_string_test.bal index 0e79b8fb..2643c24f 100644 --- a/ballerina/tests/masked_string_test.bal +++ b/ballerina/tests/masked_string_test.bal @@ -36,13 +36,13 @@ function checkJsonParsing(string maskedStr) { type User record {| string name; - @SensitiveData + @Sensitive string ssn; - @SensitiveData {strategy: {replacement: "*****"}} + @Sensitive {strategy: {replacement: "*****"}} string password; - @SensitiveData {strategy: {replacement: maskStringPartially}} + @Sensitive {strategy: {replacement: maskStringPartially}} string mail; - @SensitiveData {strategy: EXCLUDE} + @Sensitive {strategy: EXCLUDE} string creditCard; |}; @@ -100,7 +100,7 @@ function testMaskedStringWithAnydataValues() { type OpenAnydataRecord record { string name; - @SensitiveData + @Sensitive anydata sensitiveField; }; @@ -130,30 +130,30 @@ function testMaskedStringWithOpenAnydataRecord() { type Record1 record {| string field1; - @SensitiveData + @Sensitive Record2 field2; Record2 field3; |}; type Record2 record {| string subField1; - @SensitiveData {strategy: {replacement: "###"}} + @Sensitive {strategy: {replacement: "###"}} string subField2; - @SensitiveData {strategy: {replacement: maskStringPartially}} + @Sensitive {strategy: {replacement: maskStringPartially}} string subField3; - @SensitiveData {strategy: EXCLUDE} + @Sensitive {strategy: EXCLUDE} string subField4; |}; type Record3 record {| string info; - @SensitiveData + @Sensitive string details; |}; type NestedRecord record {| string name; - @SensitiveData + @Sensitive Record1 details1; Record1 details2; Record3[] records; @@ -208,7 +208,7 @@ function testMaskedStringWithNestedRecords() { type NilableSensitiveFieldRecord record {| string name; - @SensitiveData + @Sensitive string? sensitiveField; int? id; |}; @@ -230,7 +230,7 @@ function testMaskedStringWithNilableSensitiveField() { type OptionalSensitiveFieldRecord record {| string name; - @SensitiveData + @Sensitive string sensitiveField?; int id?; |}; @@ -262,7 +262,7 @@ function testMaskedStringWithOptionalSensitiveField() { type NeverSensitiveFieldRecord record {| string name; - @SensitiveData + @Sensitive never sensitiveField?; |}; @@ -282,7 +282,7 @@ function testMaskedStringWithNeverSensitiveField() { type RecordWithRestField record {| string name; - @SensitiveData + @Sensitive string sensitiveField; string...; |}; @@ -325,7 +325,7 @@ function testMaskedStringWithCyclicRecord() { type RecordWithCyclicSensitiveField record {| string name; - @SensitiveData + @Sensitive RecordWithCyclicSensitiveField child?; |}; @@ -385,15 +385,15 @@ type SpecialCharFieldsRec record {| |}; type SpecialCharSensitiveFieldsRec record {| - @SensitiveData {strategy: {replacement: "*****"}} + @Sensitive {strategy: {replacement: "*****"}} string field_with_underscores; - @SensitiveData {strategy: {replacement: "#####"}} + @Sensitive {strategy: {replacement: "#####"}} string FieldWithCamelCase; - @SensitiveData {strategy: {replacement: "1!1!1!"}} + @Sensitive {strategy: {replacement: "1!1!1!"}} string field\-With\$pecialChar\!; - @SensitiveData {strategy: {replacement: "[REDACTED]"}} + @Sensitive {strategy: {replacement: "[REDACTED]"}} string 'type; - @SensitiveData {strategy: {replacement: "~~~~~~"}} + @Sensitive {strategy: {replacement: "~~~~~~"}} string 'value\\\-Field; |}; @@ -433,13 +433,13 @@ function testMaskedStringWithSpecialCharFields() { type ReadonlyUser1 readonly & record {| string name; - @SensitiveData + @Sensitive string ssn; - @SensitiveData {strategy: {replacement: "*****"}} + @Sensitive {strategy: {replacement: "*****"}} string password; - @SensitiveData {strategy: {replacement: maskStringPartially}} + @Sensitive {strategy: {replacement: maskStringPartially}} string mail; - @SensitiveData {strategy: EXCLUDE} + @Sensitive {strategy: EXCLUDE} string creditCard; |}; diff --git a/docs/spec/spec.md b/docs/spec/spec.md index 5009a213..54554109 100644 --- a/docs/spec/spec.md +++ b/docs/spec/spec.md @@ -307,14 +307,14 @@ The Ballerina log module provides the capability to mask sensitive data in log m ### 5.1. Sensitive data annotation -The `@log:SensitiveData` annotation can be used to mark fields in a record as sensitive. When such fields are logged, their values will be excluded or masked to prevent exposure of sensitive information. +The `@log:Sensitive` annotation can be used to mark fields in a record as sensitive. When such fields are logged, their values will be excluded or masked to prevent exposure of sensitive information. ```ballerina import ballerina/log; type User record { string id; - @log:SensitiveData + @log:Sensitive string password; string name; }; @@ -331,7 +331,7 @@ Output: time=2025-08-20T09:15:30.123+05:30 level=INFO module="" message="user details" user={"id":"U001","name":"John Doe"} ``` -By default, the `@log:SensitiveData` annotation will exclude the sensitive field from the log output when sensitive data masking is enabled. +By default, the `@log:Sensitive` annotation will exclude the sensitive field from the log output when sensitive data masking is enabled. Additionally, the masking strategy can be configured using the `strategy` field of the annotation. The available strategies are: 1. `EXCLUDE`: Excludes the field from the log output (default behavior). @@ -351,13 +351,13 @@ isolated function maskString(string input) returns string { type User record { string id; - @log:SensitiveData { + @log:Sensitive { strategy: { replacement: "****" } } string password; - @log:SensitiveData { + @log:Sensitive { strategy: { replacement: maskString } @@ -388,7 +388,7 @@ import ballerina/io; type User record { string id; - @log:SensitiveData + @log:Sensitive string password; string name; }; @@ -411,7 +411,7 @@ Output: > ```ballerina > type User record { > string id; -> @log:SensitiveData +> @log:Sensitive > string password; > string name; > }; diff --git a/integration-tests/tests/resources/samples/masked-logger/main.bal b/integration-tests/tests/resources/samples/masked-logger/main.bal index b93250e9..a61f3936 100644 --- a/integration-tests/tests/resources/samples/masked-logger/main.bal +++ b/integration-tests/tests/resources/samples/masked-logger/main.bal @@ -18,13 +18,13 @@ import ballerina/log; type User record {| string name; - @log:SensitiveData + @log:Sensitive string ssn; - @log:SensitiveData {strategy: {replacement: "*****"}} + @log:Sensitive {strategy: {replacement: "*****"}} string password; - @log:SensitiveData {strategy: {replacement: maskStringPartially}} + @log:Sensitive {strategy: {replacement: maskStringPartially}} string mail; - @log:SensitiveData {strategy: log:EXCLUDE} + @log:Sensitive {strategy: log:EXCLUDE} string creditCard; |}; diff --git a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java index 6b59cd4b..4885ec32 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java +++ b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java @@ -57,7 +57,7 @@ public class MaskedStringBuilder implements AutoCloseable { private static final BString EXCLUDE_VALUE = StringUtils.fromString("EXCLUDE"); private static final String FIELD_PREFIX = "$field$."; private static final String LOG_ANNOTATION_PREFIX = "ballerina/log"; - private static final String SENSITIVE_DATA_SUFFIX = ":SensitiveData"; + private static final String SENSITIVE_SUFFIX = ":Sensitive"; private static final BString CYCLIC_REFERENCE_ERROR = StringUtils.fromString("Cyclic value reference detected " + "in the record"); @@ -520,7 +520,7 @@ public static MaskedStringBuilder create(Runtime runtime, int initialCapacity) { for (Object key : keys) { if (key instanceof BString bStringKey) { String keyValue = bStringKey.getValue(); - if (keyValue.endsWith(SENSITIVE_DATA_SUFFIX) && keyValue.startsWith(LOG_ANNOTATION_PREFIX)) { + if (keyValue.endsWith(SENSITIVE_SUFFIX) && keyValue.startsWith(LOG_ANNOTATION_PREFIX)) { Object annotation = fieldAnnotationMap.get(key); if (annotation instanceof BMap bMapAnnotation) { return Optional.of(bMapAnnotation); From a4a6766d71d1c570c48e5d850d05c58517ae9a59 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Tue, 7 Oct 2025 13:33:59 +0530 Subject: [PATCH 31/34] Deprecate processTemplate function and replace with evaluateTemplate for improved clarity --- ballerina/natives.bal | 33 ++++++++++++++++--- ballerina/root_logger.bal | 4 +-- docs/spec/spec.md | 4 +-- .../samples/logger/custom-logger/main.bal | 6 ++-- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/ballerina/natives.bal b/ballerina/natives.bal index d62a73e2..eadc5f9d 100644 --- a/ballerina/natives.bal +++ b/ballerina/natives.bal @@ -165,9 +165,34 @@ public enum FileWriteOption { # Process the raw template and return the processed string. # # + template - The raw template to be processed -# + enableSensitiveDataMasking - Flag to indicate if sensitive data masking is enabled # + return - The processed string -public isolated function processTemplate(PrintableRawTemplate template, boolean enableSensitiveDataMasking = false) returns string { +# +# # Deprecated +# The `processTemplate` function is deprecated. Use `evaluateTemplate` instead. +@deprecated +public isolated function processTemplate(PrintableRawTemplate template) returns string { + string[] templateStrings = template.strings; + Value[] insertions = template.insertions; + string result = templateStrings[0]; + + foreach int i in 1 ..< templateStrings.length() { + Value insertion = insertions[i - 1]; + string insertionStr = insertion is PrintableRawTemplate ? + processTemplate(insertion) : + insertion is Valuer ? + insertion().toString() : + insertion.toString(); + result += insertionStr + templateStrings[i]; + } + return result; +} + +# Evaluates the raw template and returns the evaluated string. +# +# + template - The raw template to be evaluated +# + enableSensitiveDataMasking - Flag to indicate if sensitive data masking is enabled +# + return - The evaluated string +public isolated function evaluateTemplate(PrintableRawTemplate template, boolean enableSensitiveDataMasking = false) returns string { string[] templateStrings = template.strings; Value[] insertions = template.insertions; string result = templateStrings[0]; @@ -175,7 +200,7 @@ public isolated function processTemplate(PrintableRawTemplate template, boolean foreach int i in 1 ..< templateStrings.length() { Value insertion = insertions[i - 1]; string insertionStr = insertion is PrintableRawTemplate ? - processTemplate(insertion, enableSensitiveDataMasking) : + evaluateTemplate(insertion, enableSensitiveDataMasking) : insertion is Valuer ? (enableSensitiveDataMasking ? toMaskedString(insertion()) : insertion().toString()) : (enableSensitiveDataMasking ? toMaskedString(insertion) : insertion.toString()); @@ -185,7 +210,7 @@ public isolated function processTemplate(PrintableRawTemplate template, boolean } isolated function processMessage(string|PrintableRawTemplate msg, boolean enableSensitiveDataMasking) returns string => - msg !is string ? processTemplate(msg, enableSensitiveDataMasking) : msg; + msg !is string ? evaluateTemplate(msg, enableSensitiveDataMasking) : msg; # Prints debug logs. # ```ballerina diff --git a/ballerina/root_logger.bal b/ballerina/root_logger.bal index 9fced72e..01ae0551 100644 --- a/ballerina/root_logger.bal +++ b/ballerina/root_logger.bal @@ -139,7 +139,7 @@ isolated class RootLogger { } foreach [string, Value] [k, v] in keyValues.entries() { logRecord[k] = v is Valuer ? v() : - (v is PrintableRawTemplate ? processMessage(v, self.enableSensitiveDataMasking) : v); + (v is PrintableRawTemplate ? evaluateTemplate(v, self.enableSensitiveDataMasking) : v); } if observe:isTracingEnabled() { map spanContext = observe:getSpanContext(); @@ -149,7 +149,7 @@ isolated class RootLogger { } foreach [string, Value] [k, v] in self.keyValues.entries() { logRecord[k] = v is Valuer ? v() : - (v is PrintableRawTemplate ? processMessage(v, self.enableSensitiveDataMasking) : v); + (v is PrintableRawTemplate ? evaluateTemplate(v, self.enableSensitiveDataMasking) : v); } string logOutput = self.format == JSON_FORMAT ? diff --git a/docs/spec/spec.md b/docs/spec/spec.md index 54554109..bf03ae7d 100644 --- a/docs/spec/spec.md +++ b/docs/spec/spec.md @@ -228,10 +228,10 @@ public type Logger isolated object { }; ``` -> **Note:** The Ballerina log module provides a function to process the PrintableRawTemplate to obtain the processed string. This can be used when implementing a logger from the above type. +> **Note:** The Ballerina log module provides a function to evaluate the `PrintableRawTemplate` to obtain the evaluated string. This can be used when implementing a logger from the above type. > > ```ballerina -> public isolated function processTemplate(PrintableRawTemplate) returns string; +> public isolated function evaluateTemplate(PrintableRawTemplate rawTemplate, boolean enableSensitiveDataMasking = false) returns string; > ``` ### 4.2. Root logger diff --git a/integration-tests/tests/resources/samples/logger/custom-logger/main.bal b/integration-tests/tests/resources/samples/logger/custom-logger/main.bal index c3dad21a..23bd209a 100644 --- a/integration-tests/tests/resources/samples/logger/custom-logger/main.bal +++ b/integration-tests/tests/resources/samples/logger/custom-logger/main.bal @@ -70,7 +70,7 @@ isolated class CustomLogger { public isolated function withContext(*log:KeyValues keyValues) returns log:Logger|error { log:AnydataKeyValues newKeyValues = {...self.keyValues}; foreach [string, log:Value] [k, v] in keyValues.entries() { - newKeyValues[k] = v is log:Valuer ? v() : v is anydata ? v : log:processTemplate(v); + newKeyValues[k] = v is log:Valuer ? v() : v is anydata ? v : log:evaluateTemplate(v); } return new CustomLogger(filePath = self.filePath, level = self.level, keyValues = newKeyValues.cloneReadOnly()); } @@ -80,7 +80,7 @@ isolated class CustomLogger { return; } string timestamp = time:utcToEmailString(time:utcNow()); - string message = msg is string ? msg : log:processTemplate(msg); + string message = msg is string ? msg : log:evaluateTemplate(msg); string logMessage = string `[${timestamp}] {${level}} "${message}" `; if 'error is error { logMessage += string `error="${'error.message()}"`; @@ -95,7 +95,7 @@ isolated class CustomLogger { logMessage += string ` ${k}="${v.toString()}"`; } foreach [string, log:Value] [k, v] in keyValues.entries() { - anydata value = v is log:Valuer ? v() : v is anydata ? v : log:processTemplate(v); + anydata value = v is log:Valuer ? v() : v is anydata ? v : log:evaluateTemplate(v); logMessage += string ` ${k}="${value.toString()}"`; } logMessage += "\n"; From 37603fbb99b5a2cec03f41ccd367ec65e3152956 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Wed, 8 Oct 2025 12:03:24 +0530 Subject: [PATCH 32/34] Address review suggestions --- README.md | 34 ++-- ballerina/README.md | 33 ++-- ballerina/sensitive_data_masking.bal | 11 +- docs/spec/spec.md | 99 ++++++----- .../tests/test_logger_masking.bal | 2 +- .../stdlib/log/MaskedStringBuilder.java | 158 +++++++++--------- 6 files changed, 174 insertions(+), 163 deletions(-) diff --git a/README.md b/README.md index 1beaa4e7..c01725df 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,22 @@ For more details and advanced usage, see the module specification and API docume The log module provides capabilities to mask sensitive data in log messages to maintain data privacy and security when dealing with personally identifiable information (PII) or other sensitive data. +By default, sensitive data masking is disabled. Enable it in `Config.toml`: + +```toml +[ballerina.log] +enableSensitiveDataMasking = true +``` + +Or configure it per logger: + +```ballerina +log:Config secureConfig = { + enableSensitiveDataMasking: true +}; +log:Logger secureLogger = log:fromConfig(secureConfig); +``` + ### Sensitive Data Annotation Use the `@log:Sensitive` annotation to mark fields in records as sensitive. When such fields are logged, their values will be excluded or masked: @@ -199,24 +215,6 @@ string maskedUser = log:toMaskedString(user); io:println(maskedUser); // {"id":"U001","name":"John Doe"} ``` -### Enable Sensitive Data Masking - -By default, sensitive data masking is disabled. Enable it in `Config.toml`: - -```toml -[ballerina.log] -enableSensitiveDataMasking = true -``` - -Or configure it per logger: - -```ballerina -log:Config secureConfig = { - enableSensitiveDataMasking: true -}; -log:Logger secureLogger = log:fromConfig(secureConfig); -``` - ## Build from the source ### Set up the prerequisites diff --git a/ballerina/README.md b/ballerina/README.md index 7643ac42..683d4c77 100644 --- a/ballerina/README.md +++ b/ballerina/README.md @@ -118,6 +118,22 @@ For more details and advanced usage, see the module specification and API docume The log module provides capabilities to mask sensitive data in log messages to maintain data privacy and security when dealing with personally identifiable information (PII) or other sensitive data. +By default, sensitive data masking is disabled. Enable it in `Config.toml`: + +```toml +[ballerina.log] +enableSensitiveDataMasking = true +``` + +Or configure it per logger: + +```ballerina +log:Config secureConfig = { + enableSensitiveDataMasking: true +}; +log:Logger secureLogger = log:fromConfig(secureConfig); +``` + ### Sensitive Data Annotation Use the `@log:Sensitive` annotation to mark fields in records as sensitive. When such fields are logged, their values will be excluded or masked: @@ -186,20 +202,3 @@ string maskedUser = log:toMaskedString(user); io:println(maskedUser); // {"id":"U001","name":"John Doe"} ``` -### Enable Sensitive Data Masking - -By default, sensitive data masking is disabled. Enable it in `Config.toml`: - -```toml -[ballerina.log] -enableSensitiveDataMasking = true -``` - -Or configure it per logger: - -```ballerina -log:Config secureConfig = { - enableSensitiveDataMasking: true -}; -log:Logger secureLogger = log:fromConfig(secureConfig); -``` diff --git a/ballerina/sensitive_data_masking.bal b/ballerina/sensitive_data_masking.bal index 3c47a4ab..f4d13dc2 100644 --- a/ballerina/sensitive_data_masking.bal +++ b/ballerina/sensitive_data_masking.bal @@ -23,10 +23,9 @@ public const EXCLUDE = "EXCLUDE"; public type ReplacementFunction isolated function (string input) returns string; # Replacement strategy for sensitive data -# -# + replacement - The replacement value. This can be a string which will be used to replace the -# entire value, or a function that takes the original value and returns a masked version. public type Replacement record {| + # The replacement value. This can be a string which will be used to replace the + # entire value, or a function that takes the original value and returns a masked version. string|ReplacementFunction replacement; |}; @@ -34,15 +33,13 @@ public type Replacement record {| public type MaskingStrategy EXCLUDE|Replacement; # Represents sensitive data with a masking strategy -# -# + strategy - The masking strategy to apply (default: EXCLUDE) public type SensitiveConfig record {| + # The masking strategy to apply MaskingStrategy strategy = EXCLUDE; |}; # Marks a record field or type as sensitive, excluding it from log output -# -# + strategy - The masking strategy to apply (default: EXCLUDE) +# The default strategy is to exclude the field from log output public annotation SensitiveConfig Sensitive on record field; configurable boolean enableSensitiveDataMasking = false; diff --git a/docs/spec/spec.md b/docs/spec/spec.md index bf03ae7d..926c64c0 100644 --- a/docs/spec/spec.md +++ b/docs/spec/spec.md @@ -3,7 +3,7 @@ _Authors_: @daneshk @MadhukaHarith92 @TharmiganK _Reviewers_: @daneshk @ThisaruGuruge _Created_: 2021/11/15 -_Updated_: 2025/10/02 +_Updated_: 2025/10/08 _Edition_: Swan Lake ## Introduction @@ -34,7 +34,7 @@ The conforming implementation of the specification is released and included in t 5. [Sensitive data masking](#5-sensitive-data-masking) * 5.1. [Sensitive data annotation](#51-sensitive-data-annotation) * 5.2. [Masked string function](#52-masked-string-function) - * 5.3. [Configure sensitive data masking](#53-configure-sensitive-data-masking) + * 5.3. [Type-based masking](#53-type-based-masking) ## 1. Overview @@ -305,10 +305,28 @@ auditLogger.printInfo("Hello World from the audit logger!"); The Ballerina log module provides the capability to mask sensitive data in log messages. This is crucial for maintaining data privacy and security, especially when dealing with personally identifiable information (PII) or other sensitive data. +By default, sensitive data masking is disabled. Enable it in `Config.toml`: + +```toml +[ballerina.log] +enableSensitiveDataMasking = true +``` + +Or configure it per logger: + +```ballerina +log:Config secureConfig = { + enableSensitiveDataMasking: true +}; +log:Logger secureLogger = log:fromConfig(secureConfig); +``` + ### 5.1. Sensitive data annotation The `@log:Sensitive` annotation can be used to mark fields in a record as sensitive. When such fields are logged, their values will be excluded or masked to prevent exposure of sensitive information. + + ```ballerina import ballerina/log; @@ -406,48 +424,43 @@ Output: {"id":"U001","name":"John Doe"} ``` -> **Note:** The masking is based on the type of the value. Since, Ballerina is a structurally typed language, same value can be assigned to different typed variables. So the masking is based on the actual value type which is determined at the value creation time. The original type information can be extracted using the `typeof` operator. -> Example: -> ```ballerina -> type User record { -> string id; -> @log:Sensitive -> string password; -> string name; -> }; -> -> type Student record { -> string id; -> string password; // Not marked as sensitive -> string name; -> }; -> -> public function main() returns error? { -> User user = {id: "U001", password: "mypassword", name: "John Doe"}; -> // password will be masked -> string maskedUser = log:toMaskedString(user); -> -> Student student = user; // Allowed since both have the same structure -> // password will be masked since the type at value creation is User -> string maskedStudent = log:toMaskedString(student); -> -> student = {id: "S001", password: "studentpass", name: "Jane Doe"}; -> user = student; // Allowed since both have the same structure -> // password will not be masked since the type at value creation is Student -> maskedStudent = log:toMaskedString(user); -> -> // Explicity creating a value with type -> user = check student.cloneWithType(); -> // password will be masked since the type at value creation is User -> maskedUser = log:toMaskedString(user); -> } -> ``` +### 5.3. Type-based masking -### 5.3. Configure sensitive data masking +The masking is based on the type of the value. Since, Ballerina is a structurally typed language, same value can be assigned to different typed variables. So the masking is based on the actual value type which is determined at the value creation time. The original type information can be extracted using the `typeof` operator. -By default, sensitive data masking is disabled. It can be enabled via a configurable variable in the `Config.toml` file. +Example: -```toml -[ballerina.log] -enableSensitiveDataMasking = true +```ballerina +type User record { + string id; + @log:Sensitive + string password; + string name; +}; + +type Student record { + string id; + string password; // Not marked as sensitive + string name; +}; + +public function main() returns error? { + User user = {id: "U001", password: "mypassword", name: "John Doe"}; + // password will be masked + string maskedUser = log:toMaskedString(user); + + Student student = user; // Allowed since both have the same structure + // password will be masked since the type at value creation is User + string maskedStudent = log:toMaskedString(student); + + student = {id: "S001", password: "studentpass", name: "Jane Doe"}; + user = student; // Allowed since both have the same structure + // password will not be masked since the type at value creation is Student + maskedStudent = log:toMaskedString(user); + + // Explicity creating a value with type + user = check student.cloneWithType(); + // password will be masked since the type at value creation is User + maskedUser = log:toMaskedString(user); +} ``` diff --git a/integration-tests/tests/test_logger_masking.bal b/integration-tests/tests/test_logger_masking.bal index 3b459d2d..8e054127 100644 --- a/integration-tests/tests/test_logger_masking.bal +++ b/integration-tests/tests/test_logger_masking.bal @@ -36,4 +36,4 @@ function testMaskedLogger() returns error? { test:assertTrue(logLines[6].includes(string `level=DEBUG module=wso2/masked_logger message="user details: {\"name\":\"John Doe\",\"password\":\"*****\",\"mail\":\"joh**************com\"}"`)); test:assertTrue(logLines[7].includes(string `level=ERROR module=wso2/masked_logger message="error occurred" userDetails={"name":"John Doe","password":"*****","mail":"joh**************com"}`)); check sc.close(); -} \ No newline at end of file +} diff --git a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java index 4885ec32..d18508f7 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java +++ b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java @@ -71,6 +71,7 @@ public class MaskedStringBuilder implements AutoCloseable { // Pre-computed hex lookup table for efficient Unicode escaping private static final char[] HEX_CHARS = "0123456789abcdef".toCharArray(); + // JSON escape character arrays for efficient escaping private static final char[] QUOTE_ESCAPE = {'\\', '"'}; private static final char[] BACKSLASH_ESCAPE = {'\\', '\\'}; private static final char[] NEWLINE_ESCAPE = {'\\', 'n'}; @@ -79,6 +80,10 @@ public class MaskedStringBuilder implements AutoCloseable { private static final char[] BACKSPACE_ESCAPE = {'\\', 'b'}; private static final char[] FORM_FEED_ESCAPE = {'\\', 'f'}; + // Control character range constants for Unicode escaping + private static final int ASCII_CONTROL_CHAR_LIMIT = 0x20; // Space character (32) + private static final int ASCII_DEL_CHAR = 0x7F; // DEL character (127) + private final Runtime runtime; private final IdentityHashMap visitedValues; private StringBuilder stringBuilder; @@ -91,16 +96,13 @@ public class MaskedStringBuilder implements AutoCloseable { private static final int ESCAPE_BUFFER_SIZE = 64; public MaskedStringBuilder(Runtime runtime) { - this.runtime = runtime; - this.visitedValues = new IdentityHashMap<>(); - this.stringBuilder = new StringBuilder(DEFAULT_INITIAL_CAPACITY); - this.escapeBuffer = new StringBuilder(ESCAPE_BUFFER_SIZE); + this(runtime, DEFAULT_INITIAL_CAPACITY); } public MaskedStringBuilder(Runtime runtime, int initialCapacity) { this.runtime = runtime; this.visitedValues = new IdentityHashMap<>(); - this.stringBuilder = new StringBuilder(Math.max(initialCapacity, DEFAULT_INITIAL_CAPACITY)); + this.stringBuilder = new StringBuilder(initialCapacity < 0 ? DEFAULT_INITIAL_CAPACITY : initialCapacity); this.escapeBuffer = new StringBuilder(ESCAPE_BUFFER_SIZE); } @@ -111,29 +113,29 @@ public MaskedStringBuilder(Runtime runtime, int initialCapacity) { * @return the masked string representation */ public String build(Object value) { - if (closed) { + if (this.closed) { throw ErrorCreator.createError(MASKED_STRING_BUILDER_HAS_BEEN_CLOSED); } try { - visitedValues.clear(); - stringBuilder.setLength(0); + this.visitedValues.clear(); + this.stringBuilder.setLength(0); String result = buildInternal(value); // If the builder grew too large, replace it with a smaller one for future use - if (stringBuilder.capacity() > MAX_REUSABLE_CAPACITY) { - stringBuilder = new StringBuilder(DEFAULT_INITIAL_CAPACITY); + if (this.stringBuilder.capacity() > MAX_REUSABLE_CAPACITY) { + this.stringBuilder = new StringBuilder(DEFAULT_INITIAL_CAPACITY); } // Reset escape buffer if it grew too large - if (escapeBuffer.capacity() > ESCAPE_BUFFER_SIZE * 4) { - escapeBuffer = new StringBuilder(ESCAPE_BUFFER_SIZE); + if (this.escapeBuffer.capacity() > ESCAPE_BUFFER_SIZE * 4) { + this.escapeBuffer = new StringBuilder(ESCAPE_BUFFER_SIZE); } return result; } finally { - visitedValues.clear(); + this.visitedValues.clear(); } } @@ -146,7 +148,7 @@ private String buildInternal(Object value) { } // Use identity-based checking for cycle detection - if (visitedValues.put(value, Boolean.TRUE) != null) { + if (this.visitedValues.put(value, Boolean.TRUE) != null) { // Panics on cyclic value references throw ErrorCreator.createError(CYCLIC_REFERENCE_ERROR); } @@ -154,7 +156,7 @@ private String buildInternal(Object value) { try { return processValue(value); } finally { - visitedValues.remove(value); + this.visitedValues.remove(value); } } @@ -215,14 +217,14 @@ private String processMapValue(BMap mapValue, Type valueType) { private String processRecordValue(BMap mapValue, Map> fieldAnnotations, Map fields) { - int startPos = stringBuilder.length(); + int startPos = this.stringBuilder.length(); - stringBuilder.append('{'); + this.stringBuilder.append('{'); addRecordFields(mapValue, fields, fieldAnnotations); - stringBuilder.append('}'); + this.stringBuilder.append('}'); - String result = stringBuilder.substring(startPos); - stringBuilder.setLength(startPos); + String result = this.stringBuilder.substring(startPos); + this.stringBuilder.setLength(startPos); return result; } @@ -245,9 +247,9 @@ private void addRecordFields(BMap mapValue, Map fields, private boolean addDynamicFieldValue(Object fieldValue, boolean first, String fieldName) { String fieldStringValue = buildInternal(fieldValue); if (!first) { - stringBuilder.append(','); + this.stringBuilder.append(','); } - appendFieldToJsonOptimized(fieldName, fieldStringValue, false, fieldValue); + appendFieldToJson(fieldName, fieldStringValue, false, fieldValue); return false; } @@ -260,63 +262,64 @@ private boolean addDefinedFieldValue(Map> fieldAnnotations, S if (fieldStringValue.isPresent()) { if (!first) { - stringBuilder.append(','); + this.stringBuilder.append(','); } - appendFieldToJsonOptimized(fieldName, fieldStringValue.get(), annotation.isPresent(), fieldValue); + appendFieldToJson(fieldName, fieldStringValue.get(), annotation.isPresent(), fieldValue); first = false; } return first; } /** - * Optimized version of appendFieldToJson that writes directly to StringBuilder + * Append field to JSON format by writing directly to StringBuilder * without creating intermediate String objects for better performance. */ - private void appendFieldToJsonOptimized(String fieldName, String value, boolean hasAnnotation, Object fieldValue) { - stringBuilder.append('"'); - appendEscapedStringOptimized(fieldName); - stringBuilder.append("\":"); + private void appendFieldToJson(String fieldName, String value, boolean hasAnnotation, Object fieldValue) { + this.stringBuilder.append('"'); + appendEscapedString(fieldName); + this.stringBuilder.append("\":"); if (hasAnnotation || fieldValue instanceof BString || fieldValue instanceof BXml) { - stringBuilder.append('"'); - appendEscapedStringOptimized(value); - stringBuilder.append('"'); + this.stringBuilder.append('"'); + appendEscapedString(value); + this.stringBuilder.append('"'); } else { - stringBuilder.append(value); + this.stringBuilder.append(value); } } /** - * Optimized method to append escaped string directly to the main StringBuilder. + * Append escaped string directly to the main StringBuilder. * This avoids creating intermediate String objects for better performance. */ - private void appendEscapedStringOptimized(String input) { + private void appendEscapedString(String input) { if (input == null) { - stringBuilder.append("null"); + this.stringBuilder.append("null"); return; } if (!needsEscaping(input)) { - stringBuilder.append(input); + this.stringBuilder.append(input); return; } for (int i = 0; i < input.length(); i++) { char c = input.charAt(i); switch (c) { - case '"' -> stringBuilder.append(QUOTE_ESCAPE); - case '\\' -> stringBuilder.append(BACKSLASH_ESCAPE); - case '\b' -> stringBuilder.append(BACKSPACE_ESCAPE); - case '\f' -> stringBuilder.append(FORM_FEED_ESCAPE); - case '\n' -> stringBuilder.append(NEWLINE_ESCAPE); - case '\r' -> stringBuilder.append(CARRIAGE_RETURN_ESCAPE); - case '\t' -> stringBuilder.append(TAB_ESCAPE); + case '"' -> this.stringBuilder.append(QUOTE_ESCAPE); + case '\\' -> this.stringBuilder.append(BACKSLASH_ESCAPE); + case '\b' -> this.stringBuilder.append(BACKSPACE_ESCAPE); + case '\f' -> this.stringBuilder.append(FORM_FEED_ESCAPE); + case '\n' -> this.stringBuilder.append(NEWLINE_ESCAPE); + case '\r' -> this.stringBuilder.append(CARRIAGE_RETURN_ESCAPE); + case '\t' -> this.stringBuilder.append(TAB_ESCAPE); default -> { - if (c < 0x20 || c == 0x7F) { - stringBuilder.append("\\u00"); - stringBuilder.append(HEX_CHARS[(c >>> 4) & 0xF]); - stringBuilder.append(HEX_CHARS[c & 0xF]); + // Escape ASCII control characters (0x00-0x1F) and DEL character (0x7F) + if (c < ASCII_CONTROL_CHAR_LIMIT || c == ASCII_DEL_CHAR) { + this.stringBuilder.append("\\u00"); + this.stringBuilder.append(HEX_CHARS[(c >>> 4) & 0xF]); + this.stringBuilder.append(HEX_CHARS[c & 0xF]); } else { - stringBuilder.append(c); + this.stringBuilder.append(c); } } } @@ -350,23 +353,23 @@ private String processTableValue(BTable tableValue) { return "[]"; } - int startPos = stringBuilder.length(); - stringBuilder.append('['); + int startPos = this.stringBuilder.length(); + this.stringBuilder.append('['); boolean first = true; for (Object row : values) { if (!first) { - stringBuilder.append(','); + this.stringBuilder.append(','); } String elementString = buildInternal(row); appendValueToArray(elementString, row); first = false; } - stringBuilder.append(']'); + this.stringBuilder.append(']'); - String result = stringBuilder.substring(startPos); - stringBuilder.setLength(startPos); + String result = this.stringBuilder.substring(startPos); + this.stringBuilder.setLength(startPos); return result; } @@ -376,34 +379,34 @@ private String processArrayValue(BArray listValue) { return "[]"; } - int startPos = stringBuilder.length(); - stringBuilder.append('['); + int startPos = this.stringBuilder.length(); + this.stringBuilder.append('['); // Using traditional for loop instead of for-each loop since BArray giving // this error: Cannot read the array length because "" is null for (long i = 0; i < length; i++) { if (i > 0) { - stringBuilder.append(','); + this.stringBuilder.append(','); } Object element = listValue.get(i); String elementString = buildInternal(element); appendValueToArray(elementString, element); } - stringBuilder.append(']'); + this.stringBuilder.append(']'); - String result = stringBuilder.substring(startPos); - stringBuilder.setLength(startPos); + String result = this.stringBuilder.substring(startPos); + this.stringBuilder.setLength(startPos); return result; } private void appendValueToArray(String value, Object originalValue) { if (originalValue instanceof BString) { - stringBuilder.append('"'); - appendEscapedStringOptimized(value); - stringBuilder.append('"'); + this.stringBuilder.append('"'); + appendEscapedString(value); + this.stringBuilder.append('"'); } else { - stringBuilder.append(value); + this.stringBuilder.append(value); } } @@ -421,7 +424,8 @@ private static boolean isBasicType(Object value) { private static boolean needsEscaping(String input) { for (int i = 0; i < input.length(); i++) { char c = input.charAt(i); - if (c == '"' || c == '\\' || (c & 0xFFE0) == 0 || c == 0x7F) { + // Check for quote, backslash, control characters (0x00-0x1F), or DEL character (0x7F) + if (c == '"' || c == '\\' || (c & 0xFFE0) == 0 || c == ASCII_DEL_CHAR) { return true; } } @@ -435,7 +439,7 @@ private static boolean needsEscaping(String input) { * @return current capacity */ public int getCapacity() { - return stringBuilder.capacity(); + return this.stringBuilder.capacity(); } /** @@ -443,12 +447,12 @@ public int getCapacity() { * This is more efficient than creating a new builder instance. */ public void reset() { - if (closed) { + if (this.closed) { throw ErrorCreator.createError(MASKED_STRING_BUILDER_HAS_BEEN_CLOSED); } - visitedValues.clear(); - stringBuilder.setLength(0); - escapeBuffer.setLength(0); + this.visitedValues.clear(); + this.stringBuilder.setLength(0); + this.escapeBuffer.setLength(0); } /** @@ -457,16 +461,16 @@ public void reset() { * @return true if closed, false otherwise */ public boolean isClosed() { - return closed; + return this.closed; } @Override public void close() { - if (!closed) { - visitedValues.clear(); - stringBuilder = null; - escapeBuffer = null; - closed = true; + if (!this.closed) { + this.visitedValues.clear(); + this.stringBuilder = null; + this.escapeBuffer = null; + this.closed = true; } } From 5e453103126dc42724c847b41e88845e28411134 Mon Sep 17 00:00:00 2001 From: TharmiganK Date: Wed, 8 Oct 2025 16:10:09 +0530 Subject: [PATCH 33/34] Address review suggestions --- README.md | 30 ++-- ballerina/README.md | 30 ++-- docs/spec/spec.md | 34 ++-- .../stdlib/log/MaskedStringBuilder.java | 167 ++++++++++++++++-- 4 files changed, 200 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index c01725df..5c5d7b54 100644 --- a/README.md +++ b/README.md @@ -131,21 +131,21 @@ For more details and advanced usage, see the module specification and API docume The log module provides capabilities to mask sensitive data in log messages to maintain data privacy and security when dealing with personally identifiable information (PII) or other sensitive data. -By default, sensitive data masking is disabled. Enable it in `Config.toml`: - -```toml -[ballerina.log] -enableSensitiveDataMasking = true -``` - -Or configure it per logger: - -```ballerina -log:Config secureConfig = { - enableSensitiveDataMasking: true -}; -log:Logger secureLogger = log:fromConfig(secureConfig); -``` +> **Note**: By default, sensitive data masking is disabled. Enable it in `Config.toml`: +> +> ```toml +> [ballerina.log] +> enableSensitiveDataMasking = true +> ``` +> +> Or configure it per logger: +> +> ```ballerina +> log:Config secureConfig = { +> enableSensitiveDataMasking: true +> }; +> log:Logger secureLogger = log:fromConfig(secureConfig); +> ``` ### Sensitive Data Annotation diff --git a/ballerina/README.md b/ballerina/README.md index 683d4c77..ad840fc6 100644 --- a/ballerina/README.md +++ b/ballerina/README.md @@ -118,21 +118,21 @@ For more details and advanced usage, see the module specification and API docume The log module provides capabilities to mask sensitive data in log messages to maintain data privacy and security when dealing with personally identifiable information (PII) or other sensitive data. -By default, sensitive data masking is disabled. Enable it in `Config.toml`: - -```toml -[ballerina.log] -enableSensitiveDataMasking = true -``` - -Or configure it per logger: - -```ballerina -log:Config secureConfig = { - enableSensitiveDataMasking: true -}; -log:Logger secureLogger = log:fromConfig(secureConfig); -``` +> **Note**: By default, sensitive data masking is disabled. Enable it in `Config.toml`: +> +> ```toml +> [ballerina.log] +> enableSensitiveDataMasking = true +> ``` +> +> Or configure it per logger: +> +> ```ballerina +> log:Config secureConfig = { +> enableSensitiveDataMasking: true +> }; +> log:Logger secureLogger = log:fromConfig(secureConfig); +> ``` ### Sensitive Data Annotation diff --git a/docs/spec/spec.md b/docs/spec/spec.md index 926c64c0..a84a2ab1 100644 --- a/docs/spec/spec.md +++ b/docs/spec/spec.md @@ -305,28 +305,26 @@ auditLogger.printInfo("Hello World from the audit logger!"); The Ballerina log module provides the capability to mask sensitive data in log messages. This is crucial for maintaining data privacy and security, especially when dealing with personally identifiable information (PII) or other sensitive data. -By default, sensitive data masking is disabled. Enable it in `Config.toml`: - -```toml -[ballerina.log] -enableSensitiveDataMasking = true -``` - -Or configure it per logger: - -```ballerina -log:Config secureConfig = { - enableSensitiveDataMasking: true -}; -log:Logger secureLogger = log:fromConfig(secureConfig); -``` +> **Note**: By default, sensitive data masking is disabled. Enable it in `Config.toml`: +> +> ```toml +> [ballerina.log] +> enableSensitiveDataMasking = true +> ``` +> +> Or configure it per logger: +> +> ```ballerina +> log:Config secureConfig = { +> enableSensitiveDataMasking: true +> }; +> log:Logger secureLogger = log:fromConfig(secureConfig); +> ``` ### 5.1. Sensitive data annotation The `@log:Sensitive` annotation can be used to mark fields in a record as sensitive. When such fields are logged, their values will be excluded or masked to prevent exposure of sensitive information. - - ```ballerina import ballerina/log; @@ -349,7 +347,7 @@ Output: time=2025-08-20T09:15:30.123+05:30 level=INFO module="" message="user details" user={"id":"U001","name":"John Doe"} ``` -By default, the `@log:Sensitive` annotation will exclude the sensitive field from the log output when sensitive data masking is enabled. +The `@log:Sensitive` annotation will exclude the sensitive field from the log output when sensitive data masking is enabled. Additionally, the masking strategy can be configured using the `strategy` field of the annotation. The available strategies are: 1. `EXCLUDE`: Excludes the field from the log output (default behavior). diff --git a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java index d18508f7..4466f235 100644 --- a/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java +++ b/native/src/main/java/io/ballerina/stdlib/log/MaskedStringBuilder.java @@ -37,11 +37,12 @@ import io.ballerina.runtime.api.values.BXml; import java.util.Collection; +import java.util.HashMap; import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; /** @@ -64,9 +65,8 @@ public class MaskedStringBuilder implements AutoCloseable { public static final BString MASKED_STRING_BUILDER_HAS_BEEN_CLOSED = StringUtils.fromString("MaskedStringBuilder" + " has been closed"); - // Cache for field annotations to avoid repeated extraction - private static final Map>> ANNOTATION_CACHE = new ConcurrentHashMap<>(); - private static final int MAX_CACHE_SIZE = 1000; + // Thread-safe LRU cache for field annotations to avoid repeated extraction + private static final LRUCache>> ANNOTATION_CACHE = new LRUCache<>(1000); // Pre-computed hex lookup table for efficient Unicode escaping private static final char[] HEX_CHARS = "0123456789abcdef".toCharArray(); @@ -328,7 +328,7 @@ private void appendEscapedString(String input) { /** * Get cached field annotations for better performance. - * Implements simple cache eviction when cache grows too large. + * Uses LRU cache which automatically handles eviction of least recently used entries. */ private Map> getCachedFieldAnnotations(RecordType recordType) { Map> cached = ANNOTATION_CACHE.get(recordType); @@ -336,12 +336,6 @@ private void appendEscapedString(String input) { return cached; } - // Implement simple cache size management - if (ANNOTATION_CACHE.size() >= MAX_CACHE_SIZE) { - // Clear half the cache when it gets too large (simple eviction strategy) - ANNOTATION_CACHE.entrySet().removeIf(entry -> System.identityHashCode(entry.getKey()) % 2 == 0); - } - Map> annotations = extractFieldAnnotations(recordType); ANNOTATION_CACHE.put(recordType, annotations); return annotations; @@ -375,7 +369,7 @@ private String processTableValue(BTable tableValue) { private String processArrayValue(BArray listValue) { long length = listValue.getLength(); - if (length == 0) { + if (listValue.isEmpty()) { return "[]"; } @@ -414,7 +408,7 @@ private void appendValueToArray(String value, Object originalValue) { * Check if a value is a basic type that doesn't need complex processing. */ private static boolean isBasicType(Object value) { - return value == null || TypeUtils.getType(value).getTag() <= TypeTags.BOOLEAN_TAG; + return value == null || TypeUtils.getType(value).getTag() < TypeTags.NULL_TAG; } /** @@ -524,6 +518,9 @@ public static MaskedStringBuilder create(Runtime runtime, int initialCapacity) { for (Object key : keys) { if (key instanceof BString bStringKey) { String keyValue = bStringKey.getValue(); + // No runtime API to get the annotation from the org name, package name and annotation name + // But the annotation key is in the format: "/::" + // This even works when the package is imported using alias since runtime is using the package name if (keyValue.endsWith(SENSITIVE_SUFFIX) && keyValue.startsWith(LOG_ANNOTATION_PREFIX)) { Object annotation = fieldAnnotationMap.get(key); if (annotation instanceof BMap bMapAnnotation) { @@ -575,4 +572,148 @@ static Optional getStringValue(BMap annotation, Object realValue, } return Optional.of(StringUtils.getStringValue(realValue)); } + + /** + * Thread-safe LRU cache implementation using HashMap + Doubly Linked List. + * + * @param the type of keys maintained by this cache + * @param the type of cached values + */ + private static class LRUCache { + private final int maxSize; + private final Map> cache; + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private final ReentrantReadWriteLock.ReadLock readLock = lock.readLock(); + private final ReentrantReadWriteLock.WriteLock writeLock = lock.writeLock(); + + // Dummy head and tail nodes for the doubly linked list + private final Node head; + private final Node tail; + + /** + * Node class for the doubly linked list. + * + * @param the type of keys + * @param the type of values + */ + private static class Node { + K key; + V value; + Node prev; + Node next; + + Node() { + // Constructor for dummy nodes + } + + Node(K key, V value) { + this.key = key; + this.value = value; + } + } + + public LRUCache(int maxSize) { + this.maxSize = maxSize; + this.cache = new HashMap<>(); + + // Initialize dummy head and tail nodes + this.head = new Node<>(); + this.tail = new Node<>(); + this.head.next = this.tail; + this.tail.prev = this.head; + } + + public V get(K key) { + writeLock.lock(); + try { + Node node = cache.get(key); + if (node == null) { + return null; + } + + // Move to head (most recently used) + moveToHead(node); + return node.value; + } finally { + writeLock.unlock(); + } + } + + public V put(K key, V value) { + writeLock.lock(); + try { + Node existingNode = cache.get(key); + + if (existingNode != null) { + // Update existing node + V oldValue = existingNode.value; + existingNode.value = value; + moveToHead(existingNode); + return oldValue; + } else { + // Add new node + Node newNode = new Node<>(key, value); + + if (cache.size() >= maxSize) { + // Remove least recently used node (tail.prev) + Node lru = tail.prev; + removeNode(lru); + cache.remove(lru.key); + } + + cache.put(key, newNode); + addToHead(newNode); + return null; + } + } finally { + writeLock.unlock(); + } + } + + public void clear() { + writeLock.lock(); + try { + cache.clear(); + head.next = tail; + tail.prev = head; + } finally { + writeLock.unlock(); + } + } + + public int size() { + readLock.lock(); + try { + return cache.size(); + } finally { + readLock.unlock(); + } + } + + /** + * Add node right after head (most recently used position). + */ + private void addToHead(Node node) { + node.prev = head; + node.next = head.next; + head.next.prev = node; + head.next = node; + } + + /** + * Remove a node from the doubly linked list. + */ + private void removeNode(Node node) { + node.prev.next = node.next; + node.next.prev = node.prev; + } + + /** + * Move a node to head (most recently used position). + */ + private void moveToHead(Node node) { + removeNode(node); + addToHead(node); + } + } } From 42156dd6182cd47e2725bd246574314da8c62599 Mon Sep 17 00:00:00 2001 From: Danesh Kuruppu Date: Wed, 8 Oct 2025 17:19:45 +0530 Subject: [PATCH 34/34] Update integration-tests/tests/resources/samples/masked-logger/Config.toml --- .../tests/resources/samples/masked-logger/Config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/tests/resources/samples/masked-logger/Config.toml b/integration-tests/tests/resources/samples/masked-logger/Config.toml index 1e78c430..936f462f 100644 --- a/integration-tests/tests/resources/samples/masked-logger/Config.toml +++ b/integration-tests/tests/resources/samples/masked-logger/Config.toml @@ -1,3 +1,3 @@ [ballerina.log] enableSensitiveDataMasking = true -level = "DEBUG" \ No newline at end of file +level = "DEBUG"